platform.h 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572
  1. /* Copyright 2016 Google Inc. All Rights Reserved.
  2. Distributed under MIT license.
  3. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  4. */
  5. /* Macros for compiler / platform specific features and build options.
  6. Build options are:
  7. * BROTLI_BUILD_32_BIT disables 64-bit optimizations
  8. * BROTLI_BUILD_64_BIT forces to use 64-bit optimizations
  9. * BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations
  10. * BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations
  11. * BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations
  12. * BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
  13. * BROTLI_BUILD_NO_UNALIGNED_READ_FAST forces off the fast-unaligned-read
  14. optimizations (mainly for testing purposes)
  15. * BROTLI_DEBUG dumps file name and line number when decoder detects stream
  16. or memory error
  17. * BROTLI_ENABLE_LOG enables asserts and dumps various state information
  18. * BROTLI_ENABLE_DUMP overrides default "dump" behaviour
  19. */
  20. #ifndef BROTLI_COMMON_PLATFORM_H_
  21. #define BROTLI_COMMON_PLATFORM_H_
  22. #include <string.h> /* memcpy */
  23. #include <brotli/port.h>
  24. #include <brotli/types.h>
  25. #include <sys/types.h> /* should include endian.h for us */
  26. #if BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
  27. #include <intrin.h>
  28. #endif
  29. #if defined(BROTLI_ENABLE_LOG) || defined(BROTLI_DEBUG)
  30. #include <assert.h>
  31. #include <stdio.h>
  32. #endif
  33. /* The following macros were borrowed from https://github.com/nemequ/hedley
  34. * with permission of original author - Evan Nemerson <evan@nemerson.com> */
  35. /* >>> >>> >>> hedley macros */
  36. /* Define "BROTLI_PREDICT_TRUE" and "BROTLI_PREDICT_FALSE" macros for capable
  37. compilers.
  38. To apply compiler hint, enclose the branching condition into macros, like this:
  39. if (BROTLI_PREDICT_TRUE(zero == 0)) {
  40. // main execution path
  41. } else {
  42. // compiler should place this code outside of main execution path
  43. }
  44. OR:
  45. if (BROTLI_PREDICT_FALSE(something_rare_or_unexpected_happens)) {
  46. // compiler should place this code outside of main execution path
  47. }
  48. */
  49. #if BROTLI_GNUC_HAS_BUILTIN(__builtin_expect, 3, 0, 0) || \
  50. BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
  51. BROTLI_SUNPRO_VERSION_CHECK(5, 15, 0) || \
  52. BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
  53. BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
  54. BROTLI_TI_VERSION_CHECK(7, 3, 0) || \
  55. BROTLI_TINYC_VERSION_CHECK(0, 9, 27)
  56. #define BROTLI_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
  57. #define BROTLI_PREDICT_FALSE(x) (__builtin_expect(x, 0))
  58. #else
  59. #define BROTLI_PREDICT_FALSE(x) (x)
  60. #define BROTLI_PREDICT_TRUE(x) (x)
  61. #endif
  62. #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
  63. !defined(__cplusplus)
  64. #define BROTLI_RESTRICT restrict
  65. #elif BROTLI_GNUC_VERSION_CHECK(3, 1, 0) || \
  66. BROTLI_MSVC_VERSION_CHECK(14, 0, 0) || \
  67. BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
  68. BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
  69. BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
  70. BROTLI_PGI_VERSION_CHECK(17, 10, 0) || \
  71. BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
  72. BROTLI_IAR_VERSION_CHECK(8, 0, 0) || \
  73. (BROTLI_SUNPRO_VERSION_CHECK(5, 14, 0) && defined(__cplusplus))
  74. #define BROTLI_RESTRICT __restrict
  75. #elif BROTLI_SUNPRO_VERSION_CHECK(5, 3, 0) && !defined(__cplusplus)
  76. #define BROTLI_RESTRICT _Restrict
  77. #else
  78. #define BROTLI_RESTRICT
  79. #endif
  80. #if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \
  81. (defined(__cplusplus) && (__cplusplus >= 199711L))
  82. #define BROTLI_MAYBE_INLINE inline
  83. #elif defined(__GNUC_STDC_INLINE__) || defined(__GNUC_GNU_INLINE__) || \
  84. BROTLI_ARM_VERSION_CHECK(6, 2, 0)
  85. #define BROTLI_MAYBE_INLINE __inline__
  86. #elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0) || \
  87. BROTLI_ARM_VERSION_CHECK(4, 1, 0) || BROTLI_TI_VERSION_CHECK(8, 0, 0)
  88. #define BROTLI_MAYBE_INLINE __inline
  89. #else
  90. #define BROTLI_MAYBE_INLINE
  91. #endif
  92. #if BROTLI_GNUC_HAS_ATTRIBUTE(always_inline, 4, 0, 0) || \
  93. BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
  94. BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
  95. BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
  96. BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
  97. BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
  98. (BROTLI_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__))
  99. #define BROTLI_INLINE BROTLI_MAYBE_INLINE __attribute__((__always_inline__))
  100. #elif BROTLI_MSVC_VERSION_CHECK(12, 0, 0)
  101. #define BROTLI_INLINE BROTLI_MAYBE_INLINE __forceinline
  102. #elif BROTLI_TI_VERSION_CHECK(7, 0, 0) && defined(__cplusplus)
  103. #define BROTLI_INLINE BROTLI_MAYBE_INLINE _Pragma("FUNC_ALWAYS_INLINE;")
  104. #elif BROTLI_IAR_VERSION_CHECK(8, 0, 0)
  105. #define BROTLI_INLINE BROTLI_MAYBE_INLINE _Pragma("inline=forced")
  106. #else
  107. #define BROTLI_INLINE BROTLI_MAYBE_INLINE
  108. #endif
  109. #if BROTLI_GNUC_HAS_ATTRIBUTE(noinline, 4, 0, 0) || \
  110. BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
  111. BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
  112. BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
  113. BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
  114. BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
  115. (BROTLI_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__))
  116. #define BROTLI_NOINLINE __attribute__((__noinline__))
  117. #elif BROTLI_MSVC_VERSION_CHECK(13, 10, 0)
  118. #define BROTLI_NOINLINE __declspec(noinline)
  119. #elif BROTLI_PGI_VERSION_CHECK(10, 2, 0)
  120. #define BROTLI_NOINLINE _Pragma("noinline")
  121. #elif BROTLI_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus)
  122. #define BROTLI_NOINLINE _Pragma("FUNC_CANNOT_INLINE;")
  123. #elif BROTLI_IAR_VERSION_CHECK(8, 0, 0)
  124. #define BROTLI_NOINLINE _Pragma("inline=never")
  125. #else
  126. #define BROTLI_NOINLINE
  127. #endif
  128. /* <<< <<< <<< end of hedley macros. */
  129. #if BROTLI_GNUC_HAS_ATTRIBUTE(unused, 2, 7, 0) || \
  130. BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
  131. #define BROTLI_UNUSED_FUNCTION static BROTLI_INLINE __attribute__ ((unused))
  132. #else
  133. #define BROTLI_UNUSED_FUNCTION static BROTLI_INLINE
  134. #endif
  135. #if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)
  136. #define BROTLI_ALIGNED(N) __attribute__((aligned(N)))
  137. #else
  138. #define BROTLI_ALIGNED(N)
  139. #endif
  140. #if (defined(__ARM_ARCH) && (__ARM_ARCH == 7)) || \
  141. (defined(M_ARM) && (M_ARM == 7))
  142. #define BROTLI_TARGET_ARMV7
  143. #endif /* ARMv7 */
  144. #if (defined(__ARM_ARCH) && (__ARM_ARCH == 8)) || \
  145. defined(__aarch64__) || defined(__ARM64_ARCH_8__)
  146. #define BROTLI_TARGET_ARMV8_ANY
  147. #if defined(__ARM_32BIT_STATE)
  148. #define BROTLI_TARGET_ARMV8_32
  149. #elif defined(__ARM_64BIT_STATE)
  150. #define BROTLI_TARGET_ARMV8_64
  151. #endif
  152. #endif /* ARMv8 */
  153. #if defined(__ARM_NEON__) || defined(__ARM_NEON)
  154. #define BROTLI_TARGET_NEON
  155. #endif
  156. #if defined(__i386) || defined(_M_IX86)
  157. #define BROTLI_TARGET_X86
  158. #endif
  159. #if defined(__x86_64__) || defined(_M_X64)
  160. #define BROTLI_TARGET_X64
  161. #endif
  162. #if defined(__PPC64__)
  163. #define BROTLI_TARGET_POWERPC64
  164. #endif
  165. #if defined(__riscv) && defined(__riscv_xlen) && __riscv_xlen == 64
  166. #define BROTLI_TARGET_RISCV64
  167. #endif
  168. #if defined(__loongarch_lp64)
  169. #define BROTLI_TARGET_LOONGARCH64
  170. #endif
  171. #if defined(BROTLI_TARGET_X64) || defined(BROTLI_TARGET_ARMV8_64) || \
  172. defined(BROTLI_TARGET_POWERPC64) || defined(BROTLI_TARGET_RISCV64) || \
  173. defined(BROTLI_TARGET_LOONGARCH64)
  174. #define BROTLI_TARGET_64_BITS 1
  175. #else
  176. #define BROTLI_TARGET_64_BITS 0
  177. #endif
  178. #if defined(BROTLI_BUILD_64_BIT)
  179. #define BROTLI_64_BITS 1
  180. #elif defined(BROTLI_BUILD_32_BIT)
  181. #define BROTLI_64_BITS 0
  182. #else
  183. #define BROTLI_64_BITS BROTLI_TARGET_64_BITS
  184. #endif
  185. #if (BROTLI_64_BITS)
  186. #define brotli_reg_t uint64_t
  187. #else
  188. #define brotli_reg_t uint32_t
  189. #endif
  190. #if defined(BROTLI_BUILD_BIG_ENDIAN)
  191. #define BROTLI_BIG_ENDIAN 1
  192. #elif defined(BROTLI_BUILD_LITTLE_ENDIAN)
  193. #define BROTLI_LITTLE_ENDIAN 1
  194. #elif defined(BROTLI_BUILD_ENDIAN_NEUTRAL)
  195. /* Just break elif chain. */
  196. #elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
  197. #define BROTLI_LITTLE_ENDIAN 1
  198. #elif defined(_WIN32) || defined(BROTLI_TARGET_X64)
  199. /* Win32 & x64 can currently always be assumed to be little endian */
  200. #define BROTLI_LITTLE_ENDIAN 1
  201. #elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
  202. #define BROTLI_BIG_ENDIAN 1
  203. /* Likely target platform is iOS / OSX. */
  204. #elif defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN)
  205. #define BROTLI_LITTLE_ENDIAN 1
  206. #elif defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN)
  207. #define BROTLI_BIG_ENDIAN 1
  208. #endif
  209. #if !defined(BROTLI_LITTLE_ENDIAN)
  210. #define BROTLI_LITTLE_ENDIAN 0
  211. #endif
  212. #if !defined(BROTLI_BIG_ENDIAN)
  213. #define BROTLI_BIG_ENDIAN 0
  214. #endif
  215. #if defined(BROTLI_BUILD_NO_UNALIGNED_READ_FAST)
  216. #define BROTLI_UNALIGNED_READ_FAST (!!0)
  217. #elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
  218. defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY) || \
  219. defined(BROTLI_TARGET_RISCV64) || defined(BROTLI_TARGET_LOONGARCH64)
  220. /* These targets are known to generate efficient code for unaligned reads
  221. * (e.g. a single instruction, not multiple 1-byte loads, shifted and or'd
  222. * together). */
  223. #define BROTLI_UNALIGNED_READ_FAST (!!1)
  224. #else
  225. #define BROTLI_UNALIGNED_READ_FAST (!!0)
  226. #endif
  227. /* Portable unaligned memory access: read / write values via memcpy. */
  228. static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
  229. uint16_t t;
  230. memcpy(&t, p, sizeof t);
  231. return t;
  232. }
  233. static BROTLI_INLINE uint32_t BrotliUnalignedRead32(const void* p) {
  234. uint32_t t;
  235. memcpy(&t, p, sizeof t);
  236. return t;
  237. }
  238. static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
  239. uint64_t t;
  240. memcpy(&t, p, sizeof t);
  241. return t;
  242. }
  243. static BROTLI_INLINE size_t BrotliUnalignedReadSizeT(const void* p) {
  244. size_t t;
  245. memcpy(&t, p, sizeof t);
  246. return t;
  247. }
  248. static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
  249. memcpy(p, &v, sizeof v);
  250. }
  251. #if BROTLI_LITTLE_ENDIAN
  252. /* Straight endianness. Just read / write values. */
  253. #define BROTLI_UNALIGNED_LOAD16LE BrotliUnalignedRead16
  254. #define BROTLI_UNALIGNED_LOAD32LE BrotliUnalignedRead32
  255. #define BROTLI_UNALIGNED_LOAD64LE BrotliUnalignedRead64
  256. #define BROTLI_UNALIGNED_STORE64LE BrotliUnalignedWrite64
  257. #elif BROTLI_BIG_ENDIAN /* BROTLI_LITTLE_ENDIAN */
  258. /* Explain compiler to byte-swap values. */
  259. #define BROTLI_BSWAP16_(V) ((uint16_t)( \
  260. (((V) & 0xFFU) << 8) | \
  261. (((V) >> 8) & 0xFFU)))
  262. static BROTLI_INLINE uint16_t BROTLI_UNALIGNED_LOAD16LE(const void* p) {
  263. uint16_t value = BrotliUnalignedRead16(p);
  264. return BROTLI_BSWAP16_(value);
  265. }
  266. #define BROTLI_BSWAP32_(V) ( \
  267. (((V) & 0xFFU) << 24) | (((V) & 0xFF00U) << 8) | \
  268. (((V) >> 8) & 0xFF00U) | (((V) >> 24) & 0xFFU))
  269. static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32LE(const void* p) {
  270. uint32_t value = BrotliUnalignedRead32(p);
  271. return BROTLI_BSWAP32_(value);
  272. }
  273. #define BROTLI_BSWAP64_(V) ( \
  274. (((V) & 0xFFU) << 56) | (((V) & 0xFF00U) << 40) | \
  275. (((V) & 0xFF0000U) << 24) | (((V) & 0xFF000000U) << 8) | \
  276. (((V) >> 8) & 0xFF000000U) | (((V) >> 24) & 0xFF0000U) | \
  277. (((V) >> 40) & 0xFF00U) | (((V) >> 56) & 0xFFU))
  278. static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void* p) {
  279. uint64_t value = BrotliUnalignedRead64(p);
  280. return BROTLI_BSWAP64_(value);
  281. }
  282. static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
  283. uint64_t value = BROTLI_BSWAP64_(v);
  284. BrotliUnalignedWrite64(p, value);
  285. }
  286. #else /* BROTLI_LITTLE_ENDIAN */
  287. /* Read / store values byte-wise; hopefully compiler will understand. */
  288. static BROTLI_INLINE uint16_t BROTLI_UNALIGNED_LOAD16LE(const void* p) {
  289. const uint8_t* in = (const uint8_t*)p;
  290. return (uint16_t)(in[0] | (in[1] << 8));
  291. }
  292. static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32LE(const void* p) {
  293. const uint8_t* in = (const uint8_t*)p;
  294. uint32_t value = (uint32_t)(in[0]);
  295. value |= (uint32_t)(in[1]) << 8;
  296. value |= (uint32_t)(in[2]) << 16;
  297. value |= (uint32_t)(in[3]) << 24;
  298. return value;
  299. }
  300. static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void* p) {
  301. const uint8_t* in = (const uint8_t*)p;
  302. uint64_t value = (uint64_t)(in[0]);
  303. value |= (uint64_t)(in[1]) << 8;
  304. value |= (uint64_t)(in[2]) << 16;
  305. value |= (uint64_t)(in[3]) << 24;
  306. value |= (uint64_t)(in[4]) << 32;
  307. value |= (uint64_t)(in[5]) << 40;
  308. value |= (uint64_t)(in[6]) << 48;
  309. value |= (uint64_t)(in[7]) << 56;
  310. return value;
  311. }
  312. static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void* p, uint64_t v) {
  313. uint8_t* out = (uint8_t*)p;
  314. out[0] = (uint8_t)v;
  315. out[1] = (uint8_t)(v >> 8);
  316. out[2] = (uint8_t)(v >> 16);
  317. out[3] = (uint8_t)(v >> 24);
  318. out[4] = (uint8_t)(v >> 32);
  319. out[5] = (uint8_t)(v >> 40);
  320. out[6] = (uint8_t)(v >> 48);
  321. out[7] = (uint8_t)(v >> 56);
  322. }
  323. #endif /* BROTLI_LITTLE_ENDIAN */
  324. static BROTLI_INLINE void* BROTLI_UNALIGNED_LOAD_PTR(const void* p) {
  325. void* v;
  326. memcpy(&v, p, sizeof(void*));
  327. return v;
  328. }
  329. static BROTLI_INLINE void BROTLI_UNALIGNED_STORE_PTR(void* p, const void* v) {
  330. memcpy(p, &v, sizeof(void*));
  331. }
  332. /* BROTLI_IS_CONSTANT macros returns true for compile-time constants. */
  333. #if BROTLI_GNUC_HAS_BUILTIN(__builtin_constant_p, 3, 0, 1) || \
  334. BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
  335. #define BROTLI_IS_CONSTANT(x) (!!__builtin_constant_p(x))
  336. #else
  337. #define BROTLI_IS_CONSTANT(x) (!!0)
  338. #endif
  339. #if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
  340. #define BROTLI_HAS_UBFX (!!1)
  341. #else
  342. #define BROTLI_HAS_UBFX (!!0)
  343. #endif
  344. #if defined(BROTLI_ENABLE_LOG)
  345. #define BROTLI_LOG(x) printf x
  346. #else
  347. #define BROTLI_LOG(x)
  348. #endif
  349. #if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
  350. #define BROTLI_ENABLE_DUMP_DEFAULT 1
  351. #define BROTLI_DCHECK(x) assert(x)
  352. #else
  353. #define BROTLI_ENABLE_DUMP_DEFAULT 0
  354. #define BROTLI_DCHECK(x)
  355. #endif
  356. #if !defined(BROTLI_ENABLE_DUMP)
  357. #define BROTLI_ENABLE_DUMP BROTLI_ENABLE_DUMP_DEFAULT
  358. #endif
  359. #if BROTLI_ENABLE_DUMP
  360. static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
  361. fprintf(stderr, "%s:%d (%s)\n", f, l, fn);
  362. fflush(stderr);
  363. }
  364. #define BROTLI_DUMP() BrotliDump(__FILE__, __LINE__, __FUNCTION__)
  365. #else
  366. #define BROTLI_DUMP() (void)(0)
  367. #endif
  368. /* BrotliRBit assumes brotli_reg_t fits native CPU register type. */
  369. #if (BROTLI_64_BITS == BROTLI_TARGET_64_BITS)
  370. /* TODO(eustas): add appropriate icc/sunpro/arm/ibm/ti checks. */
  371. #if (BROTLI_GNUC_VERSION_CHECK(3, 0, 0) || defined(__llvm__)) && \
  372. !defined(BROTLI_BUILD_NO_RBIT)
  373. #if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
  374. /* TODO(eustas): detect ARMv6T2 and enable this code for it. */
  375. static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) {
  376. brotli_reg_t output;
  377. __asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));
  378. return output;
  379. }
  380. #define BROTLI_RBIT(x) BrotliRBit(x)
  381. #endif /* armv7 / armv8 */
  382. #endif /* gcc || clang */
  383. #endif /* brotli_reg_t is native */
  384. #if !defined(BROTLI_RBIT)
  385. static BROTLI_INLINE void BrotliRBit(void) { /* Should break build if used. */ }
  386. #endif /* BROTLI_RBIT */
  387. #define BROTLI_REPEAT_4(X) {X; X; X; X;}
  388. #define BROTLI_REPEAT_5(X) {X; X; X; X; X;}
  389. #define BROTLI_REPEAT_6(X) {X; X; X; X; X; X;}
  390. #define BROTLI_UNUSED(X) (void)(X)
  391. #define BROTLI_MIN_MAX(T) \
  392. static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \
  393. static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; }
  394. BROTLI_MIN_MAX(double) BROTLI_MIN_MAX(float) BROTLI_MIN_MAX(int)
  395. BROTLI_MIN_MAX(size_t) BROTLI_MIN_MAX(uint32_t) BROTLI_MIN_MAX(uint8_t)
  396. #undef BROTLI_MIN_MAX
  397. #define BROTLI_MIN(T, A, B) (brotli_min_ ## T((A), (B)))
  398. #define BROTLI_MAX(T, A, B) (brotli_max_ ## T((A), (B)))
  399. #define BROTLI_SWAP(T, A, I, J) { \
  400. T __brotli_swap_tmp = (A)[(I)]; \
  401. (A)[(I)] = (A)[(J)]; \
  402. (A)[(J)] = __brotli_swap_tmp; \
  403. }
  404. #if BROTLI_64_BITS
  405. #if BROTLI_GNUC_HAS_BUILTIN(__builtin_ctzll, 3, 4, 0) || \
  406. BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
  407. #define BROTLI_TZCNT64 __builtin_ctzll
  408. #elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
  409. #if defined(BROTLI_TARGET_X64) && !defined(_M_ARM64EC)
  410. #define BROTLI_TZCNT64 _tzcnt_u64
  411. #else /* BROTLI_TARGET_X64 */
  412. static BROTLI_INLINE uint32_t BrotliBsf64Msvc(uint64_t x) {
  413. uint32_t lsb;
  414. _BitScanForward64(&lsb, x);
  415. return lsb;
  416. }
  417. #define BROTLI_TZCNT64 BrotliBsf64Msvc
  418. #endif /* BROTLI_TARGET_X64 */
  419. #endif /* __builtin_ctzll */
  420. #endif /* BROTLI_64_BITS */
  421. #if BROTLI_GNUC_HAS_BUILTIN(__builtin_clz, 3, 4, 0) || \
  422. BROTLI_INTEL_VERSION_CHECK(16, 0, 0)
  423. #define BROTLI_BSR32(x) (31u ^ (uint32_t)__builtin_clz(x))
  424. #elif BROTLI_MSVC_VERSION_CHECK(18, 0, 0)
  425. static BROTLI_INLINE uint32_t BrotliBsr32Msvc(uint32_t x) {
  426. unsigned long msb;
  427. _BitScanReverse(&msb, x);
  428. return (uint32_t)msb;
  429. }
  430. #define BROTLI_BSR32 BrotliBsr32Msvc
  431. #endif /* __builtin_clz */
  432. /* Default brotli_alloc_func */
  433. BROTLI_COMMON_API void* BrotliDefaultAllocFunc(void* opaque, size_t size);
  434. /* Default brotli_free_func */
  435. BROTLI_COMMON_API void BrotliDefaultFreeFunc(void* opaque, void* address);
  436. /* Circular logical rotates. */
  437. static BROTLI_INLINE uint16_t BrotliRotateRight16(uint16_t const value,
  438. size_t count) {
  439. count &= 0x0F; /* for fickle pattern recognition */
  440. return (value >> count) | (uint16_t)(value << ((0U - count) & 0x0F));
  441. }
  442. static BROTLI_INLINE uint32_t BrotliRotateRight32(uint32_t const value,
  443. size_t count) {
  444. count &= 0x1F; /* for fickle pattern recognition */
  445. return (value >> count) | (uint32_t)(value << ((0U - count) & 0x1F));
  446. }
  447. static BROTLI_INLINE uint64_t BrotliRotateRight64(uint64_t const value,
  448. size_t count) {
  449. count &= 0x3F; /* for fickle pattern recognition */
  450. return (value >> count) | (uint64_t)(value << ((0U - count) & 0x3F));
  451. }
  452. BROTLI_UNUSED_FUNCTION void BrotliSuppressUnusedFunctions(void) {
  453. BROTLI_UNUSED(&BrotliSuppressUnusedFunctions);
  454. BROTLI_UNUSED(&BrotliUnalignedRead16);
  455. BROTLI_UNUSED(&BrotliUnalignedRead32);
  456. BROTLI_UNUSED(&BrotliUnalignedRead64);
  457. BROTLI_UNUSED(&BrotliUnalignedReadSizeT);
  458. BROTLI_UNUSED(&BrotliUnalignedWrite64);
  459. BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD16LE);
  460. BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD32LE);
  461. BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD64LE);
  462. BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE64LE);
  463. BROTLI_UNUSED(&BROTLI_UNALIGNED_LOAD_PTR);
  464. BROTLI_UNUSED(&BROTLI_UNALIGNED_STORE_PTR);
  465. BROTLI_UNUSED(&BrotliRBit);
  466. BROTLI_UNUSED(&brotli_min_double);
  467. BROTLI_UNUSED(&brotli_max_double);
  468. BROTLI_UNUSED(&brotli_min_float);
  469. BROTLI_UNUSED(&brotli_max_float);
  470. BROTLI_UNUSED(&brotli_min_int);
  471. BROTLI_UNUSED(&brotli_max_int);
  472. BROTLI_UNUSED(&brotli_min_size_t);
  473. BROTLI_UNUSED(&brotli_max_size_t);
  474. BROTLI_UNUSED(&brotli_min_uint32_t);
  475. BROTLI_UNUSED(&brotli_max_uint32_t);
  476. BROTLI_UNUSED(&brotli_min_uint8_t);
  477. BROTLI_UNUSED(&brotli_max_uint8_t);
  478. BROTLI_UNUSED(&BrotliDefaultAllocFunc);
  479. BROTLI_UNUSED(&BrotliDefaultFreeFunc);
  480. BROTLI_UNUSED(&BrotliRotateRight16);
  481. BROTLI_UNUSED(&BrotliRotateRight32);
  482. BROTLI_UNUSED(&BrotliRotateRight64);
  483. #if BROTLI_ENABLE_DUMP
  484. BROTLI_UNUSED(&BrotliDump);
  485. #endif
  486. #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */
  487. # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
  488. # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
  489. # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
  490. #elif BROTLI_GNUC_HAS_BUILTIN(__builtin_prefetch, 3, 1, 0)
  491. # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
  492. # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
  493. #elif defined(__aarch64__)
  494. # define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0)
  495. # define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0)
  496. #else
  497. # define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */
  498. # define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */
  499. #endif
  500. /* The SIMD matchers are only faster at certain quality levels. */
  501. #if defined(_M_X64) && defined(BROTLI_TZCNT64)
  502. #define BROTLI_MAX_SIMD_QUALITY 7
  503. #elif defined(BROTLI_TZCNT64)
  504. #define BROTLI_MAX_SIMD_QUALITY 6
  505. #endif
  506. }
  507. #endif /* BROTLI_COMMON_PLATFORM_H_ */