encoder_dict.h 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. /* Copyright 2017 Google Inc. All Rights Reserved.
  2. Distributed under MIT license.
  3. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  4. */
  5. #ifndef BROTLI_ENC_ENCODER_DICT_H_
  6. #define BROTLI_ENC_ENCODER_DICT_H_
  7. #include <brotli/shared_dictionary.h>
  8. #include <brotli/types.h>
  9. #include "../common/dictionary.h"
  10. #include "../common/platform.h"
  11. #include "compound_dictionary.h"
  12. #include "memory.h"
  13. #include "static_dict_lut.h"
  14. #if defined(__cplusplus) || defined(c_plusplus)
  15. extern "C" {
  16. #endif
  17. /*
  18. Dictionary hierarchy for Encoder:
  19. -SharedEncoderDictionary
  20. --CompoundDictionary
  21. ---PreparedDictionary [up to 15x]
  22. = prefix dictionary with precomputed hashes
  23. --ContextualEncoderDictionary
  24. ---BrotliEncoderDictionary [up to 64x]
  25. = for each context, precomputed static dictionary with words + transforms
  26. Dictionary hiearchy from common: similar, but without precomputed hashes
  27. -BrotliSharedDictionary
  28. --BrotliDictionary [up to 64x]
  29. --BrotliTransforms [up to 64x]
  30. --const uint8_t* prefix [up to 15x]: compound dictionaries
  31. */
  32. typedef struct BrotliTrieNode {
  33. uint8_t single; /* if 1, sub is a single node for c instead of 256 */
  34. uint8_t c;
  35. uint8_t len_; /* untransformed length */
  36. uint32_t idx_; /* word index + num words * transform index */
  37. uint32_t sub; /* index of sub node(s) in the pool */
  38. } BrotliTrieNode;
  39. typedef struct BrotliTrie {
  40. BrotliTrieNode* pool;
  41. size_t pool_capacity;
  42. size_t pool_size;
  43. BrotliTrieNode root;
  44. } BrotliTrie;
  45. #if defined(BROTLI_EXPERIMENTAL)
  46. BROTLI_INTERNAL const BrotliTrieNode* BrotliTrieSub(const BrotliTrie* trie,
  47. const BrotliTrieNode* node, uint8_t c);
  48. #endif /* BROTLI_EXPERIMENTAL */
  49. /* Dictionary data (words and transforms) for 1 possible context */
  50. typedef struct BrotliEncoderDictionary {
  51. const BrotliDictionary* words;
  52. uint32_t num_transforms;
  53. /* cut off for fast encoder */
  54. uint32_t cutoffTransformsCount;
  55. uint64_t cutoffTransforms;
  56. /* from dictionary_hash.h, for fast encoder */
  57. const uint16_t* hash_table_words;
  58. const uint8_t* hash_table_lengths;
  59. /* from static_dict_lut.h, for slow encoder */
  60. const uint16_t* buckets;
  61. const DictWord* dict_words;
  62. /* Heavy version, for use by slow encoder when there are custom transforms.
  63. Contains every possible transformed dictionary word in a trie. It encodes
  64. about as fast as the non-heavy encoder but consumes a lot of memory and
  65. takes time to build. */
  66. BrotliTrie trie;
  67. BROTLI_BOOL has_words_heavy;
  68. /* Reference to other dictionaries. */
  69. const struct ContextualEncoderDictionary* parent;
  70. /* Allocated memory, used only when not using the Brotli defaults */
  71. uint16_t* hash_table_data_words_;
  72. uint8_t* hash_table_data_lengths_;
  73. size_t buckets_alloc_size_;
  74. uint16_t* buckets_data_;
  75. size_t dict_words_alloc_size_;
  76. DictWord* dict_words_data_;
  77. BrotliDictionary* words_instance_;
  78. } BrotliEncoderDictionary;
  79. /* Dictionary data for all 64 contexts */
  80. typedef struct ContextualEncoderDictionary {
  81. BROTLI_BOOL context_based;
  82. uint8_t num_dictionaries;
  83. uint8_t context_map[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
  84. const BrotliEncoderDictionary* dict[SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS];
  85. /* If num_instances_ is 1, instance_ is used, else dynamic allocation with
  86. instances_ is used. */
  87. size_t num_instances_;
  88. BrotliEncoderDictionary instance_;
  89. BrotliEncoderDictionary* instances_;
  90. } ContextualEncoderDictionary;
  91. typedef struct SharedEncoderDictionary {
  92. /* Magic value to distinguish this struct from PreparedDictionary for
  93. certain external usages. */
  94. uint32_t magic;
  95. /* LZ77 prefix, compound dictionary */
  96. CompoundDictionary compound;
  97. /* Custom static dictionary (optionally context-based) */
  98. ContextualEncoderDictionary contextual;
  99. /* The maximum quality the dictionary was computed for */
  100. int max_quality;
  101. } SharedEncoderDictionary;
  102. typedef struct ManagedDictionary {
  103. uint32_t magic;
  104. MemoryManager memory_manager_;
  105. uint32_t* dictionary;
  106. } ManagedDictionary;
  107. /* Initializes to the brotli built-in dictionary */
  108. BROTLI_INTERNAL void BrotliInitSharedEncoderDictionary(
  109. SharedEncoderDictionary* dict);
  110. #if defined(BROTLI_EXPERIMENTAL)
  111. /* Initializes to shared dictionary that will be parsed from
  112. encoded_dict. Requires that you keep the encoded_dict buffer
  113. around, parts of data will point to it. */
  114. BROTLI_INTERNAL BROTLI_BOOL BrotliInitCustomSharedEncoderDictionary(
  115. MemoryManager* m, const uint8_t* encoded_dict, size_t size,
  116. int quality, SharedEncoderDictionary* dict);
  117. #endif /* BROTLI_EXPERIMENTAL */
  118. BROTLI_INTERNAL void BrotliCleanupSharedEncoderDictionary(
  119. MemoryManager* m, SharedEncoderDictionary* dict);
  120. BROTLI_INTERNAL ManagedDictionary* BrotliCreateManagedDictionary(
  121. brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
  122. BROTLI_INTERNAL void BrotliDestroyManagedDictionary(
  123. ManagedDictionary* dictionary);
  124. #if defined(__cplusplus) || defined(c_plusplus)
  125. } /* extern "C" */
  126. #endif
  127. #endif /* BROTLI_ENC_ENCODER_DICT_H_ */