Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
SUSE:SLE-12-SP4:Update
libwebp.29259
libwebp-optimize-huffman-decoding.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File libwebp-optimize-huffman-decoding.patch of Package libwebp.29259
From f75dfbf23d1df1be52350b1a6fc5cfa6c2194499 Mon Sep 17 00:00:00 2001 From: skal <pascal.massimino@gmail.com> Date: Thu, 28 Aug 2014 08:35:19 -0700 Subject: [PATCH] Speed up Huffman decoding for lossless speed-up is ~1.6% for photographic image to 10% for graphical image (1000 images corpus was sped up by 5.8 %) Code by akramarz@google.com and jyrki@google.com Change-Id: Iceb2e50e6cc761b9315a3865d22ec9d19b8011c6 Backported by Mike Gorse <mgorse@suse.com> Also includes commits f9ced9 and 21735e --- diff -urp libwebp-0.4.3.orig/src/dec/vp8l.c libwebp-0.4.3/src/dec/vp8l.c --- libwebp-0.4.3.orig/src/dec/vp8l.c 2015-03-11 01:06:09.000000000 -0500 +++ libwebp-0.4.3/src/dec/vp8l.c 2021-05-17 14:28:42.458032440 -0500 @@ -50,6 +50,9 @@ static const uint16_t kAlphabetSize[HUFF NUM_DISTANCE_CODES }; +static const uint8_t kLiteralMap[HUFFMAN_CODES_PER_META_CODE] = { + 0, 1, 1, 1, 0 +}; #define NUM_CODE_LENGTH_CODES 19 static const uint8_t kCodeLengthCodeOrder[NUM_CODE_LENGTH_CODES] = { @@ -72,6 +75,29 @@ static const uint8_t kCodeToPlane[CODE_T 0x40, 0x72, 0x7e, 0x61, 0x6f, 0x50, 0x71, 0x7f, 0x60, 0x70 }; +// Memory needed for lookup tables of one Huffman tree group. Red, blue, alpha +// and distance alphabets are constant (256 for red, blue and alpha, 40 for +// distance) and lookup table sizes for them in worst case are 630 and 410 +// respectively. Size of green alphabet depends on color cache size and is equal +// to 256 (green component values) + 24 (length prefix values) +// + color_cache_size (between 0 and 2048). +// All values computed for 8-bit first level lookup with tests/tools/enough.cc. +#define FIXED_TABLE_SIZE (630 * 3 + 410) +static const int kTableSize[12] = { + FIXED_TABLE_SIZE + 654, + FIXED_TABLE_SIZE + 656, + FIXED_TABLE_SIZE + 658, + FIXED_TABLE_SIZE + 662, + FIXED_TABLE_SIZE + 670, + FIXED_TABLE_SIZE + 686, + FIXED_TABLE_SIZE + 718, + FIXED_TABLE_SIZE + 782, + FIXED_TABLE_SIZE + 912, + FIXED_TABLE_SIZE + 1168, + FIXED_TABLE_SIZE + 1680, + FIXED_TABLE_SIZE + 2704 +}; + static int DecodeImageStream(int xsize, int ysize, int is_level0, VP8LDecoder* const dec, @@ -151,31 +177,69 @@ static WEBP_INLINE int PlaneCodeToDistan // Decodes the next Huffman code from bit-stream. // FillBitWindow(br) needs to be called at minimum every second call // to ReadSymbol, in order to pre-fetch enough bits. -static WEBP_INLINE int ReadSymbol(const HuffmanTree* tree, +static WEBP_INLINE int ReadSymbol(const HuffmanCode* table, VP8LBitReader* const br) { - const HuffmanTreeNode* node = tree->root_; - uint32_t bits = VP8LPrefetchBits(br); - int bitpos = br->bit_pos_; - // Check if we find the bit combination from the Huffman lookup table. - const int lut_ix = bits & (HUFF_LUT - 1); - const int lut_bits = tree->lut_bits_[lut_ix]; - if (lut_bits <= HUFF_LUT_BITS) { - VP8LSetBitPos(br, bitpos + lut_bits); - return tree->lut_symbol_[lut_ix]; - } - node += tree->lut_jump_[lut_ix]; - bitpos += HUFF_LUT_BITS; - bits >>= HUFF_LUT_BITS; - - // Decode the value from a binary tree. - assert(node != NULL); - do { - node = HuffmanTreeNextNode(node, bits & 1); - bits >>= 1; - ++bitpos; - } while (HuffmanTreeNodeIsNotLeaf(node)); - VP8LSetBitPos(br, bitpos); - return node->symbol_; + int nbits; + uint32_t val = VP8LPrefetchBits(br); + table += val & HUFFMAN_TABLE_MASK; + nbits = table->bits - HUFFMAN_TABLE_BITS; + if (nbits > 0) { + VP8LSetBitPos(br, br->bit_pos_ + HUFFMAN_TABLE_BITS); + val = VP8LPrefetchBits(br); + table += table->value; + table += val & ((1 << nbits) - 1); + } + VP8LSetBitPos(br, br->bit_pos_ + table->bits); + return table->value; +} + +// Reads packed symbol depending on GREEN channel +#define BITS_SPECIAL_MARKER 0x100 // something large enough (and a bit-mask) +#define PACKED_NON_LITERAL_CODE 0 // must be < NUM_LITERAL_CODES +static WEBP_INLINE int ReadPackedSymbols(const HTreeGroup* group, + VP8LBitReader* const br, + uint32_t* const dst) { + const uint32_t val = VP8LPrefetchBits(br) & (HUFFMAN_PACKED_TABLE_SIZE - 1); + const HuffmanCode32 code = group->packed_table[val]; + assert(group->use_packed_table); + if (code.bits < BITS_SPECIAL_MARKER) { + VP8LSetBitPos(br, br->bit_pos_ + code.bits); + *dst = code.value; + return PACKED_NON_LITERAL_CODE; + } else { + VP8LSetBitPos(br, br->bit_pos_ + code.bits - BITS_SPECIAL_MARKER); + assert(code.value >= NUM_LITERAL_CODES); + return code.value; + } +} + +static int AccumulateHCode(HuffmanCode hcode, int shift, + HuffmanCode32* const huff) { + huff->bits += hcode.bits; + huff->value |= (uint32_t)hcode.value << shift; + assert(huff->bits <= HUFFMAN_TABLE_BITS); + return hcode.bits; +} + +static void BuildPackedTable(HTreeGroup* const htree_group) { + uint32_t code; + for (code = 0; code < HUFFMAN_PACKED_TABLE_SIZE; ++code) { + uint32_t bits = code; + HuffmanCode32* const huff = &htree_group->packed_table[bits]; + HuffmanCode hcode = htree_group->htrees[GREEN][bits]; + if (hcode.value >= NUM_LITERAL_CODES) { + huff->bits = hcode.bits + BITS_SPECIAL_MARKER; + huff->value = hcode.value; + } else { + huff->bits = 0; + huff->value = 0; + bits >>= AccumulateHCode(hcode, 8, huff); + bits >>= AccumulateHCode(htree_group->htrees[RED][bits], 16, huff); + bits >>= AccumulateHCode(htree_group->htrees[BLUE][bits], 0, huff); + bits >>= AccumulateHCode(htree_group->htrees[ALPHA][bits], 24, huff); + (void)bits; + } + } } static int ReadHuffmanCodeLengths( @@ -186,11 +250,11 @@ static int ReadHuffmanCodeLengths( int symbol; int max_symbol; int prev_code_len = DEFAULT_CODE_LENGTH; - HuffmanTree tree; - int huff_codes[NUM_CODE_LENGTH_CODES] = { 0 }; + HuffmanCode table[1 << LENGTHS_TABLE_BITS]; - if (!VP8LHuffmanTreeBuildImplicit(&tree, code_length_code_lengths, - huff_codes, NUM_CODE_LENGTH_CODES)) { + if (!VP8LBuildHuffmanTable(table, LENGTHS_TABLE_BITS, + code_length_code_lengths, + NUM_CODE_LENGTH_CODES)) { dec->status_ = VP8_STATUS_BITSTREAM_ERROR; return 0; } @@ -208,10 +272,13 @@ static int ReadHuffmanCodeLengths( symbol = 0; while (symbol < num_symbols) { + const HuffmanCode* p = table; int code_len; if (max_symbol-- == 0) break; VP8LFillBitWindow(br); - code_len = ReadSymbol(&tree, br); + p += VP8LPrefetchBits(br) & LENGTHS_TABLE_MASK; + VP8LSetBitPos(br, br->bit_pos_ + p->bits); + code_len = p->value; if (code_len < kCodeLengthLiterals) { code_lengths[symbol++] = code_len; if (code_len != 0) prev_code_len = code_len; @@ -233,7 +300,6 @@ static int ReadHuffmanCodeLengths( ok = 1; End: - VP8LHuffmanTreeFree(&tree); if (!ok) dec->status_ = VP8_STATUS_BITSTREAM_ERROR; return ok; } @@ -241,29 +307,25 @@ static int ReadHuffmanCodeLengths( // 'code_lengths' is pre-allocated temporary buffer, used for creating Huffman // tree. static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, - int* const code_lengths, int* const huff_codes, - HuffmanTree* const tree) { + int* const code_lengths, HuffmanCode* const table) { int ok = 0; VP8LBitReader* const br = &dec->br_; const int simple_code = VP8LReadBits(br, 1); + memset(code_lengths, 0, alphabet_size * sizeof(*code_lengths)); + if (simple_code) { // Read symbols, codes & code lengths directly. - int symbols[2]; - int codes[2]; const int num_symbols = VP8LReadBits(br, 1) + 1; const int first_symbol_len_code = VP8LReadBits(br, 1); // The first code is either 1 bit or 8 bit code. - symbols[0] = VP8LReadBits(br, (first_symbol_len_code == 0) ? 1 : 8); - codes[0] = 0; - code_lengths[0] = num_symbols - 1; + int symbol = VP8LReadBits(br, (first_symbol_len_code == 0) ? 1 : 8); + code_lengths[symbol] = 1; // The second code (if present), is always 8 bit long. if (num_symbols == 2) { - symbols[1] = VP8LReadBits(br, 8); - codes[1] = 1; - code_lengths[1] = num_symbols - 1; + symbol = VP8LReadBits(br, 8); + code_lengths[symbol] = 1; } - ok = VP8LHuffmanTreeBuildExplicit(tree, code_lengths, codes, symbols, - alphabet_size, num_symbols); + ok = 1; } else { // Decode Huffman-coded code lengths. int i; int code_length_code_lengths[NUM_CODE_LENGTH_CODES] = { 0 }; @@ -273,22 +335,20 @@ static int ReadHuffmanCode(int alphabet_ return 0; } - memset(code_lengths, 0, alphabet_size * sizeof(*code_lengths)); - for (i = 0; i < num_codes; ++i) { code_length_code_lengths[kCodeLengthCodeOrder[i]] = VP8LReadBits(br, 3); } ok = ReadHuffmanCodeLengths(dec, code_length_code_lengths, alphabet_size, code_lengths); - ok = ok && VP8LHuffmanTreeBuildImplicit(tree, code_lengths, huff_codes, - alphabet_size); } + ok = ok && !br->error_; if (!ok) { dec->status_ = VP8_STATUS_BITSTREAM_ERROR; return 0; } - return 1; + return VP8LBuildHuffmanTable(table, HUFFMAN_TABLE_BITS, + code_lengths, alphabet_size); } static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, @@ -298,10 +358,12 @@ static int ReadHuffmanCodes(VP8LDecoder* VP8LMetadata* const hdr = &dec->hdr_; uint32_t* huffman_image = NULL; HTreeGroup* htree_groups = NULL; + HuffmanCode* huffman_tables = NULL; + HuffmanCode* next = NULL; int num_htree_groups = 1; int max_alphabet_size = 0; int* code_lengths = NULL; - int* huff_codes = NULL; + const int table_size = kTableSize[color_cache_bits]; if (allow_recursion && VP8LReadBits(br, 1)) { // use meta Huffman codes. @@ -338,45 +400,82 @@ static int ReadHuffmanCodes(VP8LDecoder* } } + huffman_tables = (HuffmanCode*)WebPSafeMalloc(num_htree_groups * table_size, + sizeof(HuffmanCode)); htree_groups = VP8LHtreeGroupsNew(num_htree_groups); - code_lengths = - (int*)WebPSafeCalloc((uint64_t)max_alphabet_size, sizeof(*code_lengths)); - huff_codes = - (int*)WebPSafeMalloc((uint64_t)max_alphabet_size, sizeof(*huff_codes)); + code_lengths = (int*)WebPSafeCalloc((uint64_t)max_alphabet_size, + sizeof(*code_lengths)); - if (htree_groups == NULL || code_lengths == NULL || huff_codes == NULL) { + if (htree_groups == NULL || code_lengths == NULL || huffman_tables == NULL) { dec->status_ = VP8_STATUS_OUT_OF_MEMORY; goto Error; } + next = huffman_tables; for (i = 0; i < num_htree_groups; ++i) { - HuffmanTree* const htrees = htree_groups[i].htrees_; + HTreeGroup* const htree_group = &htree_groups[i]; + HuffmanCode** const htrees = htree_group->htrees; + int size; + int total_size = 0; + int is_trivial_literal = 1; + int max_bits = 0; for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { int alphabet_size = kAlphabetSize[j]; - HuffmanTree* const htree = htrees + j; + htrees[j] = next; if (j == 0 && color_cache_bits > 0) { alphabet_size += 1 << color_cache_bits; } - if (!ReadHuffmanCode(alphabet_size, dec, code_lengths, huff_codes, - htree)) { + size = ReadHuffmanCode(alphabet_size, dec, code_lengths, next); + if (is_trivial_literal && kLiteralMap[j] == 1) { + is_trivial_literal = (next->bits == 0); + } + total_size += next->bits; + next += size; + if (size == 0) { goto Error; } + if (j <= ALPHA) { + int local_max_bits = code_lengths[0]; + int k; + for (k = 1; k < alphabet_size; ++k) { + if (code_lengths[k] > local_max_bits) { + local_max_bits = code_lengths[k]; + } + } + max_bits += local_max_bits; + } } + htree_group->is_trivial_literal = is_trivial_literal; + htree_group->is_trivial_code = 0; + if (is_trivial_literal) { + const int red = htrees[RED][0].value; + const int blue = htrees[BLUE][0].value; + const int alpha = htrees[ALPHA][0].value; + htree_group->literal_arb = + ((uint32_t)alpha << 24) | (red << 16) | blue; + if (total_size == 0 && htrees[GREEN][0].value < NUM_LITERAL_CODES) { + htree_group->is_trivial_code = 1; + htree_group->literal_arb |= htrees[GREEN][0].value << 8; + } + } + htree_group->use_packed_table = !htree_group->is_trivial_code && + (max_bits < HUFFMAN_PACKED_BITS); + if (htree_group->use_packed_table) BuildPackedTable(htree_group); } - WebPSafeFree(huff_codes); WebPSafeFree(code_lengths); // All OK. Finalize pointers and return. hdr->huffman_image_ = huffman_image; hdr->num_htree_groups_ = num_htree_groups; hdr->htree_groups_ = htree_groups; + hdr->huffman_tables_ = huffman_tables; return 1; Error: - WebPSafeFree(huff_codes); WebPSafeFree(code_lengths); WebPSafeFree(huffman_image); - VP8LHtreeGroupsFree(htree_groups, num_htree_groups); + WebPSafeFree(huffman_tables); + VP8LHtreeGroupsFree(htree_groups); return 0; } @@ -716,10 +815,10 @@ static int Is8bOptimizable(const VP8LMet // When the Huffman tree contains only one symbol, we can skip the // call to ReadSymbol() for red/blue/alpha channels. for (i = 0; i < hdr->num_htree_groups_; ++i) { - const HuffmanTree* const htrees = hdr->htree_groups_[i].htrees_; - if (htrees[RED].num_nodes_ > 1) return 0; - if (htrees[BLUE].num_nodes_ > 1) return 0; - if (htrees[ALPHA].num_nodes_ > 1) return 0; + HuffmanCode** const htrees = hdr->htree_groups_[i].htrees; + if (htrees[RED][0].bits > 0) return 0; + if (htrees[BLUE][0].bits > 0) return 0; + if (htrees[ALPHA][0].bits > 0) return 0; } return 1; } @@ -759,7 +858,7 @@ static int DecodeAlphaData(VP8LDecoder* htree_group = GetHtreeGroupForPos(hdr, col, row); } VP8LFillBitWindow(br); - code = ReadSymbol(&htree_group->htrees_[GREEN], br); + code = ReadSymbol(htree_group->htrees[GREEN], br); if (code < NUM_LITERAL_CODES) { // Literal data[pos] = code; ++pos; @@ -775,7 +874,7 @@ static int DecodeAlphaData(VP8LDecoder* int dist_code, dist; const int length_sym = code - NUM_LITERAL_CODES; const int length = GetCopyLength(length_sym, br); - const int dist_symbol = ReadSymbol(&htree_group->htrees_[DIST], br); + const int dist_symbol = ReadSymbol(htree_group->htrees[DIST], br); VP8LFillBitWindow(br); dist_code = GetCopyDistance(dist_symbol, br); dist = PlaneCodeToDistance(width, dist_code); @@ -848,19 +947,29 @@ static int DecodeImageData(VP8LDecoder* // Only update when changing tile. Note we could use this test: // if "((((prev_col ^ col) | prev_row ^ row)) > mask)" -> tile changed // but that's actually slower and needs storing the previous col/row. - if ((col & mask) == 0) { - htree_group = GetHtreeGroupForPos(hdr, col, row); + if ((col & mask) == 0) htree_group = GetHtreeGroupForPos(hdr, col, row); + if (htree_group->is_trivial_code) { + *src = htree_group->literal_arb; + goto AdvanceByOne; } VP8LFillBitWindow(br); - code = ReadSymbol(&htree_group->htrees_[GREEN], br); + if (htree_group->use_packed_table) { + code = ReadPackedSymbols(htree_group, br, src); + if (code == PACKED_NON_LITERAL_CODE) goto AdvanceByOne; + } else { + code = ReadSymbol(htree_group->htrees[GREEN], br); + } if (code < NUM_LITERAL_CODES) { // Literal - int red, green, blue, alpha; - red = ReadSymbol(&htree_group->htrees_[RED], br); - green = code; + if (htree_group->is_trivial_literal) { + *src = htree_group->literal_arb | (code << 8); + } else { + int red, blue, alpha; + red = ReadSymbol(htree_group->htrees[RED], br); VP8LFillBitWindow(br); - blue = ReadSymbol(&htree_group->htrees_[BLUE], br); - alpha = ReadSymbol(&htree_group->htrees_[ALPHA], br); - *src = ((uint32_t)alpha << 24) | (red << 16) | (green << 8) | blue; + blue = ReadSymbol(htree_group->htrees[BLUE], br); + alpha = ReadSymbol(htree_group->htrees[ALPHA], br); + *src = ((uint32_t)alpha << 24) | (red << 16) | (code << 8) | blue; + } AdvanceByOne: ++src; ++col; @@ -880,7 +989,7 @@ static int DecodeImageData(VP8LDecoder* int dist_code, dist; const int length_sym = code - NUM_LITERAL_CODES; const int length = GetCopyLength(length_sym, br); - const int dist_symbol = ReadSymbol(&htree_group->htrees_[DIST], br); + const int dist_symbol = ReadSymbol(htree_group->htrees[DIST], br); VP8LFillBitWindow(br); dist_code = GetCopyDistance(dist_symbol, br); dist = PlaneCodeToDistance(width, dist_code); @@ -1038,7 +1147,8 @@ static void ClearMetadata(VP8LMetadata* assert(hdr); WebPSafeFree(hdr->huffman_image_); - VP8LHtreeGroupsFree(hdr->htree_groups_, hdr->num_htree_groups_); + WebPSafeFree(hdr->huffman_tables_); + VP8LHtreeGroupsFree(hdr->htree_groups_); VP8LColorCacheClear(&hdr->color_cache_); InitMetadata(hdr); } diff -urp libwebp-0.4.3.orig/src/dec/vp8li.h libwebp-0.4.3/src/dec/vp8li.h --- libwebp-0.4.3.orig/src/dec/vp8li.h 2015-03-11 01:06:09.000000000 -0500 +++ libwebp-0.4.3/src/dec/vp8li.h 2021-05-17 14:11:36.392615443 -0500 @@ -50,6 +50,7 @@ typedef struct { uint32_t *huffman_image_; int num_htree_groups_; HTreeGroup *htree_groups_; + HuffmanCode *huffman_tables_; } VP8LMetadata; typedef struct VP8LDecoder VP8LDecoder; Only in libwebp-0.4.3.orig/src/mux: muxread.c.orig diff -urp libwebp-0.4.3.orig/src/utils/huffman.c libwebp-0.4.3/src/utils/huffman.c --- libwebp-0.4.3.orig/src/utils/huffman.c 2015-03-11 01:06:09.000000000 -0500 +++ libwebp-0.4.3/src/utils/huffman.c 2021-05-17 14:11:36.392615443 -0500 @@ -18,302 +18,192 @@ #include "../utils/utils.h" #include "../webp/format_constants.h" -// Uncomment the following to use look-up table for ReverseBits() -// (might be faster on some platform) -// #define USE_LUT_REVERSE_BITS - // Huffman data read via DecodeImageStream is represented in two (red and green) // bytes. #define MAX_HTREE_GROUPS 0x10000 -#define NON_EXISTENT_SYMBOL (-1) - -static void TreeNodeInit(HuffmanTreeNode* const node) { - node->children_ = -1; // means: 'unassigned so far' -} - -static int NodeIsEmpty(const HuffmanTreeNode* const node) { - return (node->children_ < 0); -} - -static int IsFull(const HuffmanTree* const tree) { - return (tree->num_nodes_ == tree->max_nodes_); -} - -static void AssignChildren(HuffmanTree* const tree, - HuffmanTreeNode* const node) { - HuffmanTreeNode* const children = tree->root_ + tree->num_nodes_; - node->children_ = (int)(children - node); - assert(children - node == (int)(children - node)); - tree->num_nodes_ += 2; - TreeNodeInit(children + 0); - TreeNodeInit(children + 1); -} - -// A Huffman tree is a full binary tree; and in a full binary tree with L -// leaves, the total number of nodes N = 2 * L - 1. -static int HuffmanTreeMaxNodes(int num_leaves) { - return (2 * num_leaves - 1); -} - -static int HuffmanTreeAllocate(HuffmanTree* const tree, int num_nodes) { - assert(tree != NULL); - tree->root_ = - (HuffmanTreeNode*)WebPSafeMalloc(num_nodes, sizeof(*tree->root_)); - return (tree->root_ != NULL); -} - -static int TreeInit(HuffmanTree* const tree, int num_leaves) { - assert(tree != NULL); - if (num_leaves == 0) return 0; - tree->max_nodes_ = HuffmanTreeMaxNodes(num_leaves); - assert(tree->max_nodes_ < (1 << 16)); // limit for the lut_jump_ table - if (!HuffmanTreeAllocate(tree, tree->max_nodes_)) return 0; - TreeNodeInit(tree->root_); // Initialize root. - tree->num_nodes_ = 1; - memset(tree->lut_bits_, 255, sizeof(tree->lut_bits_)); - memset(tree->lut_jump_, 0, sizeof(tree->lut_jump_)); - return 1; -} - -void VP8LHuffmanTreeFree(HuffmanTree* const tree) { - if (tree != NULL) { - WebPSafeFree(tree->root_); - tree->root_ = NULL; - tree->max_nodes_ = 0; - tree->num_nodes_ = 0; - } -} HTreeGroup* VP8LHtreeGroupsNew(int num_htree_groups) { HTreeGroup* const htree_groups = - (HTreeGroup*)WebPSafeCalloc(num_htree_groups, sizeof(*htree_groups)); - assert(num_htree_groups <= MAX_HTREE_GROUPS); + (HTreeGroup*)WebPSafeMalloc(num_htree_groups, sizeof(*htree_groups)); if (htree_groups == NULL) { return NULL; } + assert(num_htree_groups <= MAX_HTREE_GROUPS); return htree_groups; } -void VP8LHtreeGroupsFree(HTreeGroup* htree_groups, int num_htree_groups) { +void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups) { if (htree_groups != NULL) { - int i, j; - for (i = 0; i < num_htree_groups; ++i) { - HuffmanTree* const htrees = htree_groups[i].htrees_; - for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { - VP8LHuffmanTreeFree(&htrees[j]); - } - } WebPSafeFree(htree_groups); } } -int VP8LHuffmanCodeLengthsToCodes( - const int* const code_lengths, int code_lengths_size, - int* const huff_codes) { - int symbol; - int code_len; - int code_length_hist[MAX_ALLOWED_CODE_LENGTH + 1] = { 0 }; - int curr_code; - int next_codes[MAX_ALLOWED_CODE_LENGTH + 1] = { 0 }; - int max_code_length = 0; +// Returns reverse(reverse(key, len) + 1, len), where reverse(key, len) is the +// bit-wise reversal of the len least significant bits of key. +static WEBP_INLINE uint32_t GetNextKey(uint32_t key, int len) { + uint32_t step = 1 << (len - 1); + while (key & step) { + step >>= 1; + } + return (key & (step - 1)) + step; +} + +// Stores code in table[0], table[step], table[2*step], ..., table[end]. +// Assumes that end is an integer multiple of step. +static WEBP_INLINE void ReplicateValue(HuffmanCode* table, + int step, int end, + HuffmanCode code) { + assert(end % step == 0); + do { + end -= step; + table[end] = code; + } while (end > 0); +} + +// Returns the table width of the next 2nd level table. count is the histogram +// of bit lengths for the remaining symbols, len is the code length of the next +// processed symbol +static WEBP_INLINE int NextTableBitSize(const int* const count, + int len, int root_bits) { + int left = 1 << (len - root_bits); + while (len < MAX_ALLOWED_CODE_LENGTH) { + left -= count[len]; + if (left <= 0) break; + ++len; + left <<= 1; + } + return len - root_bits; +} + +int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, + const int* const code_lengths, + int code_lengths_size) { + HuffmanCode* table; // next available space in table + int len; // current code length + int symbol; // symbol index in original or sorted table + int total_size; // sum of root table size and 2nd level table sizes + int* sorted = NULL; // symbols sorted by code length + int count[MAX_ALLOWED_CODE_LENGTH + 1] = { 0 }; + // number of codes of each length + int offset[MAX_ALLOWED_CODE_LENGTH + 1]; + // offsets in sorted table for each length + assert(code_lengths_size != 0); assert(code_lengths != NULL); - assert(code_lengths_size > 0); - assert(huff_codes != NULL); - - // Calculate max code length. - for (symbol = 0; symbol < code_lengths_size; ++symbol) { - if (code_lengths[symbol] > max_code_length) { - max_code_length = code_lengths[symbol]; - } - } - if (max_code_length > MAX_ALLOWED_CODE_LENGTH) return 0; - - // Calculate code length histogram. - for (symbol = 0; symbol < code_lengths_size; ++symbol) { - ++code_length_hist[code_lengths[symbol]]; - } - code_length_hist[0] = 0; - - // Calculate the initial values of 'next_codes' for each code length. - // next_codes[code_len] denotes the code to be assigned to the next symbol - // of code length 'code_len'. - curr_code = 0; - next_codes[0] = -1; // Unused, as code length = 0 implies code doesn't exist. - for (code_len = 1; code_len <= max_code_length; ++code_len) { - curr_code = (curr_code + code_length_hist[code_len - 1]) << 1; - next_codes[code_len] = curr_code; - } + assert(root_table != NULL); + assert(root_bits > 0); - // Get symbols. + // Build histogram of code lengths. for (symbol = 0; symbol < code_lengths_size; ++symbol) { - if (code_lengths[symbol] > 0) { - huff_codes[symbol] = next_codes[code_lengths[symbol]]++; - } else { - huff_codes[symbol] = NON_EXISTENT_SYMBOL; + if (code_lengths[symbol] > MAX_ALLOWED_CODE_LENGTH) { + return 0; } + ++count[code_lengths[symbol]]; } - return 1; -} -#ifndef USE_LUT_REVERSE_BITS - -static int ReverseBitsShort(int bits, int num_bits) { - int retval = 0; - int i; - assert(num_bits <= 8); // Not a hard requirement, just for coherency. - for (i = 0; i < num_bits; ++i) { - retval <<= 1; - retval |= bits & 1; - bits >>= 1; + // Error, all code lengths are zeros. + if (count[0] == code_lengths_size) { + return 0; } - return retval; -} - -#else - -static const uint8_t kReversedBits[16] = { // Pre-reversed 4-bit values. - 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, - 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf -}; - -static int ReverseBitsShort(int bits, int num_bits) { - const uint8_t v = (kReversedBits[bits & 0xf] << 4) | kReversedBits[bits >> 4]; - assert(num_bits <= 8); - return v >> (8 - num_bits); -} - -#endif -static int TreeAddSymbol(HuffmanTree* const tree, - int symbol, int code, int code_length) { - int step = HUFF_LUT_BITS; - int base_code; - HuffmanTreeNode* node = tree->root_; - const HuffmanTreeNode* const max_node = tree->root_ + tree->max_nodes_; - assert(symbol == (int16_t)symbol); - if (code_length <= HUFF_LUT_BITS) { - int i; - base_code = ReverseBitsShort(code, code_length); - for (i = 0; i < (1 << (HUFF_LUT_BITS - code_length)); ++i) { - const int idx = base_code | (i << code_length); - tree->lut_symbol_[idx] = (int16_t)symbol; - tree->lut_bits_[idx] = code_length; - } - } else { - base_code = ReverseBitsShort((code >> (code_length - HUFF_LUT_BITS)), - HUFF_LUT_BITS); - } - while (code_length-- > 0) { - if (node >= max_node) { + // Generate offsets into sorted symbol table by code length. + offset[1] = 0; + for (len = 1; len < MAX_ALLOWED_CODE_LENGTH; ++len) { + if (count[len] > (1 << len)) { return 0; } - if (NodeIsEmpty(node)) { - if (IsFull(tree)) return 0; // error: too many symbols. - AssignChildren(tree, node); - } else if (!HuffmanTreeNodeIsNotLeaf(node)) { - return 0; // leaf is already occupied. - } - node += node->children_ + ((code >> code_length) & 1); - if (--step == 0) { - tree->lut_jump_[base_code] = (int16_t)(node - tree->root_); - } + offset[len + 1] = offset[len] + count[len]; } - if (NodeIsEmpty(node)) { - node->children_ = 0; // turn newly created node into a leaf. - } else if (HuffmanTreeNodeIsNotLeaf(node)) { - return 0; // trying to assign a symbol to already used code. - } - node->symbol_ = symbol; // Add symbol in this node. - return 1; -} -int VP8LHuffmanTreeBuildImplicit(HuffmanTree* const tree, - const int* const code_lengths, - int* const codes, - int code_lengths_size) { - int symbol; - int num_symbols = 0; - int root_symbol = 0; - - assert(tree != NULL); - assert(code_lengths != NULL); + sorted = (int*)WebPSafeMalloc(code_lengths_size, sizeof(*sorted)); + if (sorted == NULL) { + return 0; + } - // Find out number of symbols and the root symbol. + // Sort symbols by length, by symbol order within each length. for (symbol = 0; symbol < code_lengths_size; ++symbol) { + const int symbol_code_length = code_lengths[symbol]; if (code_lengths[symbol] > 0) { - // Note: code length = 0 indicates non-existent symbol. - ++num_symbols; - root_symbol = symbol; + sorted[offset[symbol_code_length]++] = symbol; } } - // Initialize the tree. Will fail for num_symbols = 0 - if (!TreeInit(tree, num_symbols)) return 0; + table = root_table; + total_size = 1 << root_bits; - // Build tree. - if (num_symbols == 1) { // Trivial case. - const int max_symbol = code_lengths_size; - if (root_symbol < 0 || root_symbol >= max_symbol) { - VP8LHuffmanTreeFree(tree); - return 0; - } - return TreeAddSymbol(tree, root_symbol, 0, 0); - } else { // Normal case. - int ok = 0; - memset(codes, 0, code_lengths_size * sizeof(*codes)); - - if (!VP8LHuffmanCodeLengthsToCodes(code_lengths, code_lengths_size, - codes)) { - goto End; + // Special case code with only one value. + if (offset[MAX_ALLOWED_CODE_LENGTH] == 1) { + HuffmanCode code; + code.bits = 0; + code.value = (uint16_t)sorted[0]; + ReplicateValue(table, 1, total_size, code); + WebPSafeFree(sorted); + return total_size; + } + + { + int step; // step size to replicate values in current table + uint32_t low = -1; // low bits for current root entry + uint32_t mask = total_size - 1; // mask for low bits + uint32_t key = 0; // reversed prefix code + int num_nodes = 1; // number of Huffman tree nodes + int num_open = 1; // number of open branches in current tree level + int table_bits = root_bits; // key length of current table + int table_size = 1 << table_bits; // size of current table + symbol = 0; + // Fill in root table. + for (len = 1, step = 2; len <= root_bits; ++len, step <<= 1) { + num_open <<= 1; + num_nodes += num_open; + num_open -= count[len]; + if (num_open < 0) { + WebPSafeFree(sorted); + return 0; + } + for (; count[len] > 0; --count[len]) { + HuffmanCode code; + code.bits = (uint8_t)len; + code.value = (uint16_t)sorted[symbol++]; + ReplicateValue(&table[key], step, table_size, code); + key = GetNextKey(key, len); + } } - // Add symbols one-by-one. - for (symbol = 0; symbol < code_lengths_size; ++symbol) { - if (code_lengths[symbol] > 0) { - if (!TreeAddSymbol(tree, symbol, codes[symbol], - code_lengths[symbol])) { - goto End; + // Fill in 2nd level tables and add pointers to root table. + for (len = root_bits + 1, step = 2; len <= MAX_ALLOWED_CODE_LENGTH; + ++len, step <<= 1) { + num_open <<= 1; + num_nodes += num_open; + num_open -= count[len]; + if (num_open < 0) { + WebPSafeFree(sorted); + return 0; + } + for (; count[len] > 0; --count[len]) { + HuffmanCode code; + if ((key & mask) != low) { + table += table_size; + table_bits = NextTableBitSize(count, len, root_bits); + table_size = 1 << table_bits; + total_size += table_size; + low = key & mask; + root_table[low].bits = (uint8_t)(table_bits + root_bits); + root_table[low].value = (uint16_t)((table - root_table) - low); } + code.bits = (uint8_t)(len - root_bits); + code.value = (uint16_t)sorted[symbol++]; + ReplicateValue(&table[key >> root_bits], step, table_size, code); + key = GetNextKey(key, len); } } - ok = 1; - End: - ok = ok && IsFull(tree); - if (!ok) VP8LHuffmanTreeFree(tree); - return ok; - } -} -int VP8LHuffmanTreeBuildExplicit(HuffmanTree* const tree, - const int* const code_lengths, - const int* const codes, - const int* const symbols, int max_symbol, - int num_symbols) { - int ok = 0; - int i; - assert(tree != NULL); - assert(code_lengths != NULL); - assert(codes != NULL); - assert(symbols != NULL); - - // Initialize the tree. Will fail if num_symbols = 0. - if (!TreeInit(tree, num_symbols)) return 0; - - // Add symbols one-by-one. - for (i = 0; i < num_symbols; ++i) { - if (codes[i] != NON_EXISTENT_SYMBOL) { - if (symbols[i] < 0 || symbols[i] >= max_symbol) { - goto End; - } - if (!TreeAddSymbol(tree, symbols[i], codes[i], code_lengths[i])) { - goto End; - } + // Check if tree is full. + if (num_nodes != 2 * offset[MAX_ALLOWED_CODE_LENGTH] - 1) { + WebPSafeFree(sorted); + return 0; } } - ok = 1; - End: - ok = ok && IsFull(tree); - if (!ok) VP8LHuffmanTreeFree(tree); - return ok; + + WebPSafeFree(sorted); + return total_size; } diff -urp libwebp-0.4.3.orig/src/utils/huffman.h libwebp-0.4.3/src/utils/huffman.h --- libwebp-0.4.3.orig/src/utils/huffman.h 2015-03-11 01:06:09.000000000 -0500 +++ libwebp-0.4.3/src/utils/huffman.h 2021-05-17 14:17:20.682432970 -0500 @@ -22,78 +22,65 @@ extern "C" { #endif -// A node of a Huffman tree. +#define HUFFMAN_TABLE_BITS 8 +#define HUFFMAN_TABLE_MASK ((1 << HUFFMAN_TABLE_BITS) - 1) + +#define LENGTHS_TABLE_BITS 7 +#define LENGTHS_TABLE_MASK ((1 << LENGTHS_TABLE_BITS) - 1) + + +// Huffman lookup table entry typedef struct { - int symbol_; - int children_; // delta offset to both children (contiguous) or 0 if leaf. -} HuffmanTreeNode; - -// Huffman Tree. -#define HUFF_LUT_BITS 7 -#define HUFF_LUT (1U << HUFF_LUT_BITS) -typedef struct HuffmanTree HuffmanTree; -struct HuffmanTree { - // Fast lookup for short bit lengths. - uint8_t lut_bits_[HUFF_LUT]; - int16_t lut_symbol_[HUFF_LUT]; - int16_t lut_jump_[HUFF_LUT]; - // Complete tree for lookups. - HuffmanTreeNode* root_; // all the nodes, starting at root. - int max_nodes_; // max number of nodes - int num_nodes_; // number of currently occupied nodes -}; + uint8_t bits; // number of bits used for this symbol + uint16_t value; // symbol value or table offset +} HuffmanCode; -// Huffman Tree group. +// long version for holding 32b values +typedef struct { + int bits; // number of bits used for this symbol, + // or an impossible value if not a literal code. + uint32_t value; // 32b packed ARGB value if literal, + // or non-literal symbol otherwise +} HuffmanCode32; + +#define HUFFMAN_PACKED_BITS 6 +#define HUFFMAN_PACKED_TABLE_SIZE (1u << HUFFMAN_PACKED_BITS) + +// Huffman table group. +// Includes special handling for the following cases: +// - is_trivial_literal: one common literal base for RED/BLUE/ALPHA (not GREEN) +// - is_trivial_code: only 1 code (no bit is read from bitstream) +// - use_packed_table: few enough literal symbols, so all the bit codes +// can fit into a small look-up table packed_table[] +// The common literal base, if applicable, is stored in 'literal_arb'. typedef struct HTreeGroup HTreeGroup; struct HTreeGroup { - HuffmanTree htrees_[HUFFMAN_CODES_PER_META_CODE]; + HuffmanCode* htrees[HUFFMAN_CODES_PER_META_CODE]; + int is_trivial_literal; // True, if huffman trees for Red, Blue & Alpha + // Symbols are trivial (have a single code). + uint32_t literal_arb; // If is_trivial_literal is true, this is the + // ARGB value of the pixel, with Green channel + // being set to zero. + int is_trivial_code; // true if is_trivial_literal with only one code + int use_packed_table; // use packed table below for short literal code + // table mapping input bits to a packed values, or escape case to literal code + HuffmanCode32 packed_table[HUFFMAN_PACKED_TABLE_SIZE]; }; -// Returns true if the given node is not a leaf of the Huffman tree. -static WEBP_INLINE int HuffmanTreeNodeIsNotLeaf( - const HuffmanTreeNode* const node) { - return node->children_; -} - -// Go down one level. Most critical function. 'right_child' must be 0 or 1. -static WEBP_INLINE const HuffmanTreeNode* HuffmanTreeNextNode( - const HuffmanTreeNode* node, int right_child) { - return node + node->children_ + right_child; -} - -// Releases the nodes of the Huffman tree. -// Note: It does NOT free 'tree' itself. -void VP8LHuffmanTreeFree(HuffmanTree* const tree); - // Creates the instance of HTreeGroup with specified number of tree-groups. HTreeGroup* VP8LHtreeGroupsNew(int num_htree_groups); // Releases the memory allocated for HTreeGroup. -void VP8LHtreeGroupsFree(HTreeGroup* htree_groups, int num_htree_groups); +void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups); -// Builds Huffman tree assuming code lengths are implicitly in symbol order. -// The 'huff_codes' and 'code_lengths' are pre-allocated temporary memory -// buffers, used for creating the huffman tree. -// Returns false in case of error (invalid tree or memory error). -int VP8LHuffmanTreeBuildImplicit(HuffmanTree* const tree, - const int* const code_lengths, - int* const huff_codes, - int code_lengths_size); - -// Build a Huffman tree with explicitly given lists of code lengths, codes -// and symbols. Verifies that all symbols added are smaller than max_symbol. -// Returns false in case of an invalid symbol, invalid tree or memory error. -int VP8LHuffmanTreeBuildExplicit(HuffmanTree* const tree, - const int* const code_lengths, - const int* const codes, - const int* const symbols, int max_symbol, - int num_symbols); - -// Utility: converts Huffman code lengths to corresponding Huffman codes. -// 'huff_codes' should be pre-allocated. -// Returns false in case of error (memory allocation, invalid codes). -int VP8LHuffmanCodeLengthsToCodes(const int* const code_lengths, - int code_lengths_size, int* const huff_codes); +// Builds Huffman lookup table assuming code lengths are in symbol order. +// The 'code_lengths' is pre-allocated temporary memory buffer used for creating +// the huffman table. +// Returns built table size or 0 in case of error (invalid tree or +// memory error). +int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, + const int* const code_lengths, + int code_lengths_size); #ifdef __cplusplus } // extern "C"
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
