From 5fecba6022cbbeed8103b49d79b2fb36086be72d Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Tue, 22 Nov 2022 11:20:17 +0200 Subject: [PATCH] liblzma: Fix infinite loop in LZMA encoder init with dict_size >= 2 GiB. The encoder doesn't support dictionary sizes larger than 1536 MiB. This is validated, for example, when calculating the memory usage via lzma_raw_encoder_memusage(). It is also enforced by the LZ part of the encoder initialization. However, LZMA encoder with LZMA_MODE_NORMAL did an unsafe calculation with dict_size before such validation and that results in an infinite loop if dict_size was 2 << 30 or greater. --- src/liblzma/lzma/lzma_encoder.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c index c1552f19..bbbb5d2d 100644 --- a/src/liblzma/lzma/lzma_encoder.c +++ b/src/liblzma/lzma/lzma_encoder.c @@ -559,10 +559,9 @@ lzma_lzma_encoder_create(void **coder_ptr, lzma_lzma1_encoder *coder = *coder_ptr; - // Set compression mode. We haven't validates the options yet, - // but it's OK here, since nothing bad happens with invalid - // options in the code below, and they will get rejected by - // lzma_lzma_encoder_reset() call at the end of this function. + // Set compression mode. Note that we haven't validated the options + // yet. Invalid options will get rejected by lzma_lzma_encoder_reset() + // call at the end of this function. switch (options->mode) { case LZMA_MODE_FAST: coder->fast_mode = true; @@ -573,6 +572,18 @@ lzma_lzma_encoder_create(void **coder_ptr, // Set dist_table_size. // Round the dictionary size up to next 2^n. + // + // Currently the maximum encoder dictionary size + // is 1.5 GiB due to lz_encoder.c and here we need + // to be below 2 GiB to make the rounded up value + // fit in an uint32_t and avoid an infite while-loop + // (and undefined behavior due to a too large shift). + // So do the same check as in LZ encoder, + // limiting to 1.5 GiB. + if (options->dict_size > (UINT32_C(1) << 30) + + (UINT32_C(1) << 29)) + return LZMA_OPTIONS_ERROR; + uint32_t log_size = 0; while ((UINT32_C(1) << log_size) < options->dict_size) ++log_size;