From 3b34851de1eaf358cf9268922fa0eeed8278d680 Mon Sep 17 00:00:00 2001 From: Lasse Collin Date: Thu, 28 Aug 2008 22:53:15 +0300 Subject: [PATCH] Sort of garbage collection commit. :-| Many things are still broken. API has changed a lot and it will still change a little more here and there. The command line tool doesn't have all the required changes to reflect the API changes, so it's easy to get "internal error" or trigger assertions. --- configure.ac | 358 +++-- debug/Makefile.am | 5 +- .../lz/lz_encoder_private.h => debug/crc32.c | 47 +- debug/full_flush.c | 14 +- debug/hex2bin.c | 54 + debug/known_sizes.c | 135 ++ debug/memusage.c | 8 +- debug/sync_flush.c | 20 +- src/common/integer.h | 26 +- src/common/sysdefs.h | 42 +- src/liblzma/Makefile.am | 17 +- src/liblzma/api/Makefile.am | 6 +- src/liblzma/api/lzma.h | 169 +- src/liblzma/api/lzma/alignment.h | 6 +- src/liblzma/api/lzma/alone.h | 52 - src/liblzma/api/lzma/auto.h | 36 - src/liblzma/api/lzma/base.h | 61 +- src/liblzma/api/lzma/block.h | 38 +- src/liblzma/api/lzma/check.h | 55 +- src/liblzma/api/lzma/container.h | 252 +++ src/liblzma/api/lzma/delta.h | 36 +- src/liblzma/api/lzma/easy.h | 121 -- src/liblzma/api/lzma/filter.h | 74 +- src/liblzma/api/lzma/index.h | 40 +- src/liblzma/api/lzma/index_hash.h | 12 +- src/liblzma/api/lzma/lzma.h | 226 +-- src/liblzma/api/lzma/memlimit.h | 15 +- src/liblzma/api/lzma/raw.h | 60 - src/liblzma/api/lzma/simple.h | 2 +- src/liblzma/api/lzma/stream.h | 53 - src/liblzma/api/lzma/stream_flags.h | 17 +- src/liblzma/api/lzma/subblock.h | 4 +- src/liblzma/api/lzma/version.h | 10 +- src/liblzma/api/lzma/vli.h | 131 +- src/liblzma/check/check.c | 128 +- src/liblzma/check/check.h | 67 +- src/liblzma/check/sha256.c | 29 +- src/liblzma/common/Makefile.am | 51 +- src/liblzma/common/alignment.c | 4 +- src/liblzma/common/allocator.c | 58 - src/liblzma/common/alone_decoder.c | 49 +- src/liblzma/common/alone_decoder.h | 9 +- src/liblzma/common/alone_encoder.c | 13 +- src/liblzma/common/auto_decoder.c | 38 +- src/liblzma/common/block_decoder.c | 71 +- src/liblzma/common/block_decoder.h | 2 +- src/liblzma/common/block_encoder.c | 42 +- src/liblzma/common/block_encoder.h | 2 +- src/liblzma/common/block_header_decoder.c | 6 +- src/liblzma/common/block_header_encoder.c | 9 +- src/liblzma/common/block_util.c | 10 +- src/liblzma/common/{code.c => common.c} | 129 +- src/liblzma/common/common.h | 301 ++-- src/liblzma/common/easy.c | 18 +- src/liblzma/common/features.c | 66 - src/liblzma/common/filter_common.c | 262 ++++ src/liblzma/common/filter_common.h | 52 + src/liblzma/common/filter_decoder.c | 236 +++ src/liblzma/common/filter_decoder.h | 35 + src/liblzma/common/filter_encoder.c | 308 ++++ src/liblzma/common/filter_encoder.h | 38 + src/liblzma/common/filter_flags_decoder.c | 191 +-- src/liblzma/common/filter_flags_encoder.c | 265 +--- src/liblzma/common/index_decoder.c | 14 +- src/liblzma/common/index_encoder.c | 16 +- src/liblzma/common/index_hash.c | 8 +- src/liblzma/common/init_encoder.c | 2 +- src/liblzma/common/memory_usage.c | 112 -- src/liblzma/common/next_coder.c | 65 - src/liblzma/common/raw_common.c | 127 -- src/liblzma/common/raw_decoder.c | 116 -- src/liblzma/common/raw_encoder.c | 111 -- src/liblzma/common/stream_common.c | 23 - src/liblzma/common/stream_decoder.c | 238 ++- src/liblzma/common/stream_decoder.h | 4 +- src/liblzma/common/stream_encoder.c | 35 +- src/liblzma/common/stream_encoder.h | 2 +- ...am_flags_equal.c => stream_flags_common.c} | 14 +- ...{stream_common.h => stream_flags_common.h} | 8 +- src/liblzma/common/stream_flags_decoder.c | 2 +- src/liblzma/common/stream_flags_encoder.c | 2 +- src/liblzma/common/vli_decoder.c | 29 +- src/liblzma/common/vli_encoder.c | 23 +- src/liblzma/common/{version.c => vli_size.c} | 22 +- src/liblzma/delta/Makefile.am | 34 + src/liblzma/{common => delta}/delta_common.c | 2 +- src/liblzma/{common => delta}/delta_common.h | 0 src/liblzma/{common => delta}/delta_decoder.c | 21 + src/liblzma/{common => delta}/delta_decoder.h | 4 + src/liblzma/{common => delta}/delta_encoder.c | 21 + src/liblzma/{common => delta}/delta_encoder.h | 2 + src/liblzma/lz/Makefile.am | 35 +- src/liblzma/lz/bt2.c | 27 - src/liblzma/lz/bt2.h | 31 - src/liblzma/lz/bt3.c | 29 - src/liblzma/lz/bt3.h | 31 - src/liblzma/lz/bt4.c | 30 - src/liblzma/lz/bt4.h | 31 - src/liblzma/lz/hc3.c | 30 - src/liblzma/lz/hc3.h | 31 - src/liblzma/lz/hc4.c | 31 - src/liblzma/lz/hc4.h | 31 - src/liblzma/lz/lz_decoder.c | 533 ++----- src/liblzma/lz/lz_decoder.h | 312 ++-- src/liblzma/lz/lz_encoder.c | 782 +++++----- src/liblzma/lz/lz_encoder.h | 326 +++- src/liblzma/lz/lz_encoder_hash.h | 104 ++ src/liblzma/lz/lz_encoder_mf.c | 780 ++++++++++ src/liblzma/lz/match_c.h | 412 ----- src/liblzma/lz/match_h.h | 69 - src/liblzma/lzma/Makefile.am | 37 +- src/liblzma/lzma/fastpos.h | 8 +- src/liblzma/lzma/lzma2_decoder.c | 318 ++++ src/liblzma/lzma/lzma2_decoder.h | 35 + src/liblzma/lzma/lzma2_encoder.c | 406 +++++ .../raw_common.h => lzma/lzma2_encoder.h} | 22 +- src/liblzma/lzma/lzma_common.h | 208 ++- src/liblzma/lzma/lzma_decoder.c | 1370 ++++++++++------- src/liblzma/lzma/lzma_decoder.h | 21 +- src/liblzma/lzma/lzma_encoder.c | 586 +++++-- src/liblzma/lzma/lzma_encoder.h | 38 +- src/liblzma/lzma/lzma_encoder_features.c | 2 +- src/liblzma/lzma/lzma_encoder_getoptimum.c | 925 ----------- .../lzma/lzma_encoder_getoptimumfast.c | 201 --- src/liblzma/lzma/lzma_encoder_init.c | 228 --- src/liblzma/lzma/lzma_encoder_optimum_fast.c | 193 +++ .../lzma/lzma_encoder_optimum_normal.c | 875 +++++++++++ src/liblzma/lzma/lzma_encoder_presets.c | 52 +- src/liblzma/lzma/lzma_encoder_private.h | 174 +-- src/liblzma/lzma/lzma_literal.c | 51 - src/liblzma/lzma/lzma_literal.h | 71 - src/liblzma/rangecoder/Makefile.am | 10 +- src/liblzma/rangecoder/price.h | 111 ++ src/liblzma/rangecoder/price_table.c | 84 +- src/liblzma/rangecoder/price_table_init.c | 31 +- .../{price_table_gen.c => price_tablegen.c} | 19 +- src/liblzma/rangecoder/range_common.h | 17 +- src/liblzma/rangecoder/range_decoder.h | 217 ++- src/liblzma/rangecoder/range_encoder.h | 92 +- src/liblzma/simple/Makefile.am | 12 + src/liblzma/simple/simple_coder.c | 8 +- src/liblzma/simple/simple_decoder.c | 47 + .../raw_decoder.h => simple/simple_decoder.h} | 18 +- src/liblzma/simple/simple_encoder.c | 45 + .../raw_encoder.h => simple/simple_encoder.h} | 17 +- src/liblzma/subblock/Makefile.am | 4 +- src/liblzma/subblock/subblock_decoder.c | 20 +- .../subblock/subblock_decoder_helper.c | 2 +- src/liblzma/subblock/subblock_encoder.c | 28 +- src/lzma/args.c | 35 +- src/lzma/args.h | 4 +- src/lzma/options.c | 14 +- src/lzma/process.c | 88 +- src/lzmadec/lzmadec.c | 157 +- tests/Makefile.am | 1 + tests/files/README | 355 ++--- tests/files/bad-0-backward_size.lzma | Bin 0 -> 32 bytes tests/files/bad-0-empty-truncated.lzma | Bin 0 -> 31 bytes tests/files/bad-0-nonempty_index.lzma | Bin 0 -> 32 bytes tests/files/bad-0cat-alone.lzma | Bin 0 -> 55 bytes tests/files/bad-0catpad-empty.lzma | Bin 0 -> 69 bytes tests/files/bad-0pad-empty.lzma | Bin 0 -> 37 bytes tests/files/bad-1-block_header-1.lzma | Bin 0 -> 64 bytes tests/files/bad-1-block_header-2.lzma | Bin 0 -> 64 bytes tests/files/bad-1-block_header-3.lzma | Bin 0 -> 68 bytes tests/files/bad-1-block_header-4.lzma | Bin 0 -> 72 bytes tests/files/bad-1-check-crc32.lzma | Bin 0 -> 68 bytes tests/files/bad-1-check-crc64.lzma | Bin 0 -> 72 bytes tests/files/bad-1-check-sha256.lzma | Bin 0 -> 96 bytes tests/files/bad-1-lzma2-1.lzma | Bin 0 -> 64 bytes tests/files/bad-1-lzma2-2.lzma | Bin 0 -> 424 bytes tests/files/bad-1-lzma2-3.lzma | Bin 0 -> 424 bytes tests/files/bad-1-lzma2-4.lzma | Bin 0 -> 408 bytes tests/files/bad-1-lzma2-5.lzma | Bin 0 -> 408 bytes tests/files/bad-1-lzma2-6.lzma | Bin 0 -> 68 bytes tests/files/bad-1-lzma2-7.lzma | Bin 0 -> 408 bytes tests/files/bad-1-stream_flags-1.lzma | Bin 0 -> 68 bytes tests/files/bad-1-stream_flags-2.lzma | Bin 0 -> 68 bytes tests/files/bad-1-stream_flags-3.lzma | Bin 0 -> 68 bytes tests/files/bad-1-vli-1.lzma | Bin 0 -> 72 bytes tests/files/bad-1-vli-2.lzma | Bin 0 -> 72 bytes .../files/bad-2-compressed_data_padding.lzma | Bin 0 -> 92 bytes tests/files/bad-2-index-1.lzma | Bin 0 -> 92 bytes tests/files/bad-2-index-2.lzma | Bin 0 -> 92 bytes tests/files/bad-2-index-3.lzma | Bin 0 -> 92 bytes tests/files/bad-2-index-4.lzma | Bin 0 -> 92 bytes .../bad-cat-single-none-pad_garbage_1.lzma | Bin 65 -> 0 bytes .../bad-cat-single-none-pad_garbage_2.lzma | Bin 65 -> 0 bytes .../bad-cat-single-none-pad_garbage_3.lzma | Bin 65 -> 0 bytes tests/files/bad-multi-none-1.lzma | Bin 54 -> 0 bytes tests/files/bad-multi-none-2.lzma | Bin 53 -> 0 bytes tests/files/bad-multi-none-3.lzma | Bin 53 -> 0 bytes tests/files/bad-multi-none-block_1.lzma | Bin 66 -> 0 bytes tests/files/bad-multi-none-block_2.lzma | Bin 66 -> 0 bytes tests/files/bad-multi-none-block_3.lzma | Bin 58 -> 0 bytes tests/files/bad-multi-none-extra_1.lzma | Bin 54 -> 0 bytes tests/files/bad-multi-none-extra_2.lzma | Bin 54 -> 0 bytes tests/files/bad-multi-none-extra_3.lzma | Bin 55 -> 0 bytes tests/files/bad-multi-none-header_1.lzma | Bin 57 -> 0 bytes tests/files/bad-multi-none-header_2.lzma | Bin 61 -> 0 bytes tests/files/bad-multi-none-header_3.lzma | Bin 59 -> 0 bytes tests/files/bad-multi-none-header_4.lzma | Bin 59 -> 0 bytes tests/files/bad-multi-none-header_5.lzma | Bin 58 -> 0 bytes tests/files/bad-multi-none-header_6.lzma | Bin 59 -> 0 bytes tests/files/bad-multi-none-header_7.lzma | Bin 59 -> 0 bytes tests/files/bad-multi-none-index_1.lzma | Bin 51 -> 0 bytes tests/files/bad-multi-none-index_2.lzma | Bin 49 -> 0 bytes tests/files/bad-multi-none-index_3.lzma | Bin 51 -> 0 bytes tests/files/bad-multi-none-index_4.lzma | Bin 51 -> 0 bytes tests/files/bad-single-data_after_eopm_1.lzma | Bin 55 -> 0 bytes tests/files/bad-single-data_after_eopm_2.lzma | Bin 56 -> 0 bytes .../bad-single-lzma-flush_beginning.lzma | Bin 53 -> 0 bytes tests/files/bad-single-lzma-flush_twice.lzma | Bin 63 -> 0 bytes tests/files/bad-single-none-empty.lzma | Bin 19 -> 0 bytes .../bad-single-none-footer_filter_flags.lzma | Bin 30 -> 0 bytes tests/files/bad-single-none-too_long_vli.lzma | Bin 39 -> 0 bytes tests/files/bad-single-none-truncated.lzma | Bin 29 -> 0 bytes .../bad-single-subblock-padding_loop.lzma | Bin 43 -> 0 bytes tests/files/bad-single-subblock1023-slow.lzma | Bin 7886 -> 0 bytes tests/files/bad-single-subblock_subblock.lzma | Bin 26 -> 0 bytes tests/files/good-0-empty.lzma | Bin 0 -> 32 bytes tests/files/good-0cat-empty.lzma | Bin 0 -> 64 bytes tests/files/good-0catpad-empty.lzma | Bin 0 -> 68 bytes tests/files/good-0pad-empty.lzma | Bin 0 -> 36 bytes tests/files/good-1-3delta-lzma2.lzma | Bin 0 -> 528 bytes tests/files/good-1-block_header-1.lzma | Bin 0 -> 72 bytes tests/files/good-1-block_header-2.lzma | Bin 0 -> 68 bytes tests/files/good-1-block_header-3.lzma | Bin 0 -> 68 bytes tests/files/good-1-check-crc32.lzma | Bin 0 -> 68 bytes tests/files/good-1-check-crc64.lzma | Bin 0 -> 72 bytes tests/files/good-1-check-none.lzma | Bin 0 -> 64 bytes tests/files/good-1-check-sha256.lzma | Bin 0 -> 96 bytes tests/files/good-1-delta-lzma2.tiff.lzma | Bin 0 -> 51312 bytes tests/files/good-1-lzma2-1.lzma | Bin 0 -> 424 bytes tests/files/good-1-lzma2-2.lzma | Bin 0 -> 424 bytes tests/files/good-1-lzma2-3.lzma | Bin 0 -> 408 bytes ...parc-lzma.lzma => good-1-sparc-lzma2.lzma} | Bin 2263 -> 2292 bytes ...le-x86-lzma.lzma => good-1-x86-lzma2.lzma} | Bin 1909 -> 1936 bytes tests/files/good-2-lzma2.lzma | Bin 0 -> 92 bytes tests/files/good-cat-single-none-pad.lzma | Bin 64 -> 0 bytes tests/files/good-multi-none-1.lzma | Bin 75 -> 0 bytes tests/files/good-multi-none-2.lzma | Bin 53 -> 0 bytes tests/files/good-multi-none-block_1.lzma | Bin 66 -> 0 bytes tests/files/good-multi-none-block_2.lzma | Bin 58 -> 0 bytes tests/files/good-multi-none-extra_1.lzma | Bin 51 -> 0 bytes tests/files/good-multi-none-extra_2.lzma | Bin 79 -> 0 bytes tests/files/good-multi-none-extra_3.lzma | Bin 55 -> 0 bytes tests/files/good-multi-none-header_1.lzma | Bin 58 -> 0 bytes tests/files/good-multi-none-header_2.lzma | Bin 66 -> 0 bytes tests/files/good-multi-none-header_3.lzma | Bin 59 -> 0 bytes tests/files/good-single-delta-lzma.tiff.lzma | Bin 51409 -> 0 bytes tests/files/good-single-lzma-empty.lzma | Bin 21 -> 0 bytes tests/files/good-single-lzma-flush_1.lzma | Bin 48 -> 0 bytes tests/files/good-single-lzma-flush_2.lzma | Bin 63 -> 0 bytes tests/files/good-single-lzma.lzma | Bin 44 -> 0 bytes tests/files/good-single-none-empty_1.lzma | Bin 18 -> 0 bytes tests/files/good-single-none-empty_2.lzma | Bin 26 -> 0 bytes tests/files/good-single-none-empty_3.lzma | Bin 19 -> 0 bytes tests/files/good-single-none-pad.lzma | Bin 32 -> 0 bytes tests/files/good-single-none.lzma | Bin 30 -> 0 bytes tests/files/good-single-subblock-lzma.lzma | Bin 50 -> 0 bytes .../files/good-single-subblock_implicit.lzma | Bin 35 -> 0 bytes tests/files/good-single-subblock_rle.lzma | Bin 118 -> 0 bytes .../files/malicious-multi-metadata-64PiB.lzma | Bin 51 -> 0 bytes .../malicious-single-subblock-256MiB.lzma | Bin 30 -> 0 bytes .../malicious-single-subblock-64PiB.lzma | Bin 45 -> 0 bytes .../malicious-single-subblock31-slow.lzma | Bin 1233 -> 0 bytes tests/files/unsupported-block_header.lzma | Bin 0 -> 68 bytes tests/files/unsupported-check.lzma | Bin 0 -> 68 bytes tests/files/unsupported-filter_flags-1.lzma | Bin 0 -> 68 bytes tests/files/unsupported-filter_flags-2.lzma | Bin 0 -> 68 bytes tests/files/unsupported-filter_flags-3.lzma | Bin 0 -> 68 bytes tests/test_block_header.c | 28 +- tests/test_compress.sh | 4 +- tests/test_filter_flags.c | 51 +- tests/test_stream_flags.c | 4 +- tests/tests.h | 8 + 277 files changed, 9164 insertions(+), 7591 deletions(-) rename src/liblzma/lz/lz_encoder_private.h => debug/crc32.c (52%) create mode 100644 debug/hex2bin.c create mode 100644 debug/known_sizes.c delete mode 100644 src/liblzma/api/lzma/alone.h delete mode 100644 src/liblzma/api/lzma/auto.h create mode 100644 src/liblzma/api/lzma/container.h delete mode 100644 src/liblzma/api/lzma/easy.h delete mode 100644 src/liblzma/api/lzma/raw.h delete mode 100644 src/liblzma/api/lzma/stream.h delete mode 100644 src/liblzma/common/allocator.c rename src/liblzma/common/{code.c => common.c} (64%) delete mode 100644 src/liblzma/common/features.c create mode 100644 src/liblzma/common/filter_common.c create mode 100644 src/liblzma/common/filter_common.h create mode 100644 src/liblzma/common/filter_decoder.c create mode 100644 src/liblzma/common/filter_decoder.h create mode 100644 src/liblzma/common/filter_encoder.c create mode 100644 src/liblzma/common/filter_encoder.h delete mode 100644 src/liblzma/common/memory_usage.c delete mode 100644 src/liblzma/common/next_coder.c delete mode 100644 src/liblzma/common/raw_common.c delete mode 100644 src/liblzma/common/raw_decoder.c delete mode 100644 src/liblzma/common/raw_encoder.c delete mode 100644 src/liblzma/common/stream_common.c rename src/liblzma/common/{stream_flags_equal.c => stream_flags_common.c} (72%) rename src/liblzma/common/{stream_common.h => stream_flags_common.h} (85%) rename src/liblzma/common/{version.c => vli_size.c} (67%) create mode 100644 src/liblzma/delta/Makefile.am rename src/liblzma/{common => delta}/delta_common.c (97%) rename src/liblzma/{common => delta}/delta_common.h (100%) rename src/liblzma/{common => delta}/delta_decoder.c (81%) rename src/liblzma/{common => delta}/delta_decoder.h (88%) rename src/liblzma/{common => delta}/delta_encoder.c (85%) rename src/liblzma/{common => delta}/delta_encoder.h (92%) delete mode 100644 src/liblzma/lz/bt2.c delete mode 100644 src/liblzma/lz/bt2.h delete mode 100644 src/liblzma/lz/bt3.c delete mode 100644 src/liblzma/lz/bt3.h delete mode 100644 src/liblzma/lz/bt4.c delete mode 100644 src/liblzma/lz/bt4.h delete mode 100644 src/liblzma/lz/hc3.c delete mode 100644 src/liblzma/lz/hc3.h delete mode 100644 src/liblzma/lz/hc4.c delete mode 100644 src/liblzma/lz/hc4.h create mode 100644 src/liblzma/lz/lz_encoder_hash.h create mode 100644 src/liblzma/lz/lz_encoder_mf.c delete mode 100644 src/liblzma/lz/match_c.h delete mode 100644 src/liblzma/lz/match_h.h create mode 100644 src/liblzma/lzma/lzma2_decoder.c create mode 100644 src/liblzma/lzma/lzma2_decoder.h create mode 100644 src/liblzma/lzma/lzma2_encoder.c rename src/liblzma/{common/raw_common.h => lzma/lzma2_encoder.h} (60%) delete mode 100644 src/liblzma/lzma/lzma_encoder_getoptimum.c delete mode 100644 src/liblzma/lzma/lzma_encoder_getoptimumfast.c delete mode 100644 src/liblzma/lzma/lzma_encoder_init.c create mode 100644 src/liblzma/lzma/lzma_encoder_optimum_fast.c create mode 100644 src/liblzma/lzma/lzma_encoder_optimum_normal.c delete mode 100644 src/liblzma/lzma/lzma_literal.c delete mode 100644 src/liblzma/lzma/lzma_literal.h create mode 100644 src/liblzma/rangecoder/price.h rename src/liblzma/rangecoder/{price_table_gen.c => price_tablegen.c} (75%) create mode 100644 src/liblzma/simple/simple_decoder.c rename src/liblzma/{common/raw_decoder.h => simple/simple_decoder.h} (67%) create mode 100644 src/liblzma/simple/simple_encoder.c rename src/liblzma/{common/raw_encoder.h => simple/simple_encoder.h} (65%) create mode 100644 tests/files/bad-0-backward_size.lzma create mode 100644 tests/files/bad-0-empty-truncated.lzma create mode 100644 tests/files/bad-0-nonempty_index.lzma create mode 100644 tests/files/bad-0cat-alone.lzma create mode 100644 tests/files/bad-0catpad-empty.lzma create mode 100644 tests/files/bad-0pad-empty.lzma create mode 100644 tests/files/bad-1-block_header-1.lzma create mode 100644 tests/files/bad-1-block_header-2.lzma create mode 100644 tests/files/bad-1-block_header-3.lzma create mode 100644 tests/files/bad-1-block_header-4.lzma create mode 100644 tests/files/bad-1-check-crc32.lzma create mode 100644 tests/files/bad-1-check-crc64.lzma create mode 100644 tests/files/bad-1-check-sha256.lzma create mode 100644 tests/files/bad-1-lzma2-1.lzma create mode 100644 tests/files/bad-1-lzma2-2.lzma create mode 100644 tests/files/bad-1-lzma2-3.lzma create mode 100644 tests/files/bad-1-lzma2-4.lzma create mode 100644 tests/files/bad-1-lzma2-5.lzma create mode 100644 tests/files/bad-1-lzma2-6.lzma create mode 100644 tests/files/bad-1-lzma2-7.lzma create mode 100644 tests/files/bad-1-stream_flags-1.lzma create mode 100644 tests/files/bad-1-stream_flags-2.lzma create mode 100644 tests/files/bad-1-stream_flags-3.lzma create mode 100644 tests/files/bad-1-vli-1.lzma create mode 100644 tests/files/bad-1-vli-2.lzma create mode 100644 tests/files/bad-2-compressed_data_padding.lzma create mode 100644 tests/files/bad-2-index-1.lzma create mode 100644 tests/files/bad-2-index-2.lzma create mode 100644 tests/files/bad-2-index-3.lzma create mode 100644 tests/files/bad-2-index-4.lzma delete mode 100644 tests/files/bad-cat-single-none-pad_garbage_1.lzma delete mode 100644 tests/files/bad-cat-single-none-pad_garbage_2.lzma delete mode 100644 tests/files/bad-cat-single-none-pad_garbage_3.lzma delete mode 100644 tests/files/bad-multi-none-1.lzma delete mode 100644 tests/files/bad-multi-none-2.lzma delete mode 100644 tests/files/bad-multi-none-3.lzma delete mode 100644 tests/files/bad-multi-none-block_1.lzma delete mode 100644 tests/files/bad-multi-none-block_2.lzma delete mode 100644 tests/files/bad-multi-none-block_3.lzma delete mode 100644 tests/files/bad-multi-none-extra_1.lzma delete mode 100644 tests/files/bad-multi-none-extra_2.lzma delete mode 100644 tests/files/bad-multi-none-extra_3.lzma delete mode 100644 tests/files/bad-multi-none-header_1.lzma delete mode 100644 tests/files/bad-multi-none-header_2.lzma delete mode 100644 tests/files/bad-multi-none-header_3.lzma delete mode 100644 tests/files/bad-multi-none-header_4.lzma delete mode 100644 tests/files/bad-multi-none-header_5.lzma delete mode 100644 tests/files/bad-multi-none-header_6.lzma delete mode 100644 tests/files/bad-multi-none-header_7.lzma delete mode 100644 tests/files/bad-multi-none-index_1.lzma delete mode 100644 tests/files/bad-multi-none-index_2.lzma delete mode 100644 tests/files/bad-multi-none-index_3.lzma delete mode 100644 tests/files/bad-multi-none-index_4.lzma delete mode 100644 tests/files/bad-single-data_after_eopm_1.lzma delete mode 100644 tests/files/bad-single-data_after_eopm_2.lzma delete mode 100644 tests/files/bad-single-lzma-flush_beginning.lzma delete mode 100644 tests/files/bad-single-lzma-flush_twice.lzma delete mode 100644 tests/files/bad-single-none-empty.lzma delete mode 100644 tests/files/bad-single-none-footer_filter_flags.lzma delete mode 100644 tests/files/bad-single-none-too_long_vli.lzma delete mode 100644 tests/files/bad-single-none-truncated.lzma delete mode 100644 tests/files/bad-single-subblock-padding_loop.lzma delete mode 100644 tests/files/bad-single-subblock1023-slow.lzma delete mode 100644 tests/files/bad-single-subblock_subblock.lzma create mode 100644 tests/files/good-0-empty.lzma create mode 100644 tests/files/good-0cat-empty.lzma create mode 100644 tests/files/good-0catpad-empty.lzma create mode 100644 tests/files/good-0pad-empty.lzma create mode 100644 tests/files/good-1-3delta-lzma2.lzma create mode 100644 tests/files/good-1-block_header-1.lzma create mode 100644 tests/files/good-1-block_header-2.lzma create mode 100644 tests/files/good-1-block_header-3.lzma create mode 100644 tests/files/good-1-check-crc32.lzma create mode 100644 tests/files/good-1-check-crc64.lzma create mode 100644 tests/files/good-1-check-none.lzma create mode 100644 tests/files/good-1-check-sha256.lzma create mode 100644 tests/files/good-1-delta-lzma2.tiff.lzma create mode 100644 tests/files/good-1-lzma2-1.lzma create mode 100644 tests/files/good-1-lzma2-2.lzma create mode 100644 tests/files/good-1-lzma2-3.lzma rename tests/files/{good-single-sparc-lzma.lzma => good-1-sparc-lzma2.lzma} (95%) rename tests/files/{good-single-x86-lzma.lzma => good-1-x86-lzma2.lzma} (77%) create mode 100644 tests/files/good-2-lzma2.lzma delete mode 100644 tests/files/good-cat-single-none-pad.lzma delete mode 100644 tests/files/good-multi-none-1.lzma delete mode 100644 tests/files/good-multi-none-2.lzma delete mode 100644 tests/files/good-multi-none-block_1.lzma delete mode 100644 tests/files/good-multi-none-block_2.lzma delete mode 100644 tests/files/good-multi-none-extra_1.lzma delete mode 100644 tests/files/good-multi-none-extra_2.lzma delete mode 100644 tests/files/good-multi-none-extra_3.lzma delete mode 100644 tests/files/good-multi-none-header_1.lzma delete mode 100644 tests/files/good-multi-none-header_2.lzma delete mode 100644 tests/files/good-multi-none-header_3.lzma delete mode 100644 tests/files/good-single-delta-lzma.tiff.lzma delete mode 100644 tests/files/good-single-lzma-empty.lzma delete mode 100644 tests/files/good-single-lzma-flush_1.lzma delete mode 100644 tests/files/good-single-lzma-flush_2.lzma delete mode 100644 tests/files/good-single-lzma.lzma delete mode 100644 tests/files/good-single-none-empty_1.lzma delete mode 100644 tests/files/good-single-none-empty_2.lzma delete mode 100644 tests/files/good-single-none-empty_3.lzma delete mode 100644 tests/files/good-single-none-pad.lzma delete mode 100644 tests/files/good-single-none.lzma delete mode 100644 tests/files/good-single-subblock-lzma.lzma delete mode 100644 tests/files/good-single-subblock_implicit.lzma delete mode 100644 tests/files/good-single-subblock_rle.lzma delete mode 100644 tests/files/malicious-multi-metadata-64PiB.lzma delete mode 100644 tests/files/malicious-single-subblock-256MiB.lzma delete mode 100644 tests/files/malicious-single-subblock-64PiB.lzma delete mode 100644 tests/files/malicious-single-subblock31-slow.lzma create mode 100644 tests/files/unsupported-block_header.lzma create mode 100644 tests/files/unsupported-check.lzma create mode 100644 tests/files/unsupported-filter_flags-1.lzma create mode 100644 tests/files/unsupported-filter_flags-2.lzma create mode 100644 tests/files/unsupported-filter_flags-3.lzma diff --git a/configure.ac b/configure.ac index 761b74c6..f6aff3c3 100644 --- a/configure.ac +++ b/configure.ac @@ -39,10 +39,15 @@ echo "System type:" # This is needed to know if assembler optimizations can be used. AC_CANONICAL_HOST + echo echo "Configure options:" -# Enable/disable debugging code: + +############# +# Debugging # +############# + AC_MSG_CHECKING([if debugging code should be compiled]) AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug], [Enable debugging code.]), [], enable_debug=no) @@ -53,7 +58,11 @@ else AC_MSG_RESULT([no]) fi -# Enable/disable the encoder components: + +########### +# Encoder # +########### + AC_MSG_CHECKING([if encoder components should be built]) AC_ARG_ENABLE(encoder, AC_HELP_STRING([--disable-encoder], [Do not build the encoder components.]), @@ -67,7 +76,11 @@ else fi AM_CONDITIONAL(COND_MAIN_ENCODER, test "x$enable_encoder" = xyes) -# Enable/disable the decoder components: + +########### +# Decoder # +########### + AC_MSG_CHECKING([if decoder components should be built]) AC_ARG_ENABLE(decoder, AC_HELP_STRING([--disable-decoder], [Do not build the decoder components.]), @@ -84,146 +97,146 @@ else fi AM_CONDITIONAL(COND_MAIN_DECODER, test "x$enable_decoder" = xyes) -# Filters -AC_MSG_CHECKING([which filters to build]) -AC_ARG_ENABLE(filters, AC_HELP_STRING([--enable-filters=LIST], - [Comma-separated list of filters to build. Default=all. - Filters used in encoding are needed also in decoding. - Available filters: copy subblock x86 powerpc ia64 - arm armthumb sparc delta lzma]), - [], [enable_filters=copy,subblock,x86,powerpc,ia64,arm,armthumb,sparc,delta,lzma]) -enable_filters=`echo "$enable_filters" | sed 's/,/ /g'` -enable_filters_copy=no -enable_filters_subblock=no -enable_filters_x86=no -enable_filters_powerpc=no -enable_filters_ia64=no -enable_filters_arm=no -enable_filters_armthumb=no -enable_filters_sparc=no -enable_filters_delta=no -enable_filters_lzma=no -enable_simple_filters=no -if test "x$enable_filters" = xno || test "x$enable_filters" = x; then - AC_MSG_RESULT([]) - AC_MSG_ERROR([Please enable at least one filter.]) + +########### +# Filters # +########### + +m4_define([SUPPORTED_FILTERS], [lzma,lzma2,subblock,delta,x86,powerpc,ia64,arm,armthumb,sparc])dnl +m4_define([SIMPLE_FILTERS], [x86,powerpc,ia64,arm,armthumb,sparc]) +m4_define([LZ_FILTERS], [lzma,lzma2]) + +m4_foreach([NAME], [SUPPORTED_FILTERS], +[enable_filter_[]NAME=no +enable_encoder_[]NAME=no +enable_decoder_[]NAME=no +])dnl + +AC_MSG_CHECKING([which encoders to build]) +AC_ARG_ENABLE([encoders], AC_HELP_STRING([--enable-encoders=LIST], + [Comma-separated list of encoders to build. Default=all. + Available encoders:] + m4_translit(m4_defn([SUPPORTED_FILTERS]), [,], [ ])), + [], [enable_encoders=SUPPORTED_FILTERS]) +enable_encoders=`echo "$enable_encoders" | sed 's/,/ /g'` +if test "x$enable_encoders" = xno || test "x$enable_encoders" = x; then + AC_MSG_RESULT([(none)]) else - for arg in $enable_filters + for arg in $enable_encoders do - case $arg in - copy) - enable_filters_copy=yes - AC_DEFINE([HAVE_FILTER_COPY], 1, - [Define to 1 if support for the - Copy filter is enabled.]) - ;; - subblock) - enable_filters_subblock=yes - AC_DEFINE([HAVE_FILTER_SUBBLOCK], 1, - [Define to 1 if support for the - Subblock filter is enabled.]) - ;; - x86) - enable_filters_x86=yes - enable_simple_filters=yes - AC_DEFINE([HAVE_FILTER_X86], 1, - [Define to 1 if support for the - x86 (BCJ) filter is enabled.]) - ;; - powerpc) - enable_filters_powerpc=yes - enable_simple_filters=yes - AC_DEFINE([HAVE_FILTER_POWERPC], 1, - [Define to 1 if support for the - PowerPC filter is enabled.]) - ;; - ia64) - enable_filters_ia64=yes - enable_simple_filters=yes - AC_DEFINE([HAVE_FILTER_IA64], 1, - [Define to 1 if support for the - IA64 filter is enabled.]) - ;; - arm) - enable_filters_arm=yes - enable_simple_filters=yes - AC_DEFINE([HAVE_FILTER_ARM], 1, - [Define to 1 if support for the - ARM filter is enabled.]) - ;; - armthumb) - enable_filters_armthumb=yes - enable_simple_filters=yes - AC_DEFINE([HAVE_FILTER_ARMTHUMB], 1, - [Define to 1 if support for the - ARMThumb filter is enabled.]) - ;; - sparc) - enable_filters_sparc=yes - enable_simple_filters=yes - AC_DEFINE([HAVE_FILTER_SPARC], 1, - [Define to 1 if support for the - SPARC filter is enabled.]) - ;; - delta) - enable_filters_delta=yes - AC_DEFINE([HAVE_FILTER_DELTA], 1, - [Define to 1 if support for the - Delta filter is enabled.]) - ;; - lzma) - enable_filters_lzma=yes - AC_DEFINE([HAVE_FILTER_LZMA], 1, - [Define to 1 if support for the - LZMA filter is enabled.]) - ;; + case $arg in m4_foreach([NAME], [SUPPORTED_FILTERS], [ + NAME) + enable_filter_[]NAME=yes + enable_encoder_[]NAME=yes + AC_DEFINE(HAVE_ENCODER_[]m4_toupper(NAME), [1], + [Define to 1 if] NAME [encoder is enabled.]) + ;;]) *) AC_MSG_RESULT([]) AC_MSG_ERROR([unknown filter: $arg]) ;; esac done - AC_MSG_RESULT([$enable_filters]) + AC_MSG_RESULT([$enable_encoders]) fi -if test "x$enable_simple_filters" = xyes ; then - AC_DEFINE([HAVE_FILTER_SIMPLE], 1, [Define to 1 if support for any - of the so called simple filters is enabled.]) -fi -AM_CONDITIONAL(COND_FILTER_COPY, test "x$enable_filters_copy" = xyes) -AM_CONDITIONAL(COND_FILTER_SUBBLOCK, test "x$enable_filters_subblock" = xyes) -AM_CONDITIONAL(COND_FILTER_X86, test "x$enable_filters_x86" = xyes) -AM_CONDITIONAL(COND_FILTER_POWERPC, test "x$enable_filters_powerpc" = xyes) -AM_CONDITIONAL(COND_FILTER_IA64, test "x$enable_filters_ia64" = xyes) -AM_CONDITIONAL(COND_FILTER_ARM, test "x$enable_filters_arm" = xyes) -AM_CONDITIONAL(COND_FILTER_ARMTHUMB, test "x$enable_filters_armthumb" = xyes) -AM_CONDITIONAL(COND_FILTER_SPARC, test "x$enable_filters_sparc" = xyes) -AM_CONDITIONAL(COND_FILTER_DELTA, test "x$enable_filters_delta" = xyes) -AM_CONDITIONAL(COND_FILTER_LZMA, test "x$enable_filters_lzma" = xyes) -AM_CONDITIONAL(COND_MAIN_SIMPLE, test "x$enable_simple_filters" = xyes) -# Which match finders should be enabled: +AC_MSG_CHECKING([which decoders to build]) +AC_ARG_ENABLE([decoders], AC_HELP_STRING([--enable-decoders=LIST], + [Comma-separated list of decoders to build. Default=all. + Available decoders are the same as available encoders.]), + [], [enable_decoders=SUPPORTED_FILTERS]) +enable_decoders=`echo "$enable_decoders" | sed 's/,/ /g'` +if test "x$enable_decoders" = xno || test "x$enable_decoders" = x; then + AC_MSG_RESULT([(none)]) +else + for arg in $enable_decoders + do + case $arg in m4_foreach([NAME], [SUPPORTED_FILTERS], [ + NAME) + enable_filter_[]NAME=yes + enable_decoder_[]NAME=yes + AC_DEFINE(HAVE_DECODER_[]m4_toupper(NAME), [1], + [Define to 1 if] NAME [decoder is enabled.]) + ;;]) + *) + AC_MSG_RESULT([]) + AC_MSG_ERROR([unknown filter: $arg]) + ;; + esac + done + + # LZMA2 requires that LZMA is enabled. + test "x$enable_encoder_lzma2" = xyes && enable_encoder_lzma=yes + test "x$enable_decoder_lzma2" = xyes && enable_decoder_lzma=yes + + AC_MSG_RESULT([$enable_decoders]) +fi + +if test "x$enable_encoder_lzma2$enable_encoder_lzma" = xyesno \ + || test "x$enable_decoder_lzma2$enable_decoder_lzma" = xyesno; then + AC_MSG_ERROR([LZMA2 requires that LZMA is also enabled.]) +fi + +m4_foreach([NAME], [SUPPORTED_FILTERS], +[AM_CONDITIONAL(COND_FILTER_[]m4_toupper(NAME), test "x$enable_filter_[]NAME" = xyes) +AM_CONDITIONAL(COND_ENCODER_[]m4_toupper(NAME), test "x$enable_encoder_[]NAME" = xyes) +AM_CONDITIONAL(COND_DECODER_[]m4_toupper(NAME), test "x$enable_decoder_[]NAME" = xyes) +])dnl + +# The so called "simple filters" share common code. +enable_filter_simple=no +enable_encoder_simple=no +enable_decoder_simple=no +m4_foreach([NAME], [SIMPLE_FILTERS], +[test "x$enable_filter_[]NAME" = xyes && enable_filter_simple=yes +test "x$enable_encoder_[]NAME" = xyes && enable_encoder_simple=yes +test "x$enable_decoder_[]NAME" = xyes && enable_decoder_simple=yes +])dnl +AM_CONDITIONAL(COND_FILTER_SIMPLE, test "x$enable_filter_simple" = xyes) +AM_CONDITIONAL(COND_ENCODER_SIMPLE, test "x$enable_encoder_simple" = xyes) +AM_CONDITIONAL(COND_DECODER_SIMPLE, test "x$enable_decoder_simple" = xyes) + +# LZ-based filters share common code. +enable_filter_lz=no +enable_encoder_lz=no +enable_decoder_lz=no +m4_foreach([NAME], [LZ_FILTERS], +[test "x$enable_filter_[]NAME" = xyes && enable_filter_lz=yes +test "x$enable_encoder_[]NAME" = xyes && enable_encoder_lz=yes +test "x$enable_decoder_[]NAME" = xyes && enable_decoder_lz=yes +])dnl +AM_CONDITIONAL(COND_FILTER_LZ, test "x$enable_filter_lz" = xyes) +AM_CONDITIONAL(COND_ENCODER_LZ, test "x$enable_encoder_lz" = xyes) +AM_CONDITIONAL(COND_DECODER_LZ, test "x$enable_decoder_lz" = xyes) + + +################# +# Match finders # +################# + +m4_define([SUPPORTED_MATCH_FINDERS], [hc3,hc4,bt2,bt3,bt4]) + +m4_foreach([NAME], [SUPPORTED_MATCH_FINDERS], +[enable_match_finder_[]NAME=no +]) + AC_MSG_CHECKING([which match finders to build]) AC_ARG_ENABLE(match-finders, AC_HELP_STRING([--enable-match-finders=LIST], [Comma-separated list of match finders to build. Default=all. At least one match finder is required for encoding with - the LZMA filter. - Available match finders: hc3 hc4 bt2 bt3 bt4]), [], - [enable_match_finders=hc3,hc4,bt2,bt3,bt4]) + the LZMA filter. Available match finders:] + m4_translit(m4_defn([SUPPORTED_MATCH_FINDERS]), [,], [ ])), [], + [enable_match_finders=SUPPORTED_MATCH_FINDERS]) enable_match_finders=`echo "$enable_match_finders" | sed 's/,/ /g'` -enable_match_finders_hc3=no -enable_match_finders_hc4=no -enable_match_finders_bt2=no -enable_match_finders_bt3=no -enable_match_finders_bt4=no -if test "x$enable_encoder" = xyes && test "x$enable_filters_lzma" = xyes ; then +if test "x$enable_encoder" = xyes && test "x$enable_encoder_lz" = xyes ; then for arg in $enable_match_finders do - case $arg in - hc3) enable_match_finders_hc3=yes ;; - hc4) enable_match_finders_hc4=yes ;; - bt2) enable_match_finders_bt2=yes ;; - bt3) enable_match_finders_bt3=yes ;; - bt4) enable_match_finders_bt4=yes ;; + case $arg in m4_foreach([NAME], [SUPPORTED_MATCH_FINDERS], [ + NAME) + enable_match_finder_[]NAME=yes + AC_DEFINE(HAVE_MF_[]m4_toupper(NAME), [1], + [Define to 1 to enable] NAME [match finder.]) + ;;]) *) AC_MSG_RESULT([]) AC_MSG_ERROR([unknown match finder: $arg]) @@ -232,48 +245,39 @@ if test "x$enable_encoder" = xyes && test "x$enable_filters_lzma" = xyes ; then done AC_MSG_RESULT([$enable_match_finders]) else - AC_MSG_RESULT([(none because not building the LZMA encoder)]) + AC_MSG_RESULT([(none because not building any LZ-based encoder)]) fi -AM_CONDITIONAL(COND_MF_HC3, test "x$enable_match_finders_hc3" = xyes) -AM_CONDITIONAL(COND_MF_HC4, test "x$enable_match_finders_hc4" = xyes) -AM_CONDITIONAL(COND_MF_BT2, test "x$enable_match_finders_bt2" = xyes) -AM_CONDITIONAL(COND_MF_BT3, test "x$enable_match_finders_bt3" = xyes) -AM_CONDITIONAL(COND_MF_BT4, test "x$enable_match_finders_bt4" = xyes) -# Which integrity checks to build + +#################### +# Integrity checks # +#################### + +m4_define([SUPPORTED_CHECKS], [crc32,crc64,sha256]) + +m4_foreach([NAME], [SUPPORTED_FILTERS], +[enable_check_[]NAME=no +])dnl + AC_MSG_CHECKING([which integrity checks to build]) AC_ARG_ENABLE(checks, AC_HELP_STRING([--enable-checks=LIST], [Comma-separated list of integrity checks to build. - Default=all. Available integrity checks: crc32 crc64 sha256]), - [], [enable_checks=crc32,crc64,sha256]) + Default=all. Available integrity checks:] + m4_translit(m4_defn([SUPPORTED_CHECKS]), [,], [ ])), + [], [enable_checks=SUPPORTED_CHECKS]) enable_checks=`echo "$enable_checks" | sed 's/,/ /g'` -enable_checks_crc32=no -enable_checks_crc64=no -enable_checks_sha256=no if test "x$enable_checks" = xno || test "x$enable_checks" = x; then AC_MSG_RESULT([(none)]) else for arg in $enable_checks do - case $arg in - crc32) - enable_checks_crc32=yes - AC_DEFINE([HAVE_CHECK_CRC32], 1, - [Define to 1 if CRC32 support - is enabled.]) - ;; - crc64) - enable_checks_crc64=yes - AC_DEFINE([HAVE_CHECK_CRC64], 1, - [Define to 1 if CRC64 support - is enabled.]) - ;; - sha256) - enable_checks_sha256=yes - AC_DEFINE([HAVE_CHECK_SHA256], 1, - [Define to 1 if SHA256 support - is enabled.]) - ;; + case $arg in m4_foreach([NAME], [SUPPORTED_CHECKS], [ + NAME) + enable_check_[]NAME=yes + AC_DEFINE(HAVE_CHECK_[]m4_toupper(NAME), [1], + [Define to 1 if] NAME + [integrity check is enabled.]) + ;;]) *) AC_MSG_RESULT([]) AC_MSG_ERROR([unknown integrity check: $arg]) @@ -285,11 +289,16 @@ fi if test "x$enable_checks_crc32" = xno ; then AC_MSG_ERROR([For now, the CRC32 check must always be enabled.]) fi -AM_CONDITIONAL(COND_CHECK_CRC32, test "x$enable_checks_crc32" = xyes) -AM_CONDITIONAL(COND_CHECK_CRC64, test "x$enable_checks_crc64" = xyes) -AM_CONDITIONAL(COND_CHECK_SHA256, test "x$enable_checks_sha256" = xyes) -# Assembler optimizations +m4_foreach([NAME], [SUPPORTED_CHECKS], +[AM_CONDITIONAL(COND_CHECK_[]m4_toupper(NAME), test "x$enable_check_[]NAME" = xyes) +])dnl + + +########################### +# Assembler optimizations # +########################### + AC_MSG_CHECKING([if assembler optimizations should be used]) AC_ARG_ENABLE(assembler, AC_HELP_STRING([--disable-assembler], [Do not use assembler optimizations even if such exist @@ -321,13 +330,18 @@ case $enable_assembler in ;; *) AC_MSG_RESULT([]) - AC_MSG_ERROR([--enable-assembler accepts only \`yes', \`no', or \`x86'.]) + AC_MSG_ERROR([--enable-assembler accepts only \`yes', \`no', \`x86', or \`x86_64'.]) ;; esac AC_MSG_RESULT([$enable_assembler]) AM_CONDITIONAL(COND_ASM_X86, test "x$enable_assembler" = xx86) +AM_CONDITIONAL(COND_ASM_X86_64, test "x$enable_assembler" = xx86_64) + + +################################ +# Fast unaligned memory access # +################################ -# Fast unaligned memory access AC_MSG_CHECKING([if unaligned memory access should be used]) AC_ARG_ENABLE(unaligned-access, AC_HELP_STRING([--enable-unaligned-access], [Enable if the system supports *fast* unaligned memory access @@ -352,12 +366,15 @@ else AC_MSG_RESULT([no]) fi -# Size optimization + +##################### +# Size optimization # +##################### + AC_MSG_CHECKING([if small size is preferred over speed]) AC_ARG_ENABLE(small, AC_HELP_STRING([--enable-small], - [Omit precomputed tables to make liblzma a few kilobytes - smaller. This will increase startup time of applications - slightly, because the tables need to be computed first.]), + [Make liblzma smaller and a little slower. + This is disabled by default to optimize for speed.]), [], [enable_small=no]) if test "x$enable_small" = xyes; then AC_DEFINE([HAVE_SMALL], 1, [Define to 1 if optimizing for size.]) @@ -368,6 +385,7 @@ fi AC_MSG_RESULT([$enable_small]) AM_CONDITIONAL(COND_SMALL, test "x$enable_small" = xyes) + ############################################################################### # Checks for programs. ############################################################################### @@ -447,7 +465,6 @@ AC_TYPE_INT64_T AC_TYPE_UINT64_T AC_TYPE_UINTPTR_T -AC_CHECK_SIZEOF([unsigned long]) AC_CHECK_SIZEOF([size_t]) # The command line tool can copy high resolution timestamps if such @@ -662,11 +679,12 @@ AC_CONFIG_FILES([ src/liblzma/api/Makefile src/liblzma/common/Makefile src/liblzma/check/Makefile + src/liblzma/rangecoder/Makefile src/liblzma/lz/Makefile src/liblzma/lzma/Makefile - src/liblzma/simple/Makefile src/liblzma/subblock/Makefile - src/liblzma/rangecoder/Makefile + src/liblzma/delta/Makefile + src/liblzma/simple/Makefile src/lzma/Makefile src/lzmadec/Makefile src/scripts/Makefile diff --git a/debug/Makefile.am b/debug/Makefile.am index 71ca7e4c..6ed5bc99 100644 --- a/debug/Makefile.am +++ b/debug/Makefile.am @@ -16,7 +16,10 @@ noinst_PROGRAMS = \ repeat \ sync_flush \ full_flush \ - memusage + memusage \ + crc32 \ + known_sizes \ + hex2bin AM_CPPFLAGS = \ -I@top_srcdir@/src/common \ diff --git a/src/liblzma/lz/lz_encoder_private.h b/debug/crc32.c similarity index 52% rename from src/liblzma/lz/lz_encoder_private.h rename to debug/crc32.c index 638fcb2d..4052a863 100644 --- a/src/liblzma/lz/lz_encoder_private.h +++ b/debug/crc32.c @@ -1,10 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file lz_encoder_private.h -/// \brief Private definitions for LZ encoder +/// \file crc32.c +/// \brief Primitive CRC32 calculation tool // -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -18,23 +17,29 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_LZ_ENCODER_PRIVATE_H -#define LZMA_LZ_ENCODER_PRIVATE_H - -#include "lz_encoder.h" - -/// Value used to indicate unused slot -#define EMPTY_HASH_VALUE 0 - -/// When the dictionary and hash variables need to be adjusted to prevent -/// integer overflows. Since we use uint32_t to store the offsets, half -/// of it is the biggest safe limit. -#define MAX_VAL_FOR_NORMALIZE (UINT32_MAX / 2) +#include "sysdefs.h" +#include -struct lzma_coder_s { - lzma_next_coder next; - lzma_lz_encoder lz; -}; +int +main(void) +{ + uint32_t crc = 0; -#endif + do { + uint8_t buf[BUFSIZ]; + const size_t size = fread(buf, 1, sizeof(buf), stdin); + crc = lzma_crc32(buf, size, crc); + } while (!ferror(stdin) && !feof(stdin)); + + //printf("%08" PRIX32 "\n", crc); + + // I want it little endian so it's easy to work with hex editor. + printf("%02" PRIX32 " ", crc & 0xFF); + printf("%02" PRIX32 " ", (crc >> 8) & 0xFF); + printf("%02" PRIX32 " ", (crc >> 16) & 0xFF); + printf("%02" PRIX32 " ", crc >> 24); + printf("\n"); + + return 0; +} diff --git a/debug/full_flush.c b/debug/full_flush.c index fd775ce3..db82a60a 100644 --- a/debug/full_flush.c +++ b/debug/full_flush.c @@ -72,18 +72,24 @@ main(int argc, char **argv) file_in = argc > 1 ? fopen(argv[1], "rb") : stdin; + // Config - lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; - filters[0].id = LZMA_FILTER_SUBBLOCK; - filters[0].options = NULL; + lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; + filters[0].id = LZMA_FILTER_LZMA2; + filters[0].options = (void *)&lzma_preset_lzma[0]; filters[1].id = LZMA_VLI_VALUE_UNKNOWN; // Init - if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32) != LZMA_OK) { + if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_SHA256) != LZMA_OK) { fprintf(stderr, "init failed\n"); exit(1); } +// if (lzma_easy_encoder(&strm, 1)) { +// fprintf(stderr, "init failed\n"); +// exit(1); +// } + // Encoding encode(0, LZMA_FULL_FLUSH); encode(6, LZMA_FULL_FLUSH); diff --git a/debug/hex2bin.c b/debug/hex2bin.c new file mode 100644 index 00000000..ebfc289f --- /dev/null +++ b/debug/hex2bin.c @@ -0,0 +1,54 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file hex2bin.c +/// \brief Converts hexadecimal input strings to binary +// +// This code has been put into the public domain. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "sysdefs.h" +#include +#include + + +static int +getbin(int x) +{ + if (x >= '0' && x <= '9') + return x - '0'; + + if (x >= 'A' && x <= 'F') + return x - 'A' + 10; + + return x - 'a' + 10; +} + + +int +main(void) +{ + while (true) { + int byte = getchar(); + if (byte == EOF) + return 0; + if (!isxdigit(byte)) + continue; + + const int digit = getchar(); + if (digit == EOF || !isxdigit(digit)) { + fprintf(stderr, "Invalid input\n"); + return 1; + } + + byte = (getbin(byte) << 4) | getbin(digit); + if (putchar(byte) == EOF) { + perror(NULL); + return 1; + } + } +} diff --git a/debug/known_sizes.c b/debug/known_sizes.c new file mode 100644 index 00000000..571c105e --- /dev/null +++ b/debug/known_sizes.c @@ -0,0 +1,135 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file known_sizes.c +/// \brief Encodes .lzma Stream with sizes known in Block Header +/// +/// The input file is encoded in RAM, and the known Compressed Size +/// and/or Uncompressed Size values are stored in the Block Header. +/// As of writing there's no such Stream encoder in liblzma. +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "sysdefs.h" +#include +#include +#include +#include + + +// Support file sizes up to 1 MiB. We use this for output space too, so files +// close to 1 MiB had better compress at least a little or we have a buffer +// overflow. +#define BUFFER_SIZE (1U << 20) + + +int +main(void) +{ + // Allocate the buffers. + uint8_t *in = malloc(BUFFER_SIZE); + uint8_t *out = malloc(BUFFER_SIZE); + if (in == NULL || out == NULL) + return 1; + + // Fill the input buffer. + const size_t in_size = fread(in, 1, BUFFER_SIZE, stdin); + + // Filter setup + lzma_filter filters[] = { + { + .id = LZMA_FILTER_LZMA2, + .options = (void *)(&lzma_preset_lzma[0]) + }, + { + .id = LZMA_VLI_VALUE_UNKNOWN + } + }; + + lzma_block block = { + .check = LZMA_CHECK_CRC32, + .compressed_size = BUFFER_SIZE, // Worst case reserve + .uncompressed_size = in_size, + .filters = filters, + }; + + // FIXME Insane paranoia in liblzma. + if (lzma_block_header_size(&block) != LZMA_OK) + return 1; + + // We don't actually know the compressed size, so don't tell it to + // Block encoder. + block.compressed_size = LZMA_VLI_VALUE_UNKNOWN; + + lzma_stream strm = LZMA_STREAM_INIT; + if (lzma_block_encoder(&strm, &block) != LZMA_OK) + return 1; + + // Reserve space for Stream Header and Block Header. + size_t out_size = LZMA_STREAM_HEADER_SIZE + block.header_size; + + strm.next_in = in; + strm.avail_in = in_size; + strm.next_out = out + out_size; + strm.avail_out = BUFFER_SIZE - out_size; + + if (lzma_code(&strm, LZMA_FINISH) != LZMA_STREAM_END) + return 1; + + out_size += strm.total_out; + + if (lzma_block_header_encode(&block, out + LZMA_STREAM_HEADER_SIZE) + != LZMA_OK) + return 1; + + lzma_index *idx = lzma_index_init(NULL, NULL); + if (idx == NULL) + return 1; + + if (lzma_index_append(idx, NULL, block.header_size + strm.total_out, + strm.total_in) != LZMA_OK) + return 1; + + if (lzma_index_encoder(&strm, idx) != LZMA_OK) + return 1; + + if (lzma_code(&strm, LZMA_RUN) != LZMA_STREAM_END) + return 1; + + out_size += strm.total_out; + + lzma_end(&strm); + + lzma_index_end(idx, NULL); + + // Encode the Stream Header and Stream Footer. backwards_size is + // needed only for the Stream Footer. + lzma_stream_flags sf = { + .backward_size = strm.total_out, + .check = block.check, + }; + + if (lzma_stream_header_encode(&sf, out) != LZMA_OK) + return 1; + + if (lzma_stream_footer_encode(&sf, out + out_size) != LZMA_OK) + return 1; + + out_size += LZMA_STREAM_HEADER_SIZE; + + // Write out the file. + fwrite(out, 1, out_size, stdout); + + return 0; +} diff --git a/debug/memusage.c b/debug/memusage.c index 0716f5a2..eaf81f95 100644 --- a/debug/memusage.c +++ b/debug/memusage.c @@ -23,6 +23,7 @@ int main(void) { + lzma_init(); lzma_options_lzma lzma = { .dictionary_size = (1 << 27) + (1 << 26), @@ -31,7 +32,7 @@ main(void) .pos_bits = 2, .preset_dictionary = NULL, .preset_dictionary_size = 0, - .mode = LZMA_MODE_BEST, + .mode = LZMA_MODE_NORMAL, .fast_bytes = 48, .match_finder = LZMA_MF_BT4, .match_finder_cycles = 0, @@ -44,12 +45,13 @@ main(void) { UINT64_MAX, NULL } }; */ - lzma_options_filter filters[] = { + lzma_filter filters[] = { { LZMA_FILTER_LZMA, &lzma }, { UINT64_MAX, NULL } }; - printf("%u MiB\n", lzma_memory_usage(filters, true)); + printf("Encoder: %10" PRIu64 " B\n", lzma_memusage_encoder(filters)); + printf("Decoder: %10" PRIu64 " B\n", lzma_memusage_decoder(filters)); return 0; } diff --git a/debug/sync_flush.c b/debug/sync_flush.c index 03dfdd7d..eb6efef4 100644 --- a/debug/sync_flush.c +++ b/debug/sync_flush.c @@ -79,9 +79,10 @@ main(int argc, char **argv) .literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT, .pos_bits = LZMA_POS_BITS_DEFAULT, .preset_dictionary = NULL, - .mode = LZMA_MODE_BEST, + .persistent = true, + .mode = LZMA_MODE_NORMAL, .fast_bytes = 32, - .match_finder = LZMA_MF_BT3, + .match_finder = LZMA_MF_HC3, .match_finder_cycles = 0, }; @@ -101,24 +102,31 @@ main(int argc, char **argv) opt_subblock.subfilter_options.id = LZMA_FILTER_DELTA; opt_subblock.subfilter_options.options = &opt_delta; - lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; - filters[0].id = LZMA_FILTER_LZMA; + lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; + filters[0].id = LZMA_FILTER_LZMA2; filters[0].options = &opt_lzma; filters[1].id = LZMA_VLI_VALUE_UNKNOWN; // Init - if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_NONE) != LZMA_OK) { + if (lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC32) != LZMA_OK) { fprintf(stderr, "init failed\n"); exit(1); } // Encoding +/* encode(0, LZMA_SYNC_FLUSH); encode(6, LZMA_SYNC_FLUSH); encode(0, LZMA_SYNC_FLUSH); - encode(6, LZMA_SYNC_FLUSH); + encode(7, LZMA_SYNC_FLUSH); encode(0, LZMA_SYNC_FLUSH); encode(0, LZMA_FINISH); +*/ + encode(53, LZMA_SYNC_FLUSH); +// opt_lzma.literal_context_bits = 2; +// opt_lzma.literal_pos_bits = 1; +// opt_lzma.pos_bits = 0; + encode(404, LZMA_FINISH); // Clean up lzma_end(&strm); diff --git a/src/common/integer.h b/src/common/integer.h index 136a0f8d..a6e43be2 100644 --- a/src/common/integer.h +++ b/src/common/integer.h @@ -15,7 +15,7 @@ #define LZMA_INTEGER_H // I'm aware of AC_CHECK_ALIGNED_ACCESS_REQUIRED from Autoconf archive, but -// it's not useful for us. We don't care if unaligned access is supported, +// it's not useful here. We don't care if unaligned access is supported, // we care if it is fast. Some systems can emulate unaligned access in // software, which is horribly slow; we want to use byte-by-byte access on // such systems but the Autoconf test would detect such a system as @@ -32,13 +32,13 @@ // that also allow unaligned access. Inline assembler could be OK for that. #ifdef WORDS_BIGENDIAN # include "bswap.h" -# define integer_convert_16(n) bswap_16(n) -# define integer_convert_32(n) bswap_32(n) -# define integer_convert_64(n) bswap_64(n) +# define integer_le_16(n) bswap_16(n) +# define integer_le_32(n) bswap_32(n) +# define integer_le_64(n) bswap_64(n) #else -# define integer_convert_16(n) (n) -# define integer_convert_32(n) (n) -# define integer_convert_64(n) (n) +# define integer_le_16(n) (n) +# define integer_le_32(n) (n) +# define integer_le_64(n) (n) #endif @@ -46,7 +46,7 @@ static inline uint16_t integer_read_16(const uint8_t buf[static 2]) { uint16_t ret = *(const uint16_t *)(buf); - return integer_convert_16(ret); + return integer_le_16(ret); } @@ -54,7 +54,7 @@ static inline uint32_t integer_read_32(const uint8_t buf[static 4]) { uint32_t ret = *(const uint32_t *)(buf); - return integer_convert_32(ret); + return integer_le_32(ret); } @@ -63,7 +63,7 @@ static inline uint64_t integer_read_64(const uint8_t buf[static 8]) { uint64_t ret = *(const uint64_t *)(buf); - return integer_convert_64(ret); + return integer_le_64(ret); } */ @@ -71,14 +71,14 @@ integer_read_64(const uint8_t buf[static 8]) static inline void integer_write_16(uint8_t buf[static 2], uint16_t num) { - *(uint16_t *)(buf) = integer_convert_16(num); + *(uint16_t *)(buf) = integer_le_16(num); } static inline void integer_write_32(uint8_t buf[static 4], uint32_t num) { - *(uint32_t *)(buf) = integer_convert_32(num); + *(uint32_t *)(buf) = integer_le_32(num); } @@ -86,7 +86,7 @@ integer_write_32(uint8_t buf[static 4], uint32_t num) static inline void integer_write_64(uint8_t buf[static 8], uint64_t num) { - *(uint64_t *)(buf) = integer_convert_64(num); + *(uint64_t *)(buf) = integer_le_64(num); } */ diff --git a/src/common/sysdefs.h b/src/common/sysdefs.h index 2c7fb6ff..7f935f67 100644 --- a/src/common/sysdefs.h +++ b/src/common/sysdefs.h @@ -31,12 +31,21 @@ # include #endif -#include +// size_t and NULL +#include #ifdef HAVE_INTTYPES_H # include #endif +// C99 says that inttypes.h always includes stdint.h, but some systems +// don't do that, and require including stdint.h separately. +#ifdef HAVE_STDINT_H +# include +#endif + +// Some pre-C99 systems have SIZE_MAX in limits.h instead of stdint.h. The +// limits are also used to figure out some macros missing from pre-C99 systems. #ifdef HAVE_LIMITS_H # include #endif @@ -44,7 +53,12 @@ // Be more compatible with systems that have non-conforming inttypes.h. // We assume that int is 32-bit and that long is either 32-bit or 64-bit. // Full Autoconf test could be more correct, but this should work well enough. +// Note that this duplicates some code from lzma.h, but this is better since +// we can work without inttypes.h thanks to Autoconf tests. #ifndef UINT32_C +# if UINT_MAX != 4294967295U +# error UINT32_C is not defined and unsiged int is not 32-bit. +# endif # define UINT32_C(n) n ## U #endif #ifndef UINT32_MAX @@ -56,7 +70,8 @@ #ifndef PRIX32 # define PRIX32 "X" #endif -#if SIZEOF_UNSIGNED_LONG == 4 + +#if ULONG_MAX == 4294967295UL # ifndef UINT64_C # define UINT64_C(n) n ## ULL # endif @@ -80,16 +95,33 @@ #ifndef UINT64_MAX # define UINT64_MAX UINT64_C(18446744073709551615) #endif + +// The code currently assumes that size_t is either 32-bit or 64-bit. #ifndef SIZE_MAX # if SIZEOF_SIZE_T == 4 # define SIZE_MAX UINT32_MAX -# else +# elif SIZEOF_SIZE_T == 8 # define SIZE_MAX UINT64_MAX +# else +# error sizeof(size_t) is not 32-bit or 64-bit # endif #endif +#if SIZE_MAX != UINT32_MAX && SIZE_MAX != UINT64_MAX +# error sizeof(size_t) is not 32-bit or 64-bit +#endif #include +// Pre-C99 systems lack stdbool.h. All the code in LZMA Utils must be written +// so that it works with fake bool type, for example: +// +// bool foo = (flags & 0x100) != 0; +// bool bar = !!(flags & 0x100); +// +// This works with the real C99 bool but breaks with fake bool: +// +// bool baz = (flags & 0x100); +// #ifdef HAVE_STDBOOL_H # include #else @@ -108,11 +140,13 @@ typedef unsigned char _Bool; # ifdef NDEBUG # define assert(x) # else - // TODO: Pretty bad assert() macro. + // TODO: Pretty bad assert macro. # define assert(x) (!(x) && abort()) # endif #endif +// string.h should be enough but let's include strings.h and memory.h too if +// they exists, since that shouldn't do any harm, but may improve portability. #ifdef HAVE_STRING_H # include #endif diff --git a/src/liblzma/Makefile.am b/src/liblzma/Makefile.am index 78a072f4..a234bfd5 100644 --- a/src/liblzma/Makefile.am +++ b/src/liblzma/Makefile.am @@ -22,11 +22,15 @@ liblzma_la_LIBADD = \ common/libcommon.la \ check/libcheck.la +if COND_FILTER_LZ +SUBDIRS += lz +liblzma_la_LIBADD += lz/liblz.la +endif + if COND_FILTER_LZMA -SUBDIRS += lz lzma rangecoder +SUBDIRS += lzma rangecoder liblzma_la_LIBADD += \ - lz/liblz.la \ - lzma/liblzma4.la \ + lzma/liblzma2.la \ rangecoder/librangecoder.la endif @@ -35,7 +39,12 @@ SUBDIRS += subblock liblzma_la_LIBADD += subblock/libsubblock.la endif -if COND_MAIN_SIMPLE +if COND_FILTER_DELTA +SUBDIRS += delta +liblzma_la_LIBADD += delta/libdelta.la +endif + +if COND_FILTER_SIMPLE SUBDIRS += simple liblzma_la_LIBADD += simple/libsimple.la endif diff --git a/src/liblzma/api/Makefile.am b/src/liblzma/api/Makefile.am index 194f85db..86ce5bda 100644 --- a/src/liblzma/api/Makefile.am +++ b/src/liblzma/api/Makefile.am @@ -15,22 +15,18 @@ nobase_include_HEADERS = \ lzma.h \ lzma/alignment.h \ - lzma/alone.h \ - lzma/auto.h \ lzma/base.h \ lzma/block.h \ lzma/check.h \ + lzma/container.h \ lzma/delta.h \ - lzma/easy.h \ lzma/filter.h \ lzma/index.h \ lzma/index_hash.h \ lzma/init.h \ lzma/lzma.h \ lzma/memlimit.h \ - lzma/raw.h \ lzma/simple.h \ - lzma/stream.h \ lzma/stream_flags.h \ lzma/subblock.h \ lzma/version.h \ diff --git a/src/liblzma/api/lzma.h b/src/liblzma/api/lzma.h index 9dec904f..0f109eb3 100644 --- a/src/liblzma/api/lzma.h +++ b/src/liblzma/api/lzma.h @@ -17,37 +17,104 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * - * Before #including this file, you must make the following types available: - * - size_t - * - uint8_t - * - int32_t - * - uint32_t - * - int64_t - * - uint64_t - * - * Before #including this file, you must make the following macros available: - * - UINT32_C(n) - * - UINT64_C(n) - * - UINT32_MAX - * - UINT64_MAX - * - * Easiest way to achieve the above is to #include sys/types.h and inttypes.h - * before #including lzma.h. However, some pre-C99 libc headers don't provide - * all the required types in inttypes.h (that file may even be missing). - * Portable applications need to provide these types themselves. This way - * liblzma API can use the standard types instead of defining its own - * (e.g. lzma_uint32). - * - * Note that the API still has lzma_bool, because using stdbool.h would - * break C89 and C++ programs on many systems. */ #ifndef LZMA_H #define LZMA_H +/***************************** + * Required standard headers * + *****************************/ + +/** + * liblzma API headers need some standard types and macros. To allow + * including lzma.h without requiring the application to include other + * headers first, lzma.h includes the required standard headers unless + * they already seem to be included. + * + * Here's what types and macros are needed and from which headers: + * - stddef.h: size_t, NULL + * - stdint.h: uint8_t, uint32_t, uint64_t, UINT32_C(n), uint64_C(n), + * UINT32_MAX, UINT64_MAX + * + * However, inttypes.h is a little more portable than stdint.h, although + * inttypes.h declares some unneeded things compared to plain stdint.h. + * + * The hacks below aren't perfect, specifically they assume that inttypes.h + * exists and that it typedefs at least uint8_t, uint32_t, and uint64_t, + * and that unsigned int is 32-bit. If your application already takes care + * of setting up all the types properly (for example by using gnulib's + * stdint.h or inttypes.h), feel free to define LZMA_MANUAL_HEADERS before + * including lzma.h. + * + * Some could argue that liblzma API should provide all the required types, + * for example lzma_uint64, LZMA_UINT64_C(n), and LZMA_UINT64_MAX. This was + * seen unnecessary mess, since most systems already provide all the necessary + * types and macros in the standard headers. + * + * Note that liblzma API still has lzma_bool, because using stdbool.h would + * break C89 and C++ programs on many systems. + */ + +/* stddef.h even in C++ so that we get size_t in global namespace. */ +#include + +#if !defined(UINT32_C) || !defined(UINT64_C) \ + || !defined(UINT32_MAX) || !defined(UINT64_MAX) +# ifdef __cplusplus + /* + * C99 sections 7.18.2 and 7.18.4 specify that in C++ + * implementations define the limit and constant macros only + * if specifically requested. Note that if you want the + * format macros too, you need to define __STDC_FORMAT_MACROS + * before including lzma.h, since re-including inttypes.h + * with __STDC_FORMAT_MACROS defined doesn't necessarily work. + */ +# ifndef __STDC_LIMIT_MACROS +# define __STDC_LIMIT_MACROS 1 +# endif +# ifndef __STDC_CONSTANT_MACROS +# define __STDC_CONSTANT_MACROS 1 +# endif +# endif + +# include + + /* + * Some old systems have only the typedefs in inttypes.h, and lack + * all the macros. For those systems, we need a few more hacks. + * We assume that unsigned int is 32-bit and unsigned long is either + * 32-bit or 64-bit. If these hacks aren't enough, the application + * has to use setup the types manually before including lzma.h. + */ +# ifndef UINT32_C +# define UINT32_C(n) n # U +# endif + +# ifndef UINT64_C + /* Get ULONG_MAX. */ +# ifndef __cplusplus +# include +# else +# include +# endif +# if ULONG_MAX == 4294967295UL +# define UINT64_C(n) n ## ULL +# else +# define UINT64_C(n) n ## UL +# endif +# endif + +# ifndef UINT32_MAX +# define UINT32_MAX (UINT32_C(4294967295)) +# endif + +# ifndef UINT64_MAX +# define UINT64_MAX (UINT64_C(18446744073709551615)) +# endif +#endif + + /****************** * GCC extensions * ******************/ @@ -57,20 +124,50 @@ * break anything if these are sometimes enabled and sometimes not, only * affects warnings and optimizations. */ -#if defined(__GNUC__) && __GNUC__ >= 3 +#if __GNUC__ >= 3 # ifndef lzma_attribute # define lzma_attribute(attr) __attribute__(attr) # endif + # ifndef lzma_restrict # define lzma_restrict __restrict__ # endif + + /* warn_unused_result was added in GCC 3.4. */ +# ifndef lzma_attr_warn_unused_result +# if __GNUC__ == 3 && __GNUC_MINOR__ < 4 +# define lzma_attr_warn_unused_result +# endif +# endif + #else # ifndef lzma_attribute # define lzma_attribute(attr) # endif + # ifndef lzma_restrict -# define lzma_restrict +# if __STDC_VERSION__ >= 199901L +# define lzma_restrict restrict +# else +# define lzma_restrict +# endif # endif + +# define lzma_attr_warn_unused_result +#endif + + +#ifndef lzma_attr_pure +# define lzma_attr_pure lzma_attribute((__pure__)) +#endif + +#ifndef lzma_attr_const +# define lzma_attr_const lzma_attribute((__const__)) +#endif + +#ifndef lzma_attr_warn_unused_result +# define lzma_attr_warn_unused_result \ + lzma_attribute((__warn_unused_result__)) #endif @@ -89,36 +186,30 @@ extern "C" { #define LZMA_H_INTERNAL 1 /* Basic features */ +#include "lzma/version.h" #include "lzma/init.h" #include "lzma/base.h" #include "lzma/vli.h" -#include "lzma/filter.h" #include "lzma/check.h" /* Filters */ +#include "lzma/filter.h" #include "lzma/subblock.h" #include "lzma/simple.h" #include "lzma/delta.h" #include "lzma/lzma.h" /* Container formats */ -#include "lzma/block.h" -#include "lzma/stream.h" -#include "lzma/alone.h" -#include "lzma/raw.h" -#include "lzma/auto.h" -#include "lzma/easy.h" +#include "lzma/container.h" /* Advanced features */ +#include "lzma/alignment.h" /* FIXME */ +#include "lzma/block.h" #include "lzma/index.h" #include "lzma/index_hash.h" -#include "lzma/alignment.h" #include "lzma/stream_flags.h" #include "lzma/memlimit.h" -/* Version number */ -#include "lzma/version.h" - /* * All subheaders included. Undefine LZMA_H_INTERNAL to prevent applications * re-including the subheaders. diff --git a/src/liblzma/api/lzma/alignment.h b/src/liblzma/api/lzma/alignment.h index 6672656c..008af690 100644 --- a/src/liblzma/api/lzma/alignment.h +++ b/src/liblzma/api/lzma/alignment.h @@ -27,7 +27,7 @@ * FIXME desc */ extern uint32_t lzma_alignment_input( - const lzma_options_filter *filters, uint32_t guess); + const lzma_filter *filters, uint32_t guess); /** @@ -36,7 +36,7 @@ extern uint32_t lzma_alignment_input( * Knowing the alignment of the output data is useful e.g. in the Block * encoder which tries to align the Compressed Data field optimally. * - * \param filters Pointer to lzma_options_filter array, whose last + * \param filters Pointer to lzma_filter array, whose last * member must have .id = LZMA_VLI_VALUE_UNKNOWN. * \param guess The value to return if the alignment of the output * is the same as the alignment of the input data. @@ -57,4 +57,4 @@ extern uint32_t lzma_alignment_input( * options), UINT32_MAX is returned. */ extern uint32_t lzma_alignment_output( - const lzma_options_filter *filters, uint32_t guess); + const lzma_filter *filters, uint32_t guess); diff --git a/src/liblzma/api/lzma/alone.h b/src/liblzma/api/lzma/alone.h deleted file mode 100644 index 72299773..00000000 --- a/src/liblzma/api/lzma/alone.h +++ /dev/null @@ -1,52 +0,0 @@ -/** - * \file lzma/alone.h - * \brief Handling of the legacy LZMA_Alone format - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use instead. -#endif - - -/** - * \brief Initializes LZMA_Alone encoder - * - * LZMA_Alone files have the suffix .lzma like the .lzma Stream files. - * LZMA_Alone format supports only one filter, the LZMA filter. There is - * no support for integrity checks like CRC32. - * - * Use this format if and only if you need to create files readable by - * legacy LZMA tools. - * - * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_alone_encoder( - lzma_stream *strm, const lzma_options_lzma *options); - - -/** - * \brief Initializes decoder for LZMA_Alone file - * - * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - */ -extern lzma_ret lzma_alone_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/auto.h b/src/liblzma/api/lzma/auto.h deleted file mode 100644 index fd5bf7d2..00000000 --- a/src/liblzma/api/lzma/auto.h +++ /dev/null @@ -1,36 +0,0 @@ -/** - * \file lzma/auto.h - * \brief Decoder with automatic file format detection - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use instead. -#endif - - -/** - * \brief Decode .lzma Streams and LZMA_Alone files with autodetection - * - * Autodetects between the .lzma Stream and LZMA_Alone formats, and - * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once - * the type of the file has been detected. - * - * \param strm Pointer to propertily prepared lzma_stream - * - * \return - LZMA_OK: Initialization was successful. - * - LZMA_MEM_ERROR: Cannot allocate memory. - */ -extern lzma_ret lzma_auto_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h index b0dfed95..cb614176 100644 --- a/src/liblzma/api/lzma/base.h +++ b/src/liblzma/api/lzma/base.h @@ -134,7 +134,7 @@ typedef enum { * \brief Unknown file format */ - LZMA_MEMLIMIT_ERROR = -9 + LZMA_MEMLIMIT_ERROR = -9, /** * \brief Memory usage limit was reached * @@ -143,6 +143,9 @@ typedef enum { * the memory usage limit has to be increased. See functions * lzma_memlimit_get() and lzma_memlimit_set(). */ + + LZMA_NO_CHECK = -10, + LZMA_SEE_CHECK = -11 } lzma_ret; @@ -229,11 +232,6 @@ typedef struct { /** * \brief Pointer to custom memory allocation function * - * Set this to point to your custom memory allocation function. - * It can be useful for example if you want to limit how much - * memory liblzma is allowed to use: for this, you may use - * a pointer to lzma_memory_alloc(). - * * If you don't want a custom allocator, but still want * custom free(), set this to NULL and liblzma will use * the standard malloc(). @@ -250,16 +248,19 @@ typedef struct { * size nmemb * size, or NULL if allocation fails * for some reason. When allocation fails, functions * of liblzma return LZMA_MEM_ERROR. + * + * For performance reasons, the allocator should not waste time + * zeroing the allocated buffers. This is not only about speed, but + * also memory usage, since the operating system kernel doesn't + * necessarily allocate the requested memory until it is actually + * used. With small input files liblzma may actually need only a + * fraction of the memory that it requested for allocation. */ void *(*alloc)(void *opaque, size_t nmemb, size_t size); /** * \brief Pointer to custom memory freeing function * - * Set this to point to your custom memory freeing function. - * If lzma_memory_alloc() is used as allocator, this should - * be set to lzma_memory_free(). - * * If you don't want a custom freeing function, but still * want a custom allocator, set this to NULL and liblzma * will use the standard free(). @@ -279,10 +280,6 @@ typedef struct { * and lzma_allocator.free(). This intended to ease implementing * custom memory allocation functions for use with liblzma. * - * When using lzma_memory_alloc() and lzma_memory_free(), opaque - * must point to lzma_memory_limiter structure allocated and - * initialized with lzma_memory_limiter_create(). - * * If you don't need this, you should set it to NULL. */ void *opaque; @@ -347,6 +344,17 @@ typedef struct { /** Internal state is not visible to outsiders. */ lzma_internal *internal; + /** + * Reserved space to allow possible future extensions without + * breaking the ABI. Excluding the initialization of this structure, + * you should not touch these, because the names of these variables + * may change. + */ + void *reserved_ptr1; + void *reserved_ptr2; + uint64_t reserved_int1; + uint64_t reserved_int2; + } lzma_stream; @@ -358,22 +366,18 @@ typedef struct { * has been allocated yet: * * lzma_stream strm = LZMA_STREAM_INIT; - */ -#define LZMA_STREAM_INIT { NULL, 0, 0, NULL, 0, 0, NULL, NULL } - - -/** - * \brief Initialization for lzma_stream * - * This is like LZMA_STREAM_INIT, but this can be used when the lzma_stream - * has already been allocated: + * If you need to initialize a dynamically allocatedlzma_stream, you can use + * memset(strm_pointer, 0, sizeof(lzma_stream)). Strictly speaking, this + * violates the C standard since NULL may have different internal + * representation than zero, but it should be portable enough in practice. + * Anyway, for maximum portability, you could use this: * - * lzma_stream *strm = malloc(sizeof(lzma_stream)); - * if (strm == NULL) - * return LZMA_MEM_ERROR; - * *strm = LZMA_STREAM_INIT_VAR; + * lzma_stream tmp = LZMA_STREAM_INIT; + * *strm = tmp; */ -extern const lzma_stream LZMA_STREAM_INIT_VAR; +#define LZMA_STREAM_INIT \ + { NULL, 0, 0, NULL, 0, 0, NULL, NULL, NULL, NULL, 0, 0 } /** @@ -409,7 +413,8 @@ extern const lzma_stream LZMA_STREAM_INIT_VAR; * - LZMA_PROG_ERROR: Invalid arguments or the internal state * of the coder is corrupt. */ -extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action); +extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action) + lzma_attr_warn_unused_result; /** diff --git a/src/liblzma/api/lzma/block.h b/src/liblzma/api/lzma/block.h index a8941165..45045815 100644 --- a/src/liblzma/api/lzma/block.h +++ b/src/liblzma/api/lzma/block.h @@ -36,12 +36,13 @@ typedef struct { * \brief Size of the Block Header * * Read by: + * - lzma_block_header_encode() + * - lzma_block_header_decode() * - lzma_block_encoder() * - lzma_block_decoder() * * Written by: * - lzma_block_header_size() - * - lzma_block_header_decode() */ uint32_t header_size; # define LZMA_BLOCK_HEADER_SIZE_MIN 8 @@ -54,10 +55,12 @@ typedef struct { * Header, thus its value must be provided also when decoding. * * Read by: + * - lzma_block_header_encode() + * - lzma_block_header_decode() * - lzma_block_encoder() * - lzma_block_decoder() */ - lzma_check_type check; + lzma_check check; /** * \brief Size of the Compressed Data in bytes @@ -134,17 +137,17 @@ typedef struct { * have LZMA_BLOCK_FILTERS_MAX + 1 members or the Block * Header decoder will overflow the buffer. */ - lzma_options_filter *filters; + lzma_filter *filters; # define LZMA_BLOCK_FILTERS_MAX 4 -} lzma_options_block; +} lzma_block; /** * \brief Decodes the Block Header Size field * * To decode Block Header using lzma_block_header_decode(), the size of the - * Block Header has to be known and stored into lzma_options_block.header_size. + * Block Header has to be known and stored into lzma_block.header_size. * The size can be calculated from the first byte of a Block using this macro. * Note that if the first byte is 0x00, it indicates beginning of Index; use * this macro only when the byte is not 0x00. @@ -164,7 +167,8 @@ typedef struct { * may return LZMA_OK even if lzma_block_header_encode() or * lzma_block_encoder() would fail. */ -extern lzma_ret lzma_block_header_size(lzma_options_block *options); +extern lzma_ret lzma_block_header_size(lzma_block *options) + lzma_attr_warn_unused_result; /** @@ -183,7 +187,8 @@ extern lzma_ret lzma_block_header_size(lzma_options_block *options); * - LZMA_PROG_ERROR */ extern lzma_ret lzma_block_header_encode( - const lzma_options_block *options, uint8_t *out); + const lzma_block *options, uint8_t *out) + lzma_attr_warn_unused_result; /** @@ -203,8 +208,9 @@ extern lzma_ret lzma_block_header_encode( * - LZMA_HEADER_ERROR: Invalid or unsupported options. * - LZMA_PROG_ERROR */ -extern lzma_ret lzma_block_header_decode(lzma_options_block *options, - lzma_allocator *allocator, const uint8_t *in); +extern lzma_ret lzma_block_header_decode(lzma_block *options, + lzma_allocator *allocator, const uint8_t *in) + lzma_attr_warn_unused_result; /** @@ -227,7 +233,8 @@ extern lzma_ret lzma_block_header_decode(lzma_options_block *options, * options->header_size between 8 and 1024 inclusive. */ extern lzma_ret lzma_block_total_size_set( - lzma_options_block *options, lzma_vli total_size); + lzma_block *options, lzma_vli total_size) + lzma_attr_warn_unused_result; /** @@ -238,7 +245,8 @@ extern lzma_ret lzma_block_total_size_set( * * \return Total Size on success, or zero on error. */ -extern lzma_vli lzma_block_total_size_get(const lzma_options_block *options); +extern lzma_vli lzma_block_total_size_get(const lzma_block *options) + lzma_attr_pure; /** @@ -259,8 +267,8 @@ extern lzma_vli lzma_block_total_size_get(const lzma_options_block *options); * * lzma_code() can return FIXME */ -extern lzma_ret lzma_block_encoder( - lzma_stream *strm, lzma_options_block *options); +extern lzma_ret lzma_block_encoder(lzma_stream *strm, lzma_block *options) + lzma_attr_warn_unused_result; /** @@ -273,5 +281,5 @@ extern lzma_ret lzma_block_encoder( * - LZMA_PROG_ERROR * - LZMA_MEM_ERROR */ -extern lzma_ret lzma_block_decoder( - lzma_stream *strm, lzma_options_block *options); +extern lzma_ret lzma_block_decoder(lzma_stream *strm, lzma_block *options) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/check.h b/src/liblzma/api/lzma/check.h index dcba8269..18394a86 100644 --- a/src/liblzma/api/lzma/check.h +++ b/src/liblzma/api/lzma/check.h @@ -56,7 +56,7 @@ typedef enum { * * Size of the Check field: 32 bytes */ -} lzma_check_type; +} lzma_check; /** @@ -73,34 +73,37 @@ typedef enum { #define LZMA_CHECK_ID_MAX 15 -/** - * \brief Check IDs supported by this liblzma build - * - * If lzma_available_checks[n] is true, the Check ID n is supported by this - * liblzma build. You can assume that LZMA_CHECK_NONE and LZMA_CHECK_CRC32 - * are always available. - */ -extern const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1]; - - -/** - * \brief Size of the Check field with different Check IDs - * - * Although not all Check IDs have a check algorithm associated, the size of - * every Check is already frozen. This array contains the size (in bytes) of - * the Check field with specified Check ID. The values are taken from the - * section 2.1.1.2 of the .lzma file format specification: - * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 } - */ -extern const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1]; - - /** * \brief Maximum size of a Check field */ #define LZMA_CHECK_SIZE_MAX 64 +/** + * \brief Test if the given Check ID is supported + * + * Returns true if the given Check ID is supported by this liblzma build. + * Otherwise false is returned. It is safe to call this with a value that + * is not in the range [0, 15]; in that case the return value is always false. + */ +extern lzma_bool lzma_check_is_supported(lzma_check check) + lzma_attr_const; + + +/** + * \brief Get the size of the Check field with given Check ID + * + * Although not all Check IDs have a check algorithm associated, the size of + * every Check is already frozen. This function returns the size (in bytes) of + * the Check field with the specified Check ID. The values are taken from the + * section 2.1.1.2 of the .lzma file format specification: + * { 0, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 32, 64, 64, 64 } + * + * If the argument is not in the range [0, 15], UINT32_MAX is returned. + */ +extern uint32_t lzma_check_size(lzma_check check) lzma_attr_const; + + /** * \brief Calculate CRC32 * @@ -115,7 +118,8 @@ extern const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1]; * \return Updated CRC value, which can be passed to this function * again to continue CRC calculation. */ -extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); +extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) + lzma_attr_pure; /** @@ -125,7 +129,8 @@ extern uint32_t lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc); * * This function is used similarly to lzma_crc32(). See its documentation. */ -extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc); +extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) + lzma_attr_pure; /* diff --git a/src/liblzma/api/lzma/container.h b/src/liblzma/api/lzma/container.h new file mode 100644 index 00000000..27014856 --- /dev/null +++ b/src/liblzma/api/lzma/container.h @@ -0,0 +1,252 @@ +/** + * \file lzma/FIXME.h + * \brief File formats + * + * \author Copyright (C) 1999-2008 Igor Pavlov + * \author Copyright (C) 2007-2008 Lasse Collin + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + */ + +#ifndef LZMA_H_INTERNAL +# error Never include this file directly. Use instead. +#endif + + +/************ + * Encoding * + ************/ + +/** + * \brief Compression level names for lzma_easy_* functions + * + * At the moment, all the compression levels support LZMA_SYNC_FLUSH. + * In future there may be levels that don't support LZMA_SYNC_FLUSH. + * However, the LZMA_SYNC_FLUSH support won't be removed from the + * existing compression levels. + * + * \note If liblzma is built without encoder support, or with some + * filters disabled, some of the compression levels may be + * unsupported. In that case, the initialization functions + * will return LZMA_HEADER_ERROR. + */ +typedef enum { + LZMA_EASY_COPY = 0, + /**< + * No compression; the data is just wrapped into .lzma + * container. + */ + + LZMA_EASY_LZMA2_1 = 1, + /**< + * LZMA2 filter with fast compression (fast in terms of LZMA2). + * If you are interested in the exact options used, see + * lzma_preset_lzma[0]. Note that the exact options may + * change between liblzma versions. + * + * At the moment, the command line tool uses these settings + * when `lzma -1' is used. In future, the command line tool + * may default to some more complex way to determine the + * settings used e.g. the type of files being compressed. + * + * LZMA_EASY_LZMA_2 is equivalent to lzma_preset_lzma[1] + * and so on. + */ + + LZMA_EASY_LZMA_2 = 2, + LZMA_EASY_LZMA_3 = 3, + LZMA_EASY_LZMA_4 = 4, + LZMA_EASY_LZMA_5 = 5, + LZMA_EASY_LZMA_6 = 6, + LZMA_EASY_LZMA_7 = 7, + LZMA_EASY_LZMA_8 = 8, + LZMA_EASY_LZMA_9 = 9, +} lzma_easy_level; + + +/** + * \brief Default compression level + * + * Data Blocks contain the actual compressed data. It's not straightforward + * to recommend a default level, because in some cases keeping the resource + * usage relatively low is more important that getting the maximum + * compression ratio. + */ +#define LZMA_EASY_DEFAULT LZMA_EASY_LZMA2_7 + + +/** + * \brief Calculates rough memory requirements of a compression level + * + * This function is a wrapper for lzma_memory_usage(), which is declared + * in filter.h. + * + * \return Approximate memory usage of the encoder with the given + * compression level in mebibytes (value * 1024 * 1024 bytes). + * On error (e.g. compression level is not supported), + * UINT32_MAX is returned. + */ +extern uint32_t lzma_easy_memory_usage(lzma_easy_level level) + lzma_attr_pure; + + +/** + * \brief Initializes .lzma Stream encoder + * + * This function is intended for those who just want to use the basic features + * if liblzma (that is, most developers out there). Lots of assumptions are + * made, which are correct or at least good enough for most situations. + * + * \param strm Pointer to lzma_stream that is at least initialized + * with LZMA_STREAM_INIT. + * \param level Compression level to use. This selects a set of + * compression settings from a list of compression + * presets. + * + * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to + * encode your data. + * - LZMA_MEM_ERROR: Memory allocation failed. All memory + * previously allocated for *strm is now freed. + * - LZMA_HEADER_ERROR: The given compression level is not + * supported by this build of liblzma. + * + * If initialization succeeds, use lzma_code() to do the actual encoding. + * Valid values for `action' (the second argument of lzma_code()) are + * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, + * there may be compression levels that don't support LZMA_SYNC_FLUSH. + */ +extern lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes .lzma Stream encoder + * + * \param strm Pointer to properly prepared lzma_stream + * \param filters Array of filters. This must be terminated with + * filters[n].id = LZMA_VLI_VALUE_UNKNOWN. There must + * be 1-4 filters, but there are restrictions on how + * multiple filters can be combined. FIXME Tell where + * to find more information. + * \param check Type of the integrity check to calculate from + * uncompressed data. + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_stream_encoder(lzma_stream *strm, + const lzma_filter *filters, lzma_check check) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes LZMA_Alone (deprecated file format) encoder + * + * LZMA_Alone files have the suffix .lzma like the .lzma Stream files. + * LZMA_Alone format supports only one filter, the LZMA filter. There is + * no support for integrity checks like CRC32. + * + * Use this format if and only if you need to create files readable by + * legacy LZMA tools such as LZMA Utils 4.32.x. + * + * LZMA_Alone encoder doesn't support LZMA_SYNC_FLUSH or LZMA_FULL_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_alone_encoder( + lzma_stream *strm, const lzma_options_lzma *options) + lzma_attr_warn_unused_result; + + +/************ + * Decoding * + ************/ + +/** + * This flag makes lzma_code() return LZMA_NO_CHECK if the input stream + * being decoded has no integrity check. Note that when used with + * lzma_auto_decoder(), all LZMA_Alone files will cause trigger LZMA_NO_CHECK + * if LZMA_WARN_NO_CHECK is used. + */ +#define LZMA_WARN_NO_CHECK UINT32_C(0x01) + + +/** + * This flag makes lzma_code() return LZMA_UNSUPPORTED_CHECK if the input + * stream has an integrity check, but the type of the integrity check is not + * supported by this liblzma version or build. Such files can still be + * decoded, but the integrity check cannot be verified. + */ +#define LZMA_WARN_UNSUPPORTED_CHECK UINT32_C(0x02) + + +/** + * This flag makes lzma_code() return LZMA_READ_CHECK as soon as the type + * of the integrity check is known. The type can then be read with + * lzma_check_get(). + */ +#define LZMA_TELL_CHECK UINT32_C(0x04) + + +/** + * This flag makes lzma_code() decode concatenated .lzma files. + * FIXME Explain the changed API. + */ +#define LZMA_CONCATENATED UINT32_C(0x08) + + +/** + * \brief Initializes decoder for .lzma Stream + * + * \param strm Pointer to propertily prepared lzma_stream + * \param memlimit Rough memory usage limit as bytes + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + */ +extern lzma_ret lzma_stream_decoder( + lzma_stream *strm, uint64_t memlimit, uint32_t flags) + lzma_attr_warn_unused_result; + + +/** + * \brief Decode .lzma Streams and LZMA_Alone files with autodetection + * + * Autodetects between the .lzma Stream and LZMA_Alone formats, and + * calls lzma_stream_decoder_init() or lzma_alone_decoder_init() once + * the type of the file has been detected. + * + * \param strm Pointer to propertily prepared lzma_stream + * \param memlimit Rough memory usage limit as bytes + * \param flags Bitwise-or of flags, or zero for no flags. + * + * \return - LZMA_OK: Initialization was successful. + * - LZMA_MEM_ERROR: Cannot allocate memory. + */ +extern lzma_ret lzma_auto_decoder( + lzma_stream *strm, uint64_t memlimit, uint32_t flags) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes decoder for LZMA_Alone file + * + * The LZMA_Alone decoder supports LZMA_SYNC_FLUSH. FIXME + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + */ +extern lzma_ret lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/delta.h b/src/liblzma/api/lzma/delta.h index 58afec18..740de97c 100644 --- a/src/liblzma/api/lzma/delta.h +++ b/src/liblzma/api/lzma/delta.h @@ -24,9 +24,21 @@ /** * \brief Filter ID * - * Filter ID of the Delta filter. This is used as lzma_options_filter.id. + * Filter ID of the Delta filter. This is used as lzma_filter.id. */ -#define LZMA_FILTER_DELTA LZMA_VLI_C(0x20) +#define LZMA_FILTER_DELTA LZMA_VLI_C(0x03) + + +/** + * \brief Type of the delta calculation + * + * Currently only byte-wise delta is supported. Other possible types could + * be, for example, delta of 16/32/64-bit little/big endian integers, but + * these are not currently planned since byte-wise delta is almost as good. + */ +typedef enum { + LZMA_DELTA_TYPE_BYTE +} lzma_delta_type; /** @@ -35,8 +47,14 @@ * These options are needed by both encoder and decoder. */ typedef struct { + /** For now, this must always be LZMA_DELTA_TYPE_BYTE. */ + lzma_delta_type type; + /** - * \brief Delta distance as bytes + * \brief Delta distance + * + * With the only currently supported type, LZMA_DELTA_TYPE_BYTE, + * the distance is as bytes. * * Examples: * - 16-bit stereo audio: distance = 4 bytes @@ -46,4 +64,16 @@ typedef struct { # define LZMA_DELTA_DISTANCE_MIN 1 # define LZMA_DELTA_DISTANCE_MAX 256 + /** + * \brief Reserved space for possible future extensions + * + * You should not touch these, because the names of these variables + * may change. These are and will never be used when type is + * LZMA_DELTA_TYPE_BYTE, so it is safe to leave these uninitialized. + */ + uint32_t reserved_int1; + uint32_t reserved_int2; + void *reserved_ptr1; + void *reserved_ptr2; + } lzma_options_delta; diff --git a/src/liblzma/api/lzma/easy.h b/src/liblzma/api/lzma/easy.h deleted file mode 100644 index d83a79a2..00000000 --- a/src/liblzma/api/lzma/easy.h +++ /dev/null @@ -1,121 +0,0 @@ -/** - * \file lzma/easy.h - * \brief Easy to use encoder initialization - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2008 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use instead. -#endif - - -/** - * \brief Compression level names for lzma_easy_* functions - * - * At the moment, all the compression levels support LZMA_SYNC_FLUSH. - * In future there may be levels that don't support LZMA_SYNC_FLUSH. - * However, the LZMA_SYNC_FLUSH support won't be removed from the - * existing compression levels. - * - * \note If liblzma is built without encoder support, or with some - * filters disabled, some of the compression levels may be - * unsupported. In that case, the initialization functions - * will return LZMA_HEADER_ERROR. - */ -typedef enum { - LZMA_EASY_COPY, - /**< - * No compression; the data is just wrapped into .lzma - * container. - */ - - LZMA_EASY_LZMA_1, - /**< - * LZMA filter with fast compression (fast in terms of LZMA). - * If you are interested in the exact options used, see - * lzma_preset_lzma[0]. Note that the exact options may - * change between liblzma versions. - * - * At the moment, the command line tool uses these settings - * when `lzma -1' is used. In future, the command line tool - * may default to some more complex way to determine the - * settings used e.g. the type of files being compressed. - * - * LZMA_EASY_LZMA_2 is equivalent to lzma_preset_lzma[1] - * and so on. - */ - - LZMA_EASY_LZMA_2, - LZMA_EASY_LZMA_3, - LZMA_EASY_LZMA_4, - LZMA_EASY_LZMA_5, - LZMA_EASY_LZMA_6, - LZMA_EASY_LZMA_7, - LZMA_EASY_LZMA_8, - LZMA_EASY_LZMA_9, -} lzma_easy_level; - - -/** - * \brief Default compression level - * - * Data Blocks contain the actual compressed data. It's not straightforward - * to recommend a default level, because in some cases keeping the resource - * usage relatively low is more important that getting the maximum - * compression ratio. - */ -#define LZMA_EASY_DEFAULT LZMA_EASY_LZMA_7 - - -/** - * \brief Calculates rough memory requirements of a compression level - * - * This function is a wrapper for lzma_memory_usage(), which is declared - * in lzma/filter.h. - * - * \return Approximate memory usage of the encoder with the given - * compression level in mebibytes (value * 1024 * 1024 bytes). - * On error (e.g. compression level is not supported), - * UINT32_MAX is returned. - */ -extern uint32_t lzma_easy_memory_usage(lzma_easy_level level); - - -/** - * \brief Initializes .lzma Stream encoder - * - * This function is intended for those who just want to use the basic LZMA - * features (that is, most developers out there). Lots of assumptions are - * made, which are correct or at least good enough for most situations. - * - * \param strm Pointer to lzma_stream that is at least initialized - * with LZMA_STREAM_INIT. - * \param level Compression level to use. This selects a set of - * compression settings from a list of compression - * presets. - * - * \return - LZMA_OK: Initialization succeeded. Use lzma_code() to - * encode your data. - * - LZMA_MEM_ERROR: Memory allocation failed. All memory - * previously allocated for *strm is now freed. - * - LZMA_HEADER_ERROR: The given compression level is not - * supported by this build of liblzma. - * - * If initialization succeeds, use lzma_code() to do the actual encoding. - * Valid values for `action' (the second argument of lzma_code()) are - * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, - * there may be compression levels that don't support LZMA_SYNC_FLUSH. - */ -extern lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level); diff --git a/src/liblzma/api/lzma/filter.h b/src/liblzma/api/lzma/filter.h index 412c30e5..d5903f89 100644 --- a/src/liblzma/api/lzma/filter.h +++ b/src/liblzma/api/lzma/filter.h @@ -51,7 +51,7 @@ typedef struct { */ void *options; -} lzma_options_filter; +} lzma_filter; /** @@ -65,7 +65,7 @@ typedef struct { * encoding-specific functions are probably missing from the library * API/ABI completely. */ -extern const lzma_vli *const lzma_available_filter_encoders; +extern const lzma_vli *const lzma_filter_encoders; /** @@ -79,7 +79,7 @@ extern const lzma_vli *const lzma_available_filter_encoders; * decoding-specific functions are probably missing from the library * API/ABI completely. */ -extern const lzma_vli *const lzma_available_filter_decoders; +extern const lzma_vli *const lzma_filter_decoders; /** @@ -87,8 +87,6 @@ extern const lzma_vli *const lzma_available_filter_decoders; * * \param filters Array of filters terminated with * .id == LZMA_VLI_VALUE_UNKNOWN. - * \param is_encoder Set to true when calculating memory requirements - * of an encoder; false for decoder. * * \return Number of mebibytes (MiB i.e. 2^20) required for the given * encoder or decoder filter chain. @@ -98,8 +96,55 @@ extern const lzma_vli *const lzma_available_filter_decoders; * if calculating memory requirements of decoder, lzma_init() or * lzma_init_decoder() must have been called earlier. */ -extern uint32_t lzma_memory_usage( - const lzma_options_filter *filters, lzma_bool is_encoder); +// extern uint32_t lzma_memory_usage( +// const lzma_filter *filters, lzma_bool is_encoder); + +extern uint64_t lzma_memusage_encoder(const lzma_filter *filters) + lzma_attr_pure; + +extern uint64_t lzma_memusage_decoder(const lzma_filter *filters) + lzma_attr_pure; + + +/** + * \brief Initializes raw encoder + * + * This function may be useful when implementing custom file formats. + * + * \param strm Pointer to properly prepared lzma_stream + * \param options Array of lzma_filter structures. + * The end of the array must be marked with + * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum + * number of filters is one and the maximum is four. + * + * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the + * filter chain supports it), or LZMA_FINISH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_raw_encoder( + lzma_stream *strm, const lzma_filter *options) + lzma_attr_warn_unused_result; + + +/** + * \brief Initializes raw decoder + * + * The initialization of raw decoder goes similarly to raw encoder. + * + * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH. + * + * \return - LZMA_OK + * - LZMA_MEM_ERROR + * - LZMA_HEADER_ERROR + * - LZMA_PROG_ERROR + */ +extern lzma_ret lzma_raw_decoder( + lzma_stream *strm, const lzma_filter *options) + lzma_attr_warn_unused_result; /** @@ -119,10 +164,11 @@ extern uint32_t lzma_memory_usage( * - LZMA_PROG_ERROR: Invalid options * * \note If you need to calculate size of List of Filter Flags, - * you need to loop over every lzma_options_filter entry. + * you need to loop over every lzma_filter entry. */ extern lzma_ret lzma_filter_flags_size( - uint32_t *size, const lzma_options_filter *options); + uint32_t *size, const lzma_filter *options) + lzma_attr_warn_unused_result; /** @@ -143,8 +189,9 @@ extern lzma_ret lzma_filter_flags_size( * buffer space (you should have checked it with * lzma_filter_flags_size()). */ -extern lzma_ret lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, - size_t out_size, const lzma_options_filter *options); +extern lzma_ret lzma_filter_flags_encode(const lzma_filter *options, + uint8_t *out, size_t *out_pos, size_t out_size) + lzma_attr_warn_unused_result; /** @@ -163,5 +210,6 @@ extern lzma_ret lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, * - LZMA_PROG_ERROR */ extern lzma_ret lzma_filter_flags_decode( - lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *in, size_t *in_pos, size_t in_size); + lzma_filter *options, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/index.h b/src/liblzma/api/lzma/index.h index 13cddf47..44be10b1 100644 --- a/src/liblzma/api/lzma/index.h +++ b/src/liblzma/api/lzma/index.h @@ -66,7 +66,8 @@ typedef struct { * In this case, return value cannot be NULL or a different pointer than * the i given as argument. */ -extern lzma_index *lzma_index_init(lzma_index *i, lzma_allocator *allocator); +extern lzma_index *lzma_index_init(lzma_index *i, lzma_allocator *allocator) + lzma_attr_warn_unused_result; /** @@ -91,13 +92,14 @@ extern void lzma_index_end(lzma_index *i, lzma_allocator *allocator); * - LZMA_PROG_ERROR */ extern lzma_ret lzma_index_append(lzma_index *i, lzma_allocator *allocator, - lzma_vli total_size, lzma_vli uncompressed_size); + lzma_vli total_size, lzma_vli uncompressed_size) + lzma_attr_warn_unused_result; /** * \brief Get the number of Records */ -extern lzma_vli lzma_index_count(const lzma_index *i); +extern lzma_vli lzma_index_count(const lzma_index *i) lzma_attr_pure; /** @@ -105,7 +107,7 @@ extern lzma_vli lzma_index_count(const lzma_index *i); * * This is needed to verify the Index Size field from the Stream Footer. */ -extern lzma_vli lzma_index_size(const lzma_index *i); +extern lzma_vli lzma_index_size(const lzma_index *i) lzma_attr_pure; /** @@ -114,7 +116,7 @@ extern lzma_vli lzma_index_size(const lzma_index *i); * This doesn't include the Stream Header, Stream Footer, Stream Padding, * or Index fields. */ -extern lzma_vli lzma_index_total_size(const lzma_index *i); +extern lzma_vli lzma_index_total_size(const lzma_index *i) lzma_attr_pure; /** @@ -123,7 +125,7 @@ extern lzma_vli lzma_index_total_size(const lzma_index *i); * If multiple Indexes have been combined, this works as if the Blocks * were in a single Stream. */ -extern lzma_vli lzma_index_stream_size(const lzma_index *i); +extern lzma_vli lzma_index_stream_size(const lzma_index *i) lzma_attr_pure; /** @@ -133,19 +135,21 @@ extern lzma_vli lzma_index_stream_size(const lzma_index *i); * identical to lzma_index_stream_size(). If multiple Indexes have been * combined, this includes also the possible Stream Padding fields. */ -extern lzma_vli lzma_index_file_size(const lzma_index *i); +extern lzma_vli lzma_index_file_size(const lzma_index *i) lzma_attr_pure; /** * \brief Get the uncompressed size of the Stream */ -extern lzma_vli lzma_index_uncompressed_size(const lzma_index *i); +extern lzma_vli lzma_index_uncompressed_size(const lzma_index *i) + lzma_attr_pure; /** * \brief Get the next Record from the Index */ -extern lzma_bool lzma_index_read(lzma_index *i, lzma_index_record *record); +extern lzma_bool lzma_index_read(lzma_index *i, lzma_index_record *record) + lzma_attr_warn_unused_result; /** @@ -179,7 +183,8 @@ extern void lzma_index_rewind(lzma_index *i); * and the read position are not modified, and this function returns true. */ extern lzma_bool lzma_index_locate( - lzma_index *i, lzma_index_record *record, lzma_vli target); + lzma_index *i, lzma_index_record *record, lzma_vli target) + lzma_attr_warn_unused_result; /** @@ -202,7 +207,8 @@ extern lzma_bool lzma_index_locate( */ extern lzma_ret lzma_index_cat(lzma_index *lzma_restrict dest, lzma_index *lzma_restrict src, - lzma_allocator *allocator, lzma_vli padding); + lzma_allocator *allocator, lzma_vli padding) + lzma_attr_warn_unused_result; /** @@ -211,22 +217,26 @@ extern lzma_ret lzma_index_cat(lzma_index *lzma_restrict dest, * \return A copy of the Index, or NULL if memory allocation failed. */ extern lzma_index *lzma_index_dup( - const lzma_index *i, lzma_allocator *allocator); + const lzma_index *i, lzma_allocator *allocator) + lzma_attr_warn_unused_result; /** * \brief Compares if two Index lists are identical */ -extern lzma_bool lzma_index_equal(const lzma_index *a, const lzma_index *b); +extern lzma_bool lzma_index_equal(const lzma_index *a, const lzma_index *b) + lzma_attr_pure; /** * \brief Initializes Index encoder */ -extern lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i); +extern lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i) + lzma_attr_warn_unused_result; /** * \brief Initializes Index decoder */ -extern lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i); +extern lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i) + lzma_attr_warn_unused_result; diff --git a/src/liblzma/api/lzma/index_hash.h b/src/liblzma/api/lzma/index_hash.h index 1edbbeaa..58fc8061 100644 --- a/src/liblzma/api/lzma/index_hash.h +++ b/src/liblzma/api/lzma/index_hash.h @@ -42,7 +42,8 @@ typedef struct lzma_index_hash_s lzma_index_hash; * pointer than the index_hash given as argument. */ extern lzma_index_hash *lzma_index_hash_init( - lzma_index_hash *index_hash, lzma_allocator *allocator); + lzma_index_hash *index_hash, lzma_allocator *allocator) + lzma_attr_warn_unused_result; /** @@ -66,7 +67,8 @@ extern void lzma_index_hash_end( * used when lzma_index_hash_decode() has already been used. */ extern lzma_ret lzma_index_hash_append(lzma_index_hash *index_hash, - lzma_vli total_size, lzma_vli uncompressed_size); + lzma_vli total_size, lzma_vli uncompressed_size) + lzma_attr_warn_unused_result; /** @@ -83,7 +85,8 @@ extern lzma_ret lzma_index_hash_append(lzma_index_hash *index_hash, * Records can be added using lzma_index_hash_append(). */ extern lzma_ret lzma_index_hash_decode(lzma_index_hash *index_hash, - const uint8_t *in, size_t *in_pos, size_t in_size); + const uint8_t *in, size_t *in_pos, size_t in_size) + lzma_attr_warn_unused_result; /** @@ -91,4 +94,5 @@ extern lzma_ret lzma_index_hash_decode(lzma_index_hash *index_hash, * * This is needed to verify the Index Size field from the Stream Footer. */ -extern lzma_vli lzma_index_hash_size(const lzma_index_hash *index_hash); +extern lzma_vli lzma_index_hash_size(const lzma_index_hash *index_hash) + lzma_attr_pure; diff --git a/src/liblzma/api/lzma/lzma.h b/src/liblzma/api/lzma/lzma.h index 9473f448..5a1cd912 100644 --- a/src/liblzma/api/lzma/lzma.h +++ b/src/liblzma/api/lzma/lzma.h @@ -24,43 +24,11 @@ /** * \brief Filter ID * - * Filter ID of the LZMA filter. This is used as lzma_options_filter.id. + * Filter ID of the LZMA filter. This is used as lzma_filter.id. */ #define LZMA_FILTER_LZMA LZMA_VLI_C(0x40) - -/** - * \brief LZMA compression modes - * - * Currently there are only two modes. Earlier LZMA SDKs had also third - * mode between fast and best. - */ -typedef enum { - LZMA_MODE_INVALID = -1, - /**< - * \brief Invalid mode - * - * Used as array terminator in lzma_available_modes. - */ - - - LZMA_MODE_FAST = 0, - /**< - * \brief Fast compression - * - * Fast mode is usually at its best when combined with - * a hash chain match finder. - */ - - LZMA_MODE_BEST = 2 - /**< - * \brief Best compression ratio - * - * This is usually notably slower than fast mode. Use this - * together with binary tree match finders to expose the - * full potential of the LZMA encoder. - */ -} lzma_mode; +#define LZMA_FILTER_LZMA2 LZMA_VLI_C(0x21) /** @@ -128,6 +96,72 @@ typedef enum { } lzma_match_finder; +/** + * \brief Test if given match finder is supported + * + * Returns true if the given match finder is supported by this liblzma build. + * Otherwise false is returned. It is safe to call this with a value that + * isn't listed in lzma_match_finder enumeration; the return value will be + * false. + * + * There is no way to list which match finders are available in this + * particular liblzma version and build. It would be useless, because + * a new match finder, which the application developer wasn't aware, + * could require giving additional options to the encoder that the older + * match finders don't need. + */ +extern lzma_bool lzma_mf_is_supported(lzma_match_finder match_finder) + lzma_attr_const; + + +/** + * \brief LZMA compression modes + * + * This selects the function used to analyze the data produced by the match + * finder. + */ +typedef enum { + LZMA_MODE_INVALID = -1, + /**< + * \brief Invalid mode + * + * Used as array terminator in lzma_available_modes. + */ + + LZMA_MODE_FAST = 0, + /**< + * \brief Fast compression + * + * Fast mode is usually at its best when combined with + * a hash chain match finder. + */ + + LZMA_MODE_NORMAL = 1 + /**< + * \brief Normal compression + * + * This is usually notably slower than fast mode. Use this + * together with binary tree match finders to expose the + * full potential of the LZMA encoder. + */ +} lzma_mode; + + +/** + * \brief Test if given compression mode is supported + * + * Returns true if the given compression mode is supported by this liblzma + * build. Otherwise false is returned. It is safe to call this with a value + * that isn't listed in lzma_mode enumeration; the return value will be false. + * + * There is no way to list which modes are available in this particular + * liblzma version and build. It would be useless, because a new compression + * mode, which the application developer wasn't aware, could require giving + * additional options to the encoder that the older modes don't need. + */ +extern lzma_bool lzma_mode_is_available(lzma_mode mode) lzma_attr_const; + + /** * \brief Options specific to the LZMA method handler */ @@ -156,6 +190,44 @@ typedef struct { # define LZMA_DICTIONARY_SIZE_MAX (UINT32_C(1) << 30) # define LZMA_DICTIONARY_SIZE_DEFAULT (UINT32_C(1) << 23) + /** + * \brief Pointer to an initial dictionary + * + * It is possible to initialize the LZ77 history window using + * a preset dictionary. Here is a good quote from zlib's + * documentation; this applies to LZMA as is: + * + * "The dictionary should consist of strings (byte sequences) that + * are likely to be encountered later in the data to be compressed, + * with the most commonly used strings preferably put towards the + * end of the dictionary. Using a dictionary is most useful when + * the data to be compressed is short and can be predicted with + * good accuracy; the data can then be compressed better than + * with the default empty dictionary." + * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3) + * + * This feature should be used only in special situations. + * It works correctly only with raw encoding and decoding. + * Currently none of the container formats supported by + * liblzma allow preset dictionary when decoding, thus if + * you create a .lzma file with preset dictionary, it cannot + * be decoded with the regular .lzma decoder functions. + * + * \todo This feature is not implemented yet. + */ + const uint8_t *preset_dictionary; + + /** + * \brief Size of the preset dictionary + * + * Specifies the size of the preset dictionary. If the size is + * bigger than dictionary_size, only the last dictionary_size + * bytes are processed. + * + * This variable is read only when preset_dictionary is not NULL. + */ + uint32_t preset_dictionary_size; + /** * \brief Number of literal context bits * @@ -203,48 +275,23 @@ typedef struct { # define LZMA_POS_BITS_MAX 4 # define LZMA_POS_BITS_DEFAULT 2 - /** - * \brief Pointer to an initial dictionary - * - * It is possible to initialize the LZ77 history window using - * a preset dictionary. Here is a good quote from zlib's - * documentation; this applies to LZMA as is: - * - * "The dictionary should consist of strings (byte sequences) that - * are likely to be encountered later in the data to be compressed, - * with the most commonly used strings preferably put towards the - * end of the dictionary. Using a dictionary is most useful when - * the data to be compressed is short and can be predicted with - * good accuracy; the data can then be compressed better than - * with the default empty dictionary." - * (From deflateSetDictionary() in zlib.h of zlib version 1.2.3) - * - * This feature should be used only in special situations. - * It works correctly only with raw encoding and decoding. - * Currently none of the container formats supported by - * liblzma allow preset dictionary when decoding, thus if - * you create a .lzma file with preset dictionary, it cannot - * be decoded with the regular .lzma decoder functions. - * - * \todo This feature is not implemented yet. - */ - const uint8_t *preset_dictionary; - - /** - * \brief Size of the preset dictionary - * - * Specifies the size of the preset dictionary. If the size is - * bigger than dictionary_size, only the last dictionary_size - * bytes are processed. - * - * This variable is read only when preset_dictionary is not NULL. - */ - uint32_t preset_dictionary_size; - /****************************************** * LZMA options needed only when encoding * ******************************************/ + /** + * \brief Indicate if the options structure is persistent + * + * If this is true, the application must keep this options structure + * available after the LZMA2 encoder has been initialized. With + * persistent structure it is possible to change some encoder options + * in the middle of the encoding process without resetting the encoder. + * + * This option is used only by LZMA2. LZMA1 ignores this and it is + * safeto not initialize this when encoding with LZMA1. + */ + lzma_bool persistent; + /** LZMA compression mode */ lzma_mode mode; @@ -275,6 +322,20 @@ typedef struct { */ uint32_t match_finder_cycles; + /** + * \brief Reserved space for possible future extensions + * + * You should not touch these, because the names of these variables + * may change. These are and will never be used with the currently + * supported options, so it is safe to leave these uninitialized. + */ + uint32_t reserved_int1; + uint32_t reserved_int2; + uint32_t reserved_int3; + uint32_t reserved_int4; + void *reserved_ptr1; + void *reserved_ptr2; + } lzma_options_lzma; @@ -286,27 +347,6 @@ typedef struct { #define LZMA_LITERAL_BITS_MAX 4 -/** - * \brief Available LZMA encoding modes - * - * Pointer to an array containing the list of available encoding modes. - * - * This variable is available only if LZMA encoder has been enabled. - */ -extern const lzma_mode *const lzma_available_modes; - - -/** - * \brief Available match finders - * - * Pointer to an array containing the list of available match finders. - * The last element is LZMA_MF_INVALID. - * - * This variable is available only if LZMA encoder has been enabled. - */ -extern const lzma_match_finder *const lzma_available_match_finders; - - /** * \brief Table of presets for the LZMA filter * diff --git a/src/liblzma/api/lzma/memlimit.h b/src/liblzma/api/lzma/memlimit.h index 7a856a27..836b0854 100644 --- a/src/liblzma/api/lzma/memlimit.h +++ b/src/liblzma/api/lzma/memlimit.h @@ -58,7 +58,8 @@ typedef struct lzma_memlimit_s lzma_memlimit; * lzma_memlimit_ can be used even if lzma_init() hasn't been * called. */ -extern lzma_memlimit *lzma_memlimit_create(size_t limit); +extern lzma_memlimit *lzma_memlimit_create(size_t limit) + lzma_attr_warn_unused_result; /** @@ -79,7 +80,8 @@ extern void lzma_memlimit_set(lzma_memlimit *mem, size_t limit); /** * \brief Gets the current memory usage limit */ -extern size_t lzma_memlimit_get(const lzma_memlimit *mem); +extern size_t lzma_memlimit_get(const lzma_memlimit *mem) + lzma_attr_pure; /** @@ -89,7 +91,8 @@ extern size_t lzma_memlimit_get(const lzma_memlimit *mem); * thus it will always be larger than the total number of * bytes allocated via lzma_memlimit_alloc(). */ -extern size_t lzma_memlimit_used(const lzma_memlimit *mem); +extern size_t lzma_memlimit_used(const lzma_memlimit *mem) + lzma_attr_pure; /** @@ -134,7 +137,8 @@ extern lzma_bool lzma_memlimit_reached(lzma_memlimit *mem, lzma_bool clear); * been allocated with lzma_memlimit_alloc() or all memory allocated * has been freed or detached, this will return zero. */ -extern size_t lzma_memlimit_count(const lzma_memlimit *mem); +extern size_t lzma_memlimit_count(const lzma_memlimit *mem) + lzma_attr_pure; /** @@ -157,7 +161,8 @@ extern size_t lzma_memlimit_count(const lzma_memlimit *mem); * invalid amount of memory being allocated. */ extern void *lzma_memlimit_alloc( - lzma_memlimit *mem, size_t nmemb, size_t size); + lzma_memlimit *mem, size_t nmemb, size_t size) + lzma_attr_warn_unused_result; /** diff --git a/src/liblzma/api/lzma/raw.h b/src/liblzma/api/lzma/raw.h deleted file mode 100644 index db8cba15..00000000 --- a/src/liblzma/api/lzma/raw.h +++ /dev/null @@ -1,60 +0,0 @@ -/** - * \file lzma/raw.h - * \brief Raw encoder and decoder - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use instead. -#endif - - -/** - * \brief Initializes raw encoder - * - * This function may be useful when implementing custom file formats. - * - * \param strm Pointer to properly prepared lzma_stream - * \param options Array of lzma_options_filter structures. - * The end of the array must be marked with - * .id = LZMA_VLI_VALUE_UNKNOWN. The minimum - * number of filters is one and the maximum is four. - * - * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the - * filter chain supports it), or LZMA_FINISH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - * - LZMA_HEADER_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_raw_encoder( - lzma_stream *strm, const lzma_options_filter *options); - - -/** - * \brief Initializes raw decoder - * - * The initialization of raw decoder goes similarly to raw encoder. - * - * The `action' with lzma_code() can be LZMA_RUN or LZMA_SYNC_FLUSH. - * - * \return - LZMA_OK - * - LZMA_MEM_ERROR - * - LZMA_HEADER_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_raw_decoder( - lzma_stream *strm, const lzma_options_filter *options); diff --git a/src/liblzma/api/lzma/simple.h b/src/liblzma/api/lzma/simple.h index 807a4c46..13417480 100644 --- a/src/liblzma/api/lzma/simple.h +++ b/src/liblzma/api/lzma/simple.h @@ -21,7 +21,7 @@ #endif -/* Filter IDs for lzma_options_filter.id */ +/* Filter IDs for lzma_filter.id */ #define LZMA_FILTER_X86 LZMA_VLI_C(0x04) /**< diff --git a/src/liblzma/api/lzma/stream.h b/src/liblzma/api/lzma/stream.h deleted file mode 100644 index 4bb17e7d..00000000 --- a/src/liblzma/api/lzma/stream.h +++ /dev/null @@ -1,53 +0,0 @@ -/** - * \file lzma/stream.h - * \brief .lzma Stream handling - * - * \author Copyright (C) 1999-2006 Igor Pavlov - * \author Copyright (C) 2007 Lasse Collin - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - */ - -#ifndef LZMA_H_INTERNAL -# error Never include this file directly. Use instead. -#endif - - -/** - * \brief Initializes .lzma Stream encoder - * - * \param strm Pointer to properly prepared lzma_stream - * \param filters Array of filters. This must be terminated with - * filters[n].id = LZMA_VLI_VALUE_UNKNOWN. There must - * be 1-4 filters, but there are restrictions on how - * multiple filters can be combined. FIXME Tell where - * to find more information. - * \param check Type of the integrity check to calculate from - * uncompressed data. - * - * \return - LZMA_OK: Initialization was successful. - * - LZMA_MEM_ERROR - * - LZMA_HEADER_ERROR - * - LZMA_PROG_ERROR - */ -extern lzma_ret lzma_stream_encoder(lzma_stream *strm, - const lzma_options_filter *filters, lzma_check_type check); - - -/** - * \brief Initializes decoder for .lzma Stream - * - * \param strm Pointer to propertily prepared lzma_stream - * - * \return - LZMA_OK: Initialization was successful. - * - LZMA_MEM_ERROR: Cannot allocate memory. - */ -extern lzma_ret lzma_stream_decoder(lzma_stream *strm); diff --git a/src/liblzma/api/lzma/stream_flags.h b/src/liblzma/api/lzma/stream_flags.h index f4c5c335..80c5f00f 100644 --- a/src/liblzma/api/lzma/stream_flags.h +++ b/src/liblzma/api/lzma/stream_flags.h @@ -46,7 +46,7 @@ typedef struct { /** * Type of the Check calculated from uncompressed data */ - lzma_check_type check; + lzma_check check; } lzma_stream_flags; @@ -64,7 +64,8 @@ typedef struct { * - LZMA_PROG_ERROR: Invalid options. */ extern lzma_ret lzma_stream_header_encode( - const lzma_stream_flags *options, uint8_t *out); + const lzma_stream_flags *options, uint8_t *out) + lzma_attr_warn_unused_result; /** @@ -78,7 +79,8 @@ extern lzma_ret lzma_stream_header_encode( * - LZMA_PROG_ERROR: Invalid options. */ extern lzma_ret lzma_stream_footer_encode( - const lzma_stream_flags *options, uint8_t *out); + const lzma_stream_flags *options, uint8_t *out) + lzma_attr_warn_unused_result; /** @@ -101,7 +103,8 @@ extern lzma_ret lzma_stream_footer_encode( * in the header. */ extern lzma_ret lzma_stream_header_decode( - lzma_stream_flags *options, const uint8_t *in); + lzma_stream_flags *options, const uint8_t *in) + lzma_attr_warn_unused_result; /** @@ -120,7 +123,8 @@ extern lzma_ret lzma_stream_header_decode( * in the footer. */ extern lzma_ret lzma_stream_footer_decode( - lzma_stream_flags *options, const uint8_t *in); + lzma_stream_flags *options, const uint8_t *in) + lzma_attr_warn_unused_result; /** @@ -131,4 +135,5 @@ extern lzma_ret lzma_stream_footer_decode( * \return true if both structures are considered equal; false otherwise. */ extern lzma_bool lzma_stream_flags_equal( - const lzma_stream_flags *a, lzma_stream_flags *b); + const lzma_stream_flags *a, const lzma_stream_flags *b) + lzma_attr_pure; diff --git a/src/liblzma/api/lzma/subblock.h b/src/liblzma/api/lzma/subblock.h index 1db35b13..b9a3025b 100644 --- a/src/liblzma/api/lzma/subblock.h +++ b/src/liblzma/api/lzma/subblock.h @@ -24,7 +24,7 @@ /** * \brief Filter ID * - * Filter ID of the Subblock filter. This is used as lzma_options_filter.id. + * Filter ID of the Subblock filter. This is used as lzma_filter.id. */ #define LZMA_FILTER_SUBBLOCK LZMA_VLI_C(0x01) @@ -199,6 +199,6 @@ typedef struct { * * \note This variable is ignored if allow_subfilters is false. */ - lzma_options_filter subfilter_options; + lzma_filter subfilter_options; } lzma_options_subblock; diff --git a/src/liblzma/api/lzma/version.h b/src/liblzma/api/lzma/version.h index 252458a3..811f93e0 100644 --- a/src/liblzma/api/lzma/version.h +++ b/src/liblzma/api/lzma/version.h @@ -41,17 +41,17 @@ /** * \brief liblzma version number as an integer * - * This is the value of LZMA_VERSION macro at the compile time of liblzma. + * Returns the value of LZMA_VERSION macro at the compile time of liblzma. * This allows the application to compare if it was built against the same, * older, or newer version of liblzma that is currently running. */ -extern const uint32_t lzma_version_number; +extern uint32_t lzma_version_number(void) lzma_attr_const; /** - * \brief Returns versions number of liblzma as a string + * \brief Version number of liblzma as a string * * This function may be useful if you want to display which version of - * libilzma your application is currently using. + * liblzma your application is currently using. */ -extern const char *const lzma_version_string; +extern const char *lzma_version_string(void) lzma_attr_const; diff --git a/src/liblzma/api/lzma/vli.h b/src/liblzma/api/lzma/vli.h index 15a9d0bf..294e5cdd 100644 --- a/src/liblzma/api/lzma/vli.h +++ b/src/liblzma/api/lzma/vli.h @@ -71,91 +71,25 @@ typedef uint64_t lzma_vli; ((vli) <= LZMA_VLI_VALUE_MAX || (vli) == LZMA_VLI_VALUE_UNKNOWN) -/** - * \brief Sets VLI to given value with error checking - * - * \param dest Target variable which must have type of lzma_vli. - * \param src New value to be stored to dest. - * \param limit Maximum allowed value for src. - * - * \return False on success, true on error. If an error occurred, - * dest is left in undefined state (i.e. it's possible that - * it will be different in newer liblzma versions). - */ -#define lzma_vli_set_lim(dest, src, limit) \ - ((src) > (limit) || ((dest) = (src)) > (limit)) - -/** - * \brief - */ -#define lzma_vli_add_lim(dest, src, limit) \ - ((src) > (limit) || ((dest) += (src)) > (limit)) - -#define lzma_vli_add2_lim(dest, src1, src2, limit) \ - (lzma_vli_add_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit)) - -#define lzma_vli_add3_lim(dest, src1, src2, src3, limit) \ - (lzma_vli_add_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit)) - -#define lzma_vli_add4_lim(dest, src1, src2, src3, src4, limit) \ - (lzma_vli_add_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit) \ - || lzma_vli_add_lim(dest, src4, limit)) - -#define lzma_vli_sum_lim(dest, src1, src2, limit) \ - (lzma_vli_set_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit)) - -#define lzma_vli_sum3_lim(dest, src1, src2, src3, limit) \ - (lzma_vli_set_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit)) - -#define lzma_vli_sum4_lim(dest, src1, src2, src3, src4, limit) \ - (lzma_vli_set_lim(dest, src1, limit) \ - || lzma_vli_add_lim(dest, src2, limit) \ - || lzma_vli_add_lim(dest, src3, limit) \ - || lzma_vli_add_lim(dest, src4, limit)) - -#define lzma_vli_set(dest, src) lzma_vli_set_lim(dest, src, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add(dest, src) lzma_vli_add_lim(dest, src, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add2(dest, src1, src2) \ - lzma_vli_add2_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add3(dest, src1, src2, src3) \ - lzma_vli_add3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_add4(dest, src1, src2, src3, src4) \ - lzma_vli_add4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_sum(dest, src1, src2) \ - lzma_vli_sum_lim(dest, src1, src2, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_sum3(dest, src1, src2, src3) \ - lzma_vli_sum3_lim(dest, src1, src2, src3, LZMA_VLI_VALUE_MAX) - -#define lzma_vli_sum4(dest, src1, src2, src3, src4) \ - lzma_vli_sum4_lim(dest, src1, src2, src3, src4, LZMA_VLI_VALUE_MAX) - - /** * \brief Encodes variable-length integer * - * In the new .lzma format, most integers are encoded in variable-length + * In the .lzma format, most integers are encoded in variable-length * representation. This saves space when smaller values are more likely * than bigger values. * * The encoding scheme encodes seven bits to every byte, using minimum - * number of bytes required to represent the given value. In other words, - * it puts 7-63 bits into 1-9 bytes. This implementation limits the number - * of bits used to 63, thus num must be at maximum of UINT64_MAX / 2. You - * may use LZMA_VLI_VALUE_MAX for clarity. + * number of bytes required to represent the given value. Encodings that use + * non-minimum number of bytes are invalid, thus every integer has exactly + * one encoded representation. The maximum number of bits in a VLI is 63, + * thus the vli argument must be at maximum of UINT64_MAX / 2. You should + * use LZMA_VLI_VALUE_MAX for clarity. + * + * This function has two modes: single-call and multi-call. Single-call mode + * encodes the whole integer at once; it is an error if the output buffer is + * too small. Multi-call mode saves the position in *vli_pos, and thus it is + * possible to continue encoding if the buffer becomes full before the whole + * integer has been encoded. * * \param vli Integer to be encoded * \param vli_pos How many VLI-encoded bytes have already been written @@ -170,19 +104,19 @@ typedef uint64_t lzma_vli; * \return Slightly different return values are used in multi-call and * single-call modes. * + * Single-call (vli_pos == NULL): + * - LZMA_OK: Integer successfully encoded. + * - LZMA_PROG_ERROR: Arguments are not sane. This can be due + * to too little output space; single-call mode doesn't use + * LZMA_BUF_ERROR, since the application should have checked + * the encoded size with lzma_vli_size(). + * * Multi-call (vli_pos != NULL): * - LZMA_OK: So far all OK, but the integer is not * completely written out yet. * - LZMA_STREAM_END: Integer successfully encoded. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be due - * to no *out_pos == out_size; this function doesn't use - * LZMA_BUF_ERROR. - * - * Single-call (vli_pos == NULL): - * - LZMA_OK: Integer successfully encoded. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be due - * to too little output space; this function doesn't use - * LZMA_BUF_ERROR. + * - LZMA_BUF_ERROR: No output space was provided. + * - LZMA_PROG_ERROR: Arguments are not sane. */ extern lzma_ret lzma_vli_encode( lzma_vli vli, size_t *lzma_restrict vli_pos, @@ -193,6 +127,8 @@ extern lzma_ret lzma_vli_encode( /** * \brief Decodes variable-length integer * + * Like lzma_vli_encode(), this function has single-call and multi-call modes. + * * \param vli Pointer to decoded integer. The decoder will * initialize it to zero when *vli_pos == 0, so * application isn't required to initialize *vli. @@ -208,20 +144,20 @@ extern lzma_ret lzma_vli_encode( * \return Slightly different return values are used in multi-call and * single-call modes. * + * Single-call (vli_pos == NULL): + * - LZMA_OK: Integer successfully decoded. + * - LZMA_DATA_ERROR: Integer is corrupt. This includes hitting + * the end of the input buffer before the whole integer was + * decoded; providing no input at all will use LZMA_DATA_ERROR. + * - LZMA_PROG_ERROR: Arguments are not sane. + * * Multi-call (vli_pos != NULL): * - LZMA_OK: So far all OK, but the integer is not * completely decoded yet. * - LZMA_STREAM_END: Integer successfully decoded. * - LZMA_DATA_ERROR: Integer is corrupt. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be - * due to *in_pos == in_size; this function doesn't use - * LZMA_BUF_ERROR. - * - * Single-call (vli_pos == NULL): - * - LZMA_OK: Integer successfully decoded. - * - LZMA_DATA_ERROR: Integer is corrupt. - * - LZMA_PROG_ERROR: Arguments are not sane. This can be due to - * too little input; this function doesn't use LZMA_BUF_ERROR. + * - LZMA_BUF_ERROR: No input was provided. + * - LZMA_PROG_ERROR: Arguments are not sane. */ extern lzma_ret lzma_vli_decode(lzma_vli *lzma_restrict vli, size_t *lzma_restrict vli_pos, const uint8_t *lzma_restrict in, @@ -234,4 +170,5 @@ extern lzma_ret lzma_vli_decode(lzma_vli *lzma_restrict vli, * \return Number of bytes on success (1-9). If vli isn't valid, * zero is returned. */ -extern uint32_t lzma_vli_size(lzma_vli vli); +extern uint32_t lzma_vli_size(lzma_vli vli) + lzma_attr_pure; diff --git a/src/liblzma/check/check.c b/src/liblzma/check/check.c index 388b57e8..ed64fe5c 100644 --- a/src/liblzma/check/check.c +++ b/src/liblzma/check/check.c @@ -13,60 +13,77 @@ #include "check.h" -// See the .lzma header format specification section 2.1.1.2. -LZMA_API const uint32_t lzma_check_sizes[LZMA_CHECK_ID_MAX + 1] = { - 0, - 4, 4, 4, - 8, 8, 8, - 16, 16, 16, - 32, 32, 32, - 64, 64, 64 -}; +extern LZMA_API lzma_bool +lzma_check_is_supported(lzma_check type) +{ + if ((unsigned)(type) > LZMA_CHECK_ID_MAX) + return false; -LZMA_API const lzma_bool lzma_available_checks[LZMA_CHECK_ID_MAX + 1] = { - true, // LZMA_CHECK_NONE + static const lzma_bool available_checks[LZMA_CHECK_ID_MAX + 1] = { + true, // LZMA_CHECK_NONE #ifdef HAVE_CHECK_CRC32 - true, + true, #else - false, + false, #endif - false, // Reserved - false, // Reserved + false, // Reserved + false, // Reserved #ifdef HAVE_CHECK_CRC64 - true, + true, #else - false, + false, #endif - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved #ifdef HAVE_CHECK_SHA256 - true, + true, #else - false, + false, #endif - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved - false, // Reserved -}; + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + false, // Reserved + }; + + return available_checks[(unsigned)(type)]; +} -extern lzma_ret -lzma_check_init(lzma_check *check, lzma_check_type type) +extern LZMA_API uint32_t +lzma_check_size(lzma_check type) { - lzma_ret ret = LZMA_OK; + if ((unsigned)(type) > LZMA_CHECK_ID_MAX) + return UINT32_MAX; + // See file-format.txt section 2.1.1.2. + static const uint8_t check_sizes[LZMA_CHECK_ID_MAX + 1] = { + 0, + 4, 4, 4, + 8, 8, 8, + 16, 16, 16, + 32, 32, 32, + 64, 64, 64 + }; + + return check_sizes[(unsigned)(type)]; +} + + +extern void +lzma_check_init(lzma_check_state *check, lzma_check type) +{ switch (type) { case LZMA_CHECK_NONE: break; @@ -90,19 +107,15 @@ lzma_check_init(lzma_check *check, lzma_check_type type) #endif default: - if ((unsigned)(type) <= LZMA_CHECK_ID_MAX) - ret = LZMA_UNSUPPORTED_CHECK; - else - ret = LZMA_PROG_ERROR; break; } - return ret; + return; } extern void -lzma_check_update(lzma_check *check, lzma_check_type type, +lzma_check_update(lzma_check_state *check, lzma_check type, const uint8_t *buf, size_t size) { switch (type) { @@ -133,18 +146,18 @@ lzma_check_update(lzma_check *check, lzma_check_type type, extern void -lzma_check_finish(lzma_check *check, lzma_check_type type) +lzma_check_finish(lzma_check_state *check, lzma_check type) { switch (type) { #ifdef HAVE_CHECK_CRC32 case LZMA_CHECK_CRC32: - *(uint32_t *)(check->buffer) = check->state.crc32; + check->buffer.u32[0] = integer_le_32(check->state.crc32); break; #endif #ifdef HAVE_CHECK_CRC64 case LZMA_CHECK_CRC64: - *(uint64_t *)(check->buffer) = check->state.crc64; + check->buffer.u64[0] = integer_le_64(check->state.crc64); break; #endif @@ -160,34 +173,3 @@ lzma_check_finish(lzma_check *check, lzma_check_type type) return; } - - -/* -extern bool -lzma_check_compare( - lzma_check *check1, lzma_check *check2, lzma_check_type type) -{ - bool ret; - - switch (type) { - case LZMA_CHECK_NONE: - break; - - case LZMA_CHECK_CRC32: - ret = check1->crc32 != check2->crc32; - break; - - case LZMA_CHECK_CRC64: - ret = check1->crc64 != check2->crc64; - break; - - default: - // Unsupported check - assert(type <= 7); - ret = false; - break; - } - - return ret; -} -*/ diff --git a/src/liblzma/check/check.h b/src/liblzma/check/check.h index 45ca25e9..8f387799 100644 --- a/src/liblzma/check/check.h +++ b/src/liblzma/check/check.h @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file check.h -/// \brief Prototypes for different check functions +/// \brief Internal API to different integrity check functions // // This code has been put into the public domain. // @@ -17,8 +17,8 @@ #include "common.h" -// Index hashing used to verify the Index with O(1) memory usage needs -// a good hash function. +// Index hashing needs the best possible hash function (preferably +// a cryptographic hash) for maximum reliability. #if defined(HAVE_CHECK_SHA256) # define LZMA_CHECK_BEST LZMA_CHECK_SHA256 #elif defined(HAVE_CHECK_CRC64) @@ -28,24 +28,17 @@ #endif +/// \brief Structure to hold internal state of the check being calculated +/// +/// \note This is not in the public API because this structure may +/// change in future if new integrity check algorithms are added. typedef struct { - /// Internal state - uint32_t state[8]; - - /// Size of the message excluding padding - uint64_t size; - -} lzma_sha256; - - -/// \note This is not in the public API because this structure will -/// change in future. -typedef struct { - // FIXME Guarantee 8-byte alignment - - /// Buffer to hold the final result; this is also used as a temporary - /// buffer in SHA256. Note that this buffer must be 8-byte aligned. - uint8_t buffer[64]; + /// Buffer to hold the final result and a temporary buffer for SHA256. + union { + uint8_t u8[64]; + uint32_t u32[16]; + uint64_t u64[8]; + } buffer; /// Check-specific data union { @@ -61,7 +54,7 @@ typedef struct { } sha256; } state; -} lzma_check; +} lzma_check_state; #ifdef HAVE_SMALL @@ -72,7 +65,6 @@ extern const uint32_t lzma_crc32_table[8][256]; extern const uint64_t lzma_crc64_table[4][256]; #endif -// Generic /// \brief Initializes *check depending on type /// @@ -80,46 +72,31 @@ extern const uint64_t lzma_crc64_table[4][256]; /// supported by the current version or build of liblzma. /// LZMA_PROG_ERROR if type > LZMA_CHECK_ID_MAX. /// -extern lzma_ret lzma_check_init(lzma_check *check, lzma_check_type type); +extern void lzma_check_init(lzma_check_state *check, lzma_check type); + /// \brief Updates *check /// -extern void lzma_check_update(lzma_check *check, lzma_check_type type, +extern void lzma_check_update(lzma_check_state *check, lzma_check type, const uint8_t *buf, size_t size); + /// \brief Finishes *check /// -extern void lzma_check_finish(lzma_check *check, lzma_check_type type); +extern void lzma_check_finish(lzma_check_state *check, lzma_check type); -/* -/// \brief Compare two checks -/// -/// \return false if the checks are identical; true if they differ. -/// -extern bool lzma_check_compare( - lzma_check *check1, lzma_check *check2, lzma_check_type type); -*/ - - -// CRC32 - extern void lzma_crc32_init(void); -// CRC64 - extern void lzma_crc64_init(void); -// SHA256 - -extern void lzma_sha256_init(lzma_check *check); +extern void lzma_sha256_init(lzma_check_state *check); extern void lzma_sha256_update( - const uint8_t *buf, size_t size, lzma_check *check); - -extern void lzma_sha256_finish(lzma_check *check); + const uint8_t *buf, size_t size, lzma_check_state *check); +extern void lzma_sha256_finish(lzma_check_state *check); #endif diff --git a/src/liblzma/check/sha256.c b/src/liblzma/check/sha256.c index ea51896e..9f90f7e7 100644 --- a/src/liblzma/check/sha256.c +++ b/src/liblzma/check/sha256.c @@ -104,16 +104,16 @@ transform(uint32_t state[static 8], const uint32_t data[static 16]) static void -process(lzma_check *check) +process(lzma_check_state *check) { #ifdef WORDS_BIGENDIAN - transform(check->state.sha256.state, (uint32_t *)(check->buffer)); + transform(check->state.sha256.state, check->buffer.u32); #else uint32_t data[16]; for (size_t i = 0; i < 16; ++i) - data[i] = bswap_32(*((uint32_t*)(check->buffer) + i)); + data[i] = bswap_32(check->buffer.u32[i]); transform(check->state.sha256.state, data); #endif @@ -123,7 +123,7 @@ process(lzma_check *check) extern void -lzma_sha256_init(lzma_check *check) +lzma_sha256_init(lzma_check_state *check) { static const uint32_t s[8] = { 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, @@ -138,7 +138,7 @@ lzma_sha256_init(lzma_check *check) extern void -lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check) +lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check_state *check) { // Copy the input data into a properly aligned temporary buffer. // This way we can be called with arbitrarily sized buffers @@ -150,7 +150,7 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check) if (copy_size > size) copy_size = size; - memcpy(check->buffer + copy_start, buf, copy_size); + memcpy(check->buffer.u8 + copy_start, buf, copy_size); buf += copy_size; size -= copy_size; @@ -165,12 +165,12 @@ lzma_sha256_update(const uint8_t *buf, size_t size, lzma_check *check) extern void -lzma_sha256_finish(lzma_check *check) +lzma_sha256_finish(lzma_check_state *check) { // Add padding as described in RFC 3174 (it describes SHA-1 but // the same padding style is used for SHA-256 too). size_t pos = check->state.sha256.size & 0x3F; - check->buffer[pos++] = 0x80; + check->buffer.u8[pos++] = 0x80; while (pos != 64 - 8) { if (pos == 64) { @@ -178,28 +178,25 @@ lzma_sha256_finish(lzma_check *check) pos = 0; } - check->buffer[pos++] = 0x00; + check->buffer.u8[pos++] = 0x00; } // Convert the message size from bytes to bits. check->state.sha256.size *= 8; #ifdef WORDS_BIGENDIAN - *(uint64_t *)(check->buffer + 64 - 8) = check->state.sha256.size; + check->buffer.u64[(64 - 8) / 8] = check->state.sha256.size; #else - *(uint64_t *)(check->buffer + 64 - 8) - = bswap_64(check->state.sha256.size); + check->buffer.u64[(64 - 8) / 8] = bswap_64(check->state.sha256.size); #endif process(check); for (size_t i = 0; i < 8; ++i) #ifdef WORDS_BIGENDIAN - ((uint32_t *)(check->buffer))[i] - = check->state.sha256.state[i]; + check->buffer.u32[i] = check->state.sha256.state[i]; #else - ((uint32_t *)(check->buffer))[i] - = bswap_32(check->state.sha256.state[i]); + check->buffer.u32[i] = bswap_32(check->state.sha256.state[i]); #endif return; diff --git a/src/liblzma/common/Makefile.am b/src/liblzma/common/Makefile.am index 40b42250..3ec2e270 100644 --- a/src/liblzma/common/Makefile.am +++ b/src/liblzma/common/Makefile.am @@ -16,62 +16,43 @@ noinst_LTLIBRARIES = libcommon.la libcommon_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/check \ + -I@top_srcdir@/src/liblzma/rangecoder \ -I@top_srcdir@/src/liblzma/lz \ -I@top_srcdir@/src/liblzma/lzma \ - -I@top_srcdir@/src/liblzma/simple \ -I@top_srcdir@/src/liblzma/subblock \ - -I@top_srcdir@/src/liblzma/rangecoder + -I@top_srcdir@/src/liblzma/delta \ + -I@top_srcdir@/src/liblzma/simple + libcommon_la_SOURCES = \ + common.c \ common.h \ bsr.h \ - allocator.c \ block_util.c \ block_private.h \ - features.c \ + filter_common.c \ + filter_common.h \ index.c \ + index.h \ init.c \ memory_limiter.c \ - memory_usage.c \ - next_coder.c \ - raw_common.c \ - raw_common.h \ - stream_flags_equal.c \ - code.c \ - version.c - -if COND_FILTER_DELTA -libcommon_la_SOURCES += \ - delta_common.c \ - delta_common.h -if COND_MAIN_ENCODER -libcommon_la_SOURCES += \ - delta_encoder.c \ - delta_encoder.h -endif -if COND_MAIN_DECODER -libcommon_la_SOURCES += \ - delta_decoder.c \ - delta_decoder.h -endif -endif + stream_flags_common.c \ + stream_flags_common.h \ + vli_size.c if COND_MAIN_ENCODER libcommon_la_SOURCES += \ alignment.c \ - auto_decoder.c \ alone_encoder.c \ block_encoder.c \ block_encoder.h \ block_header_encoder.c \ easy.c \ + filter_encoder.c \ + filter_encoder.h \ filter_flags_encoder.c \ index_encoder.c \ index_encoder.h \ init_encoder.c \ - raw_encoder.c \ - raw_encoder.h \ - stream_common.c \ - stream_common.h \ stream_encoder.c \ stream_encoder.h \ stream_flags_encoder.c \ @@ -82,16 +63,18 @@ if COND_MAIN_DECODER libcommon_la_SOURCES += \ alone_decoder.c \ alone_decoder.h \ + auto_decoder.c \ block_decoder.c \ block_decoder.h \ block_header_decoder.c \ + filter_decoder.c \ + filter_decoder.h \ filter_flags_decoder.c \ index_decoder.c \ index_hash.c \ init_decoder.c \ - raw_decoder.c \ - raw_decoder.h \ stream_decoder.c \ + stream_decoder.h \ stream_flags_decoder.c \ stream_flags_decoder.h \ vli_decoder.c diff --git a/src/liblzma/common/alignment.c b/src/liblzma/common/alignment.c index c80e5fab..ba9ecb03 100644 --- a/src/liblzma/common/alignment.c +++ b/src/liblzma/common/alignment.c @@ -21,7 +21,7 @@ extern LZMA_API uint32_t -lzma_alignment_input(const lzma_options_filter *filters, uint32_t guess) +lzma_alignment_input(const lzma_filter *filters, uint32_t guess) { for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { switch (filters[i].id) { @@ -66,7 +66,7 @@ lzma_alignment_input(const lzma_options_filter *filters, uint32_t guess) extern LZMA_API uint32_t -lzma_alignment_output(const lzma_options_filter *filters, uint32_t guess) +lzma_alignment_output(const lzma_filter *filters, uint32_t guess) { if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN) return UINT32_MAX; diff --git a/src/liblzma/common/allocator.c b/src/liblzma/common/allocator.c deleted file mode 100644 index 5ced9d16..00000000 --- a/src/liblzma/common/allocator.c +++ /dev/null @@ -1,58 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file allocator.c -/// \brief Allocating and freeing memory -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - -#undef lzma_free - -extern void * lzma_attribute((malloc)) -lzma_alloc(size_t size, lzma_allocator *allocator) -{ - // Some malloc() variants return NULL if called with size == 0. - if (size == 0) - size = 1; - - void *ptr; - - if (allocator != NULL && allocator->alloc != NULL) - ptr = allocator->alloc(allocator->opaque, 1, size); - else - ptr = malloc(size); - -#ifndef NDEBUG - // This helps to catch some stupid mistakes, but also hides them from - // Valgrind. Uncomment when useful. -// if (ptr != NULL) -// memset(ptr, 0xFD, size); -#endif - - return ptr; -} - - -extern void -lzma_free(void *ptr, lzma_allocator *allocator) -{ - if (allocator != NULL && allocator->free != NULL) - allocator->free(allocator->opaque, ptr); - else - free(ptr); - - return; -} diff --git a/src/liblzma/common/alone_decoder.c b/src/liblzma/common/alone_decoder.c index 062f6fab..006740f4 100644 --- a/src/liblzma/common/alone_decoder.c +++ b/src/liblzma/common/alone_decoder.c @@ -19,6 +19,7 @@ #include "alone_decoder.h" #include "lzma_decoder.h" +#include "lz_decoder.h" struct lzma_coder_s { @@ -38,6 +39,9 @@ struct lzma_coder_s { /// Uncompressed size decoded from the header lzma_vli uncompressed_size; + /// Memory usage limit + uint64_t memlimit; + /// Options decoded from the header needed to initialize /// the LZMA decoder lzma_options_lzma options; @@ -56,7 +60,7 @@ alone_decode(lzma_coder *coder, && (coder->sequence == SEQ_CODE || *in_pos < in_size)) switch (coder->sequence) { case SEQ_PROPERTIES: - if (lzma_lzma_decode_properties(&coder->options, in[*in_pos])) + if (lzma_lzma_lclppb_decode(&coder->options, in[*in_pos])) return LZMA_FORMAT_ERROR; coder->sequence = SEQ_DICTIONARY_SIZE; @@ -69,8 +73,6 @@ alone_decode(lzma_coder *coder, if (++coder->pos == 4) { if (coder->options.dictionary_size - < LZMA_DICTIONARY_SIZE_MIN - || coder->options.dictionary_size > LZMA_DICTIONARY_SIZE_MAX) return LZMA_FORMAT_ERROR; @@ -119,7 +121,20 @@ alone_decode(lzma_coder *coder, break; case SEQ_CODER_INIT: { - // Two is enough because there won't be implicit filters. + // FIXME It is unfair that this doesn't add a fixed amount + // like lzma_memusage_common() does. + const uint64_t memusage + = lzma_lzma_decoder_memusage(&coder->options); + + // Use LZMA_PROG_ERROR since LZMA_Alone decoder cannot be + // built without LZMA support. + // FIXME TODO Make the above comment true. + if (memusage == UINT64_MAX) + return LZMA_PROG_ERROR; + + if (memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; + lzma_filter_info filters[2] = { { .init = &lzma_lzma_decoder_init, @@ -135,7 +150,7 @@ alone_decode(lzma_coder *coder, return ret; // Use a hack to set the uncompressed size. - lzma_lzma_decoder_uncompressed_size(&coder->next, + lzma_lz_decoder_uncompressed(coder->next.coder, coder->uncompressed_size); coder->sequence = SEQ_CODE; @@ -160,15 +175,18 @@ alone_decode(lzma_coder *coder, static void alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +extern lzma_ret +lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit) { + lzma_next_coder_init(lzma_alone_decoder_init, next, allocator); + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -183,25 +201,20 @@ alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) next->coder->pos = 0; next->coder->options.dictionary_size = 0; next->coder->uncompressed_size = 0; + next->coder->memlimit = memlimit; return LZMA_OK; } -extern lzma_ret -lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) -{ - lzma_next_coder_init0(alone_decoder_init, next, allocator); -} - - extern LZMA_API lzma_ret -lzma_alone_decoder(lzma_stream *strm) +lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) { - lzma_next_strm_init0(strm, alone_decoder_init); + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; FIXME + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/src/liblzma/common/alone_decoder.h b/src/liblzma/common/alone_decoder.h index a9b7e84b..13284043 100644 --- a/src/liblzma/common/alone_decoder.h +++ b/src/liblzma/common/alone_decoder.h @@ -17,8 +17,13 @@ // /////////////////////////////////////////////////////////////////////////////// +#ifndef LZMA_ALONE_DECODER_H +#define LZMA_ALONE_DECODER_H + #include "common.h" -extern lzma_ret lzma_alone_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator); +extern lzma_ret lzma_alone_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, uint64_t memlimit); + +#endif diff --git a/src/liblzma/common/alone_encoder.c b/src/liblzma/common/alone_encoder.c index f94a21c1..7fb11570 100644 --- a/src/liblzma/common/alone_encoder.c +++ b/src/liblzma/common/alone_encoder.c @@ -48,7 +48,7 @@ alone_encode(lzma_coder *coder, while (*out_pos < out_size) switch (coder->sequence) { case SEQ_HEADER: - bufcpy(coder->header, &coder->header_pos, + lzma_bufcpy(coder->header, &coder->header_pos, ALONE_HEADER_SIZE, out, out_pos, out_size); if (coder->header_pos < ALONE_HEADER_SIZE) @@ -74,7 +74,7 @@ alone_encode(lzma_coder *coder, static void alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } @@ -85,6 +85,8 @@ static lzma_ret alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_options_lzma *options) { + lzma_next_coder_init(alone_encoder_init, next, allocator); + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -101,7 +103,7 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Encode the header: // - Properties (1 byte) - if (lzma_lzma_encode_properties(options, next->coder->header)) + if (lzma_lzma_lclppb_encode(options, next->coder->header)) return LZMA_PROG_ERROR; // - Dictionary size (4 bytes) @@ -113,6 +115,9 @@ alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // one is the next. While the header would allow any 32-bit integer, // we do this to keep the decoder of liblzma accepting the resulting // files. + // + // FIXME Maybe LZMA_Alone needs some lower limit for maximum + // dictionary size? Must check decoders from old LZMA SDK version. uint32_t d = options->dictionary_size - 1; d |= d >> 2; d |= d >> 3; @@ -153,7 +158,7 @@ lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API lzma_ret lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) { - lzma_next_strm_init(strm, alone_encoder_init, options); + lzma_next_strm_init(alone_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c index 765a27b1..5fcdf168 100644 --- a/src/liblzma/common/auto_decoder.c +++ b/src/liblzma/common/auto_decoder.c @@ -23,6 +23,8 @@ struct lzma_coder_s { lzma_next_coder next; + uint64_t memlimit; + uint32_t flags; bool initialized; }; @@ -41,9 +43,11 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, if (in[*in_pos] == 0xFF) ret = lzma_stream_decoder_init( - &coder->next, allocator); + &coder->next, allocator, + coder->memlimit, coder->flags); else - ret = lzma_alone_decoder_init(&coder->next, allocator); + ret = lzma_alone_decoder_init(&coder->next, + allocator, coder->memlimit); if (ret != LZMA_OK) return ret; @@ -59,15 +63,21 @@ auto_decode(lzma_coder *coder, lzma_allocator *allocator, static void auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } static lzma_ret -auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) { + lzma_next_coder_init(auto_decoder_init, next, allocator); + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_HEADER_ERROR; + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -78,30 +88,22 @@ auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) next->coder->next = LZMA_NEXT_CODER_INIT; } + next->coder->memlimit = memlimit; + next->coder->flags = flags; next->coder->initialized = false; return LZMA_OK; } -/* -extern lzma_ret -lzma_auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_extra **header, lzma_extra **footer) -{ - lzma_next_coder_init( - auto_decoder_init, next, allocator, header, footer); -} -*/ - - extern LZMA_API lzma_ret -lzma_auto_decoder(lzma_stream *strm) +lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init0(strm, auto_decoder_init); + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; FIXME + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/src/liblzma/common/block_decoder.c b/src/liblzma/common/block_decoder.c index f07c4e06..2c16a204 100644 --- a/src/liblzma/common/block_decoder.c +++ b/src/liblzma/common/block_decoder.c @@ -19,7 +19,7 @@ #include "block_decoder.h" #include "block_private.h" -#include "raw_decoder.h" +#include "filter_decoder.h" #include "check.h" @@ -35,7 +35,7 @@ struct lzma_coder_s { /// Decoding options; we also write Compressed Size and Uncompressed /// Size back to this structure when the encoding has been finished. - lzma_options_block *options; + lzma_block *options; /// Compressed Size calculated while encoding lzma_vli compressed_size; @@ -52,7 +52,7 @@ struct lzma_coder_s { size_t check_pos; /// Check of the uncompressed data - lzma_check check; + lzma_check_state check; }; @@ -64,9 +64,6 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, { switch (coder->sequence) { case SEQ_CODE: { - if (*out_pos >= out_size) - return LZMA_OK; - const size_t in_start = *in_pos; const size_t out_start = *out_pos; @@ -98,7 +95,7 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_PADDING: - // If Compressed Data is padded to a multiple of four bytes. + // Compressed Data is padded to a multiple of four bytes. while (coder->compressed_size & 3) { if (*in_pos >= in_size) return LZMA_OK; @@ -132,19 +129,29 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, // Fall through - case SEQ_CHECK: - while (*in_pos < in_size) { - if (in[(*in_pos)++] != coder->check.buffer[ - coder->check_pos]) - return LZMA_DATA_ERROR; + case SEQ_CHECK: { + const bool chksup = lzma_check_is_supported( + coder->options->check); - if (++coder->check_pos == lzma_check_sizes[ - coder->options->check]) + while (*in_pos < in_size) { + // coder->check.buffer[] may be uninitialized when + // the Check ID is not supported. + if (chksup && coder->check.buffer.u8[coder->check_pos] + != in[*in_pos]) { + ++*in_pos; + return LZMA_DATA_ERROR; + } + + ++*in_pos; + + if (++coder->check_pos == lzma_check_size( + coder->options->check)) return LZMA_STREAM_END; } return LZMA_OK; } + } return LZMA_PROG_ERROR; } @@ -153,21 +160,28 @@ block_decode(lzma_coder *coder, lzma_allocator *allocator, static void block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) +extern lzma_ret +lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_block *options) { + lzma_next_coder_init(lzma_block_decoder_init, next, allocator); + // While lzma_block_total_size_get() is meant to calculate the Total // Size, it also validates the options excluding the filters. if (lzma_block_total_size_get(options) == 0) return LZMA_PROG_ERROR; + // options->check is used for array indexing so we need to know that + // it is in the valid range. + if ((unsigned)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + // Allocate and initialize *next->coder if needed. if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); @@ -192,30 +206,25 @@ block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, = options->compressed_size == LZMA_VLI_VALUE_UNKNOWN ? (LZMA_VLI_VALUE_MAX & ~LZMA_VLI_C(3)) - options->header_size - - lzma_check_sizes[options->check] + - lzma_check_size(options->check) : options->compressed_size; - // Initialize the check + // Initialize the check. It's caller's problem if the Check ID is not + // supported, and the Block decoder cannot verify the Check field. + // Caller can test lzma_checks[options->check]. next->coder->check_pos = 0; - return_if_error(lzma_check_init(&next->coder->check, options->check)); + lzma_check_init(&next->coder->check, options->check); + // Initialize the filter chain. return lzma_raw_decoder_init(&next->coder->next, allocator, options->filters); } -extern lzma_ret -lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) -{ - lzma_next_coder_init(block_decoder_init, next, allocator, options); -} - - extern LZMA_API lzma_ret -lzma_block_decoder(lzma_stream *strm, lzma_options_block *options) +lzma_block_decoder(lzma_stream *strm, lzma_block *options) { - lzma_next_strm_init(strm, block_decoder_init, options); + lzma_next_strm_init(lzma_block_decoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; diff --git a/src/liblzma/common/block_decoder.h b/src/liblzma/common/block_decoder.h index af71128d..999aa748 100644 --- a/src/liblzma/common/block_decoder.h +++ b/src/liblzma/common/block_decoder.h @@ -24,6 +24,6 @@ extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_options_block *options); + lzma_allocator *allocator, lzma_block *options); #endif diff --git a/src/liblzma/common/block_encoder.c b/src/liblzma/common/block_encoder.c index 3add45a9..5aa3626b 100644 --- a/src/liblzma/common/block_encoder.c +++ b/src/liblzma/common/block_encoder.c @@ -19,7 +19,7 @@ #include "block_encoder.h" #include "block_private.h" -#include "raw_encoder.h" +#include "filter_encoder.h" #include "check.h" @@ -30,7 +30,7 @@ struct lzma_coder_s { /// Encoding options; we also write Total Size, Compressed Size, and /// Uncompressed Size back to this structure when the encoding has /// been finished. - lzma_options_block *options; + lzma_block *options; enum { SEQ_CODE, @@ -48,7 +48,7 @@ struct lzma_coder_s { size_t check_pos; /// Check of the uncompressed data - lzma_check check; + lzma_check_state check; }; @@ -147,11 +147,11 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, // Fall through case SEQ_CHECK: - out[*out_pos] = coder->check.buffer[coder->check_pos]; + out[*out_pos] = coder->check.buffer.u8[coder->check_pos]; ++*out_pos; if (++coder->check_pos - == lzma_check_sizes[coder->options->check]) + == lzma_check_size(coder->options->check)) return LZMA_STREAM_END; break; @@ -167,21 +167,31 @@ block_encode(lzma_coder *coder, lzma_allocator *allocator, static void block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) +extern lzma_ret +lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_block *options) { + lzma_next_coder_init(lzma_block_encoder_init, next, allocator); + // While lzma_block_total_size_get() is meant to calculate the Total // Size, it also validates the options excluding the filters. if (lzma_block_total_size_get(options) == 0) return LZMA_PROG_ERROR; + // If the Check ID is not supported, we cannot calculate the check and + // thus not create a proper Block. + if ((unsigned)(options->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(options->check)) + return LZMA_UNSUPPORTED_CHECK; + // Allocate and initialize *next->coder if needed. if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); @@ -201,7 +211,7 @@ block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Initialize the check next->coder->check_pos = 0; - return_if_error(lzma_check_init(&next->coder->check, options->check)); + lzma_check_init(&next->coder->check, options->check); // Initialize the requested filters. return lzma_raw_encoder_init(&next->coder->next, allocator, @@ -209,18 +219,10 @@ block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -extern lzma_ret -lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_options_block *options) -{ - lzma_next_coder_init(block_encoder_init, next, allocator, options); -} - - extern LZMA_API lzma_ret -lzma_block_encoder(lzma_stream *strm, lzma_options_block *options) +lzma_block_encoder(lzma_stream *strm, lzma_block *options) { - lzma_next_strm_init(strm, block_encoder_init, options); + lzma_next_strm_init(lzma_block_encoder_init, strm, options); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_FINISH] = true; diff --git a/src/liblzma/common/block_encoder.h b/src/liblzma/common/block_encoder.h index eafcc618..7bc40139 100644 --- a/src/liblzma/common/block_encoder.h +++ b/src/liblzma/common/block_encoder.h @@ -24,6 +24,6 @@ extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, lzma_options_block *options); + lzma_allocator *allocator, lzma_block *options); #endif diff --git a/src/liblzma/common/block_header_decoder.c b/src/liblzma/common/block_header_decoder.c index b9e072e0..1d75f335 100644 --- a/src/liblzma/common/block_header_decoder.c +++ b/src/liblzma/common/block_header_decoder.c @@ -22,7 +22,7 @@ static void -free_properties(lzma_options_block *options, lzma_allocator *allocator) +free_properties(lzma_block *options, lzma_allocator *allocator) { // Free allocated filter options. The last array member is not // touched after the initialization in the beginning of @@ -38,12 +38,12 @@ free_properties(lzma_options_block *options, lzma_allocator *allocator) extern LZMA_API lzma_ret -lzma_block_header_decode(lzma_options_block *options, +lzma_block_header_decode(lzma_block *options, lzma_allocator *allocator, const uint8_t *in) { // NOTE: We consider the header to be corrupt not only when the // CRC32 doesn't match, but also when variable-length integers - // are invalid or not over 63 bits, or if the header is too small + // are invalid or over 63 bits, or if the header is too small // to contain the claimed information. // Initialize the filter options array. This way the caller can diff --git a/src/liblzma/common/block_header_encoder.c b/src/liblzma/common/block_header_encoder.c index ed0c88ba..3a16e6c3 100644 --- a/src/liblzma/common/block_header_encoder.c +++ b/src/liblzma/common/block_header_encoder.c @@ -22,7 +22,7 @@ extern LZMA_API lzma_ret -lzma_block_header_size(lzma_options_block *options) +lzma_block_header_size(lzma_block *options) { // Block Header Size + Block Flags + CRC32. size_t size = 1 + 1 + 4; @@ -77,7 +77,7 @@ lzma_block_header_size(lzma_options_block *options) extern LZMA_API lzma_ret -lzma_block_header_encode(const lzma_options_block *options, uint8_t *out) +lzma_block_header_encode(const lzma_block *options, uint8_t *out) { if ((options->header_size & 3) || options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN @@ -127,8 +127,9 @@ lzma_block_header_encode(const lzma_options_block *options, uint8_t *out) if (filter_count == 4) return LZMA_PROG_ERROR; - return_if_error(lzma_filter_flags_encode(out, &out_pos, - out_size, options->filters + filter_count)); + return_if_error(lzma_filter_flags_encode( + options->filters + filter_count, + out, &out_pos, out_size)); } while (options->filters[++filter_count].id != LZMA_VLI_VALUE_UNKNOWN); diff --git a/src/liblzma/common/block_util.c b/src/liblzma/common/block_util.c index 6bffc2f1..798163bb 100644 --- a/src/liblzma/common/block_util.c +++ b/src/liblzma/common/block_util.c @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file block_header.c -/// \brief Utility functions to handle lzma_options_block +/// \brief Utility functions to handle lzma_block // // Copyright (C) 2008 Lasse Collin // @@ -21,7 +21,7 @@ extern LZMA_API lzma_ret -lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size) +lzma_block_total_size_set(lzma_block *options, lzma_vli total_size) { // Validate. if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN @@ -32,7 +32,7 @@ lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size) return LZMA_PROG_ERROR; const uint32_t container_size = options->header_size - + lzma_check_sizes[options->check]; + + lzma_check_size(options->check); // Validate that Compressed Size will be greater than zero. if (container_size <= total_size) @@ -45,7 +45,7 @@ lzma_block_total_size_set(lzma_options_block *options, lzma_vli total_size) extern LZMA_API lzma_vli -lzma_block_total_size_get(const lzma_options_block *options) +lzma_block_total_size_get(const lzma_block *options) { // Validate the values that we are interested in. if (options->header_size < LZMA_BLOCK_HEADER_SIZE_MIN @@ -61,7 +61,7 @@ lzma_block_total_size_get(const lzma_options_block *options) const lzma_vli total_size = options->compressed_size + options->header_size - + lzma_check_sizes[options->check]; + + lzma_check_size(options->check); // Validate the calculated Total Size. if (options->compressed_size > LZMA_VLI_VALUE_MAX diff --git a/src/liblzma/common/code.c b/src/liblzma/common/common.c similarity index 64% rename from src/liblzma/common/code.c rename to src/liblzma/common/common.c index 0e3929b6..feac9cbf 100644 --- a/src/liblzma/common/code.c +++ b/src/liblzma/common/common.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file code.c -/// \brief zlib-like API wrapper for liblzma's internal API +/// \file common.h +/// \brief Common functions needed in many places in liblzma // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -20,17 +20,115 @@ #include "common.h" -LZMA_API const lzma_stream LZMA_STREAM_INIT_VAR = { - .next_in = NULL, - .avail_in = 0, - .total_in = 0, - .next_out = NULL, - .avail_out = 0, - .total_out = 0, - .allocator = NULL, - .internal = NULL, -}; +///////////// +// Version // +///////////// +extern LZMA_API uint32_t +lzma_version_number(void) +{ + return LZMA_VERSION; +} + + +extern LZMA_API const char * +lzma_version_string(void) +{ + return PACKAGE_VERSION; +} + + +/////////////////////// +// Memory allocation // +/////////////////////// + +extern void * lzma_attribute((malloc)) +lzma_alloc(size_t size, lzma_allocator *allocator) +{ + // Some malloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) + ptr = allocator->alloc(allocator->opaque, 1, size); + else + ptr = malloc(size); + + return ptr; +} + + +extern void +lzma_free(void *ptr, lzma_allocator *allocator) +{ + if (allocator != NULL && allocator->free != NULL) + allocator->free(allocator->opaque, ptr); + else + free(ptr); + + return; +} + + +////////// +// Misc // +////////// + +extern size_t +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) +{ + const size_t in_avail = in_size - *in_pos; + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(in_avail, out_avail); + + memcpy(out + *out_pos, in + *in_pos, copy_size); + + *in_pos += copy_size; + *out_pos += copy_size; + + return copy_size; +} + + +extern lzma_ret +lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + lzma_next_coder_init(filters[0].init, next, allocator); + + return filters[0].init == NULL + ? LZMA_OK : filters[0].init(next, allocator, filters); +} + + +extern void +lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +{ + if (next->init != (uintptr_t)(NULL)) { + // To avoid tiny end functions that simply call + // lzma_free(coder, allocator), we allow leaving next->end + // NULL and call lzma_free() here. + if (next->end != NULL) + next->end(next->coder, allocator); + else + lzma_free(next->coder, allocator); + + // Reset the variables so the we don't accidentally think + // that it is an already initialized coder. + *next = LZMA_NEXT_CODER_INIT; + } + + return; +} + + +////////////////////////////////////// +// External to internal API wrapper // +////////////////////////////////////// extern lzma_ret lzma_strm_init(lzma_stream *strm) @@ -191,10 +289,7 @@ extern LZMA_API void lzma_end(lzma_stream *strm) { if (strm != NULL && strm->internal != NULL) { - if (strm->internal->next.end != NULL) - strm->internal->next.end(strm->internal->next.coder, - strm->allocator); - + lzma_next_end(&strm->internal->next, strm->allocator); lzma_free(strm->internal, strm->allocator); strm->internal = NULL; } diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h index 4f30427d..81f2a9a4 100644 --- a/src/liblzma/common/common.h +++ b/src/liblzma/common/common.h @@ -3,7 +3,7 @@ /// \file common.h /// \brief Definitions common to the whole liblzma library // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -46,16 +46,32 @@ #define LZMA_BUFFER_SIZE 4096 +/// Start of internal Filter ID space. These IDs must never be used +/// in Streams. +#define LZMA_FILTER_RESERVED_START (LZMA_VLI_C(1) << 62) + + /// Internal helper filter used by Subblock decoder. It is mapped to an /// otherwise invalid Filter ID, which is impossible to get from any input /// file (even if malicious file). -#define LZMA_FILTER_SUBBLOCK_HELPER (UINT64_MAX - 2) +#define LZMA_FILTER_SUBBLOCK_HELPER (LZMA_FILTER_RESERVED_START + 1) + + +/// Supported flags that can be passed to lzma_stream_decoder() +/// or lzma_auto_decoder(). +#define LZMA_SUPPORTED_FLAGS \ + (LZMA_WARN_NO_CHECK \ + | LZMA_WARN_UNSUPPORTED_CHECK \ + | LZMA_TELL_CHECK \ + | LZMA_CONCATENATED) /////////// // Types // /////////// +/// Type of encoder/decoder specific data; the actual structure is defined +/// differently in different coders. typedef struct lzma_coder_s lzma_coder; typedef struct lzma_next_coder_s lzma_next_coder; @@ -63,10 +79,15 @@ typedef struct lzma_next_coder_s lzma_next_coder; typedef struct lzma_filter_info_s lzma_filter_info; +/// Type of a function used to initialize a filter encoder or decoder typedef lzma_ret (*lzma_init_function)( lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); +/// Type of a function to do some kind of coding work (filters, Stream, +/// Block encoders/decoders etc.). Some special coders use don't use both +/// input and output buffers, but for simplicity they still use this same +/// function prototype. typedef lzma_ret (*lzma_code_function)( lzma_coder *coder, lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, @@ -74,54 +95,14 @@ typedef lzma_ret (*lzma_code_function)( size_t *restrict out_pos, size_t out_size, lzma_action action); +/// Type of a function to free the memory allocated for the coder typedef void (*lzma_end_function)( lzma_coder *coder, lzma_allocator *allocator); -/// Hold data and function pointers of the next filter in the chain. -struct lzma_next_coder_s { - /// Pointer to coder-specific data - lzma_coder *coder; - - /// "Pointer" to init function. This is never called here. - /// We need only to detect if we are initializing a coder - /// that was allocated earlier. See code.c and next_coder.c. - uintptr_t init; - - /// Pointer to function to do the actual coding - lzma_code_function code; - - /// Pointer to function to free lzma_next_coder.coder - lzma_end_function end; -}; - -#define LZMA_NEXT_CODER_INIT \ - (lzma_next_coder){ \ - .coder = NULL, \ - .init = 0, \ - .code = NULL, \ - .end = NULL, \ - } - - -struct lzma_internal_s { - lzma_next_coder next; - - enum { - ISEQ_RUN, - ISEQ_SYNC_FLUSH, - ISEQ_FULL_FLUSH, - ISEQ_FINISH, - ISEQ_END, - ISEQ_ERROR, - } sequence; - - bool supported_actions[4]; - bool allow_buf_error; - size_t avail_in; -}; - - +/// Raw coder validates and converts an array of lzma_filter structures to +/// an array of lzma_filter_info structures. This array is used with +/// lzma_next_filter_init to initialize the filter chain. struct lzma_filter_info_s { /// Pointer to function used to initialize the filter. /// This is NULL to indicate end of array. @@ -132,15 +113,76 @@ struct lzma_filter_info_s { }; -/* -typedef struct { - lzma_init_function init; - uint32_t (*input_alignment)(lzma_vli id, const void *options); - uint32_t (*output_alignment)(lzma_vli id, const void *options); - bool changes_uncompressed_size; - bool supports_eopm; -} lzma_filter_hook; -*/ +/// Hold data and function pointers of the next filter in the chain. +struct lzma_next_coder_s { + /// Pointer to coder-specific data + lzma_coder *coder; + + /// "Pointer" to init function. This is never called here. + /// We need only to detect if we are initializing a coder + /// that was allocated earlier. See lzma_next_coder_init and + /// lzma_next_strm_init macros in this file. + uintptr_t init; + + /// Pointer to function to do the actual coding + lzma_code_function code; + + /// Pointer to function to free lzma_next_coder.coder. This can + /// be NULL; in that case, lzma_free is called to free + /// lzma_next_coder.coder. + lzma_end_function end; + + /// Pointer to function to return the type of the integrity check. + /// Most coders won't support this. + lzma_check (*see_check)(const lzma_coder *coder); + +// uint64_t (*memconfig)( +// lzma_coder *coder, uint64_t memlimit, bool change); +}; + + +/// Macro to initialize lzma_next_coder structure +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = (uintptr_t)(NULL), \ + .code = NULL, \ + .end = NULL, \ + .see_check = NULL, \ + } + + +/// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to +/// this is stored in lzma_stream. +struct lzma_internal_s { + /// The actual coder that should do something useful + lzma_next_coder next; + + /// Track the state of the coder. This is used to validate arguments + /// so that the actual coders can rely on e.g. that LZMA_SYNC_FLUSH + /// is used on every call to lzma_code until next.code has returned + /// LZMA_STREAM_END. + enum { + ISEQ_RUN, + ISEQ_SYNC_FLUSH, + ISEQ_FULL_FLUSH, + ISEQ_FINISH, + ISEQ_END, + ISEQ_ERROR, + } sequence; + + /// A copy of lzma_stream avail_in. This is used to verify that the + /// amount of input doesn't change once e.g. LZMA_FINISH has been + /// used. + size_t avail_in; + + /// Indicates which lzma_action values are allowed by next.code. + bool supported_actions[4]; + + /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was + /// made (no input consumed and no output produced by next.code). + bool allow_buf_error; +}; /////////////// @@ -154,126 +196,69 @@ extern void *lzma_alloc(size_t size, lzma_allocator *allocator) /// Frees memory extern void lzma_free(void *ptr, lzma_allocator *allocator); -/// Initializes lzma_stream FIXME desc + +/// Allocates strm->internal if it is NULL, and initializes *strm and +/// strm->internal. This function is only called via lzma_next_strm_init macro. extern lzma_ret lzma_strm_init(lzma_stream *strm); -/// +/// Initializes the next filter in the chain, if any. This takes care of +/// freeing the memory of previously initialized filter if it is different +/// than the filter being initialized now. This way the actual filter +/// initialization functions don't need to use lzma_next_coder_init macro. extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); -/// -extern void lzma_next_coder_end(lzma_next_coder *next, - lzma_allocator *allocator); +/// Frees the memory allocated for next->coder either using next->end or, +/// if next->end is NULL, using lzma_free. +extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); -/// \brief Wrapper for memcpy() -/// -/// This function copies as much data as possible from in[] to out[] and -/// updates *in_pos and *out_pos accordingly. -/// -static inline size_t -bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, - uint8_t *restrict out, size_t *restrict out_pos, - size_t out_size) -{ - const size_t in_avail = in_size - *in_pos; - const size_t out_avail = out_size - *out_pos; - const size_t copy_size = MIN(in_avail, out_avail); - - memcpy(out + *out_pos, in + *in_pos, copy_size); - - *in_pos += copy_size; - *out_pos += copy_size; - - return copy_size; -} - - -/// \brief Initializing the next coder -/// -/// lzma_next_coder can point to different types of coders. The existing -/// coder may be different than what we are initializing now. In that case -/// we must git rid of the old coder first. Otherwise we reuse the existing -/// coder structure. -/// -#define lzma_next_coder_init2(next, allocator, cmpfunc, func, ...) \ -do { \ - if ((uintptr_t)(&cmpfunc) != (next)->init) \ - lzma_next_coder_end(next, allocator); \ - const lzma_ret ret = func(next, __VA_ARGS__); \ - if (ret == LZMA_OK) { \ - (next)->init = (uintptr_t)(&cmpfunc); \ - assert((next)->code != NULL); \ - assert((next)->end != NULL); \ - } else { \ - lzma_next_coder_end(next, allocator); \ - } \ - return ret; \ -} while (0) - -/// \brief Initializing lzma_next_coder -/// -/// Call the initialization function, which must take at least one -/// argument in addition to lzma_next_coder and lzma_allocator. -#define lzma_next_coder_init(func, next, allocator, ...) \ - lzma_next_coder_init2(next, allocator, \ - func, func, allocator, __VA_ARGS__) - -/// \brief Initializing lzma_next_coder -/// -/// Call the initialization function, which takes no other arguments than -/// lzma_next_coder and lzma_allocator. -#define lzma_next_coder_init0(func, next, allocator) \ - lzma_next_coder_init2(next, allocator, func, func, allocator) - - -/// \brief Initializing lzma_stream -/// -/// lzma_strm initialization with more detailed options. -#define lzma_next_strm_init2(strm, cmpfunc, func, ...) \ -do { \ - lzma_ret ret = lzma_strm_init(strm); \ - if (ret != LZMA_OK) \ - return ret; \ - if ((uintptr_t)(&cmpfunc) != (strm)->internal->next.init) \ - lzma_next_coder_end(\ - &(strm)->internal->next, (strm)->allocator); \ - ret = func(&(strm)->internal->next, __VA_ARGS__); \ - if (ret != LZMA_OK) { \ - lzma_end(strm); \ - return ret; \ - } \ - (strm)->internal->next.init = (uintptr_t)(&cmpfunc); \ - assert((strm)->internal->next.code != NULL); \ - assert((strm)->internal->next.end != NULL); \ -} while (0) - -/// \brief Initializing lzma_stream -/// -/// Call the initialization function, which must take at least one -/// argument in addition to lzma_next_coder and lzma_allocator. -#define lzma_next_strm_init(strm, func, ...) \ - lzma_next_strm_init2(strm, func, func, (strm)->allocator, __VA_ARGS__) - -/// \brief Initializing lzma_stream -/// -/// Call the initialization function, which takes no other arguments than -/// lzma_next_coder and lzma_allocator. -#define lzma_next_strm_init0(strm, func) \ - lzma_next_strm_init2(strm, func, func, (strm)->allocator) +/// Copy as much data as possible from in[] to out[] and update *in_pos +/// and *out_pos accordingly. Returns the number of bytes copied. +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); /// \brief Return if expression doesn't evaluate to LZMA_OK /// /// There are several situations where we want to return immediatelly /// with the value of expr if it isn't LZMA_OK. This macro shortens -/// the code a bit. -/// +/// the code a little. #define return_if_error(expr) \ do { \ - const lzma_ret ret_ = expr; \ + const lzma_ret ret_ = (expr); \ if (ret_ != LZMA_OK) \ return ret_; \ } while (0) + +/// If next isn't already initialized, free the previous coder. Then mark +/// that next is _possibly_ initialized for the coder using this macro. +/// "Possibly" means that if e.g. allocation of next->coder fails, the +/// structure isn't actually initialized for this coder, but leaving +/// next->init to func is still OK. +#define lzma_next_coder_init(func, next, allocator) \ +do { \ + if ((uintptr_t)(&func) != (next)->init) \ + lzma_next_end(next, allocator); \ + (next)->init = (uintptr_t)(&func); \ +} while (0) + + +/// Initializes lzma_strm and calls func() to initialize strm->internal->next. +/// (The function being called will use lzma_next_coder_init()). If +/// initialization fails, memory that wasn't freed by func() is freed +/// along strm->internal. +#define lzma_next_strm_init(func, strm, ...) \ +do { \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ + return ret_; \ + } \ +} while (0) + #endif diff --git a/src/liblzma/common/easy.c b/src/liblzma/common/easy.c index 6c258204..ae0e4f74 100644 --- a/src/liblzma/common/easy.c +++ b/src/liblzma/common/easy.c @@ -25,12 +25,12 @@ struct lzma_coder_s { /// We need to keep the filters array available in case /// LZMA_FULL_FLUSH is used. - lzma_options_filter filters[5]; + lzma_filter filters[5]; }; static bool -easy_set_filters(lzma_options_filter *filters, uint32_t level) +easy_set_filters(lzma_filter *filters, uint32_t level) { bool error = false; @@ -38,9 +38,9 @@ easy_set_filters(lzma_options_filter *filters, uint32_t level) // TODO FIXME Use Subblock or LZMA2 with no compression. error = true; -#ifdef HAVE_FILTER_LZMA +#ifdef HAVE_ENCODER_LZMA2 } else if (level <= 9) { - filters[0].id = LZMA_FILTER_LZMA; + filters[0].id = LZMA_FILTER_LZMA2; filters[0].options = (void *)(&lzma_preset_lzma[level - 1]); filters[1].id = LZMA_VLI_VALUE_UNKNOWN; #endif @@ -68,7 +68,7 @@ easy_encode(lzma_coder *coder, lzma_allocator *allocator, static void easy_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->stream_encoder, allocator); + lzma_next_end(&coder->stream_encoder, allocator); lzma_free(coder, allocator); return; } @@ -78,6 +78,8 @@ static lzma_ret easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_easy_level level) { + lzma_next_coder_init(easy_encoder_init, next, allocator); + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -100,7 +102,7 @@ easy_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, extern LZMA_API lzma_ret lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) { - lzma_next_strm_init(strm, easy_encoder_init, level); + lzma_next_strm_init(easy_encoder_init, strm, level); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; @@ -114,9 +116,9 @@ lzma_easy_encoder(lzma_stream *strm, lzma_easy_level level) extern LZMA_API uint32_t lzma_easy_memory_usage(lzma_easy_level level) { - lzma_options_filter filters[5]; + lzma_filter filters[5]; if (easy_set_filters(filters, level)) return UINT32_MAX; - return lzma_memory_usage(filters, true); + return lzma_memusage_encoder(filters); } diff --git a/src/liblzma/common/features.c b/src/liblzma/common/features.c deleted file mode 100644 index a02949d9..00000000 --- a/src/liblzma/common/features.c +++ /dev/null @@ -1,66 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file features.c -/// \brief Information about features enabled at compile time -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - - -static const lzma_vli filters[] = { -#ifdef HAVE_FILTER_SUBBLOCK - LZMA_FILTER_SUBBLOCK, -#endif - -#ifdef HAVE_FILTER_X86 - LZMA_FILTER_X86, -#endif - -#ifdef HAVE_FILTER_POWERPC - LZMA_FILTER_POWERPC, -#endif - -#ifdef HAVE_FILTER_IA64 - LZMA_FILTER_IA64, -#endif - -#ifdef HAVE_FILTER_ARM - LZMA_FILTER_ARM, -#endif - -#ifdef HAVE_FILTER_ARMTHUMB - LZMA_FILTER_ARMTHUMB, -#endif - -#ifdef HAVE_FILTER_SPARC - LZMA_FILTER_SPARC, -#endif - -#ifdef HAVE_FILTER_DELTA - LZMA_FILTER_DELTA, -#endif - -#ifdef HAVE_FILTER_LZMA - LZMA_FILTER_LZMA, -#endif - - LZMA_VLI_VALUE_UNKNOWN -}; - - -LZMA_API const lzma_vli *const lzma_available_filter_encoders = filters; - -LZMA_API const lzma_vli *const lzma_available_filter_decoders = filters; diff --git a/src/liblzma/common/filter_common.c b/src/liblzma/common/filter_common.c new file mode 100644 index 00000000..886ddb53 --- /dev/null +++ b/src/liblzma/common/filter_common.c @@ -0,0 +1,262 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_common.c +/// \brief Filter-specific stuff common for both encoder and decoder +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_common.h" + + +static const struct { + /// Filter ID + lzma_vli id; + + /// True if it is OK to use this filter as non-last filter in + /// the chain. + bool non_last_ok; + + /// True if it is OK to use this filter as the last filter in + /// the chain. + bool last_ok; + + /// True if the filter may change the size of the data (that is, the + /// amount of encoded output can be different than the amount of + /// uncompressed input). + bool changes_size; + +} features[] = { +#if defined (HAVE_ENCODER_LZMA) || defined(HAVE_DECODER_LZMA) + { + .id = LZMA_FILTER_LZMA, + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, +#endif +#ifdef HAVE_DECODER_LZMA2 + { + .id = LZMA_FILTER_LZMA2, + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, +#endif +#if defined(HAVE_ENCODER_SUBBLOCK) || defined(HAVE_DECODER_SUBBLOCK) + { + .id = LZMA_FILTER_SUBBLOCK, + .non_last_ok = true, + .last_ok = true, + .changes_size = true, + }, +#endif +#ifdef HAVE_DECODER_X86 + { + .id = LZMA_FILTER_X86, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) + { + .id = LZMA_FILTER_POWERPC, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#ifdef HAVE_DECODER_IA64 + { + .id = LZMA_FILTER_IA64, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) + { + .id = LZMA_FILTER_ARM, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) + { + .id = LZMA_FILTER_ARMTHUMB, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) + { + .id = LZMA_FILTER_SPARC, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) + { + .id = LZMA_FILTER_DELTA, + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif + { + .id = LZMA_VLI_VALUE_UNKNOWN + } +}; + + +static lzma_ret +validate_chain(const lzma_filter *filters, size_t *count) +{ + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_PROG_ERROR; + + // Number of non-last filters that may change the size of the data + // significantly (that is, more than 1-2 % or so). + size_t changes_size_count = 0; + + // True if it is OK to add a new filter after the current filter. + bool non_last_ok = true; + + // True if the last filter in the given chain is actually usable as + // the last filter. Only filters that support embedding End of Payload + // Marker can be used as the last filter in the chain. + bool last_ok = false; + + size_t i = 0; + do { + size_t j; + for (j = 0; filters[i].id != features[j].id; ++j) + if (features[j].id == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_HEADER_ERROR; + + // If the previous filter in the chain cannot be a non-last + // filter, the chain is invalid. + if (!non_last_ok) + return LZMA_HEADER_ERROR; + + non_last_ok = features[j].non_last_ok; + last_ok = features[j].last_ok; + changes_size_count += features[j].changes_size; + + } while (filters[++i].id != LZMA_VLI_VALUE_UNKNOWN); + + // There must be 1-4 filters. The last filter must be usable as + // the last filter in the chain. At maximum of three filters are + // allowed to change the size of the data. + if (i > LZMA_BLOCK_FILTERS_MAX || !last_ok || changes_size_count > 3) + return LZMA_HEADER_ERROR; + + *count = i; + return LZMA_OK; +} + + +extern lzma_ret +lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options, + lzma_filter_find coder_find, bool is_encoder) +{ + // Do some basic validation and get the number of filters. + size_t count; + return_if_error(validate_chain(options, &count)); + + // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_BLOCK_FILTERS_MAX + 1]; + if (is_encoder) { + for (size_t i = 0; i < count; ++i) { + // The order of the filters is reversed in the + // encoder. It allows more efficient handling + // of the uncompressed data. + const size_t j = count - i - 1; + + const lzma_filter_coder *const fc + = coder_find(options[i].id); + if (fc == NULL || fc->init == NULL) + return LZMA_HEADER_ERROR; + + filters[j].init = fc->init; + filters[j].options = options[i].options; + } + } else { + for (size_t i = 0; i < count; ++i) { + const lzma_filter_coder *const fc + = coder_find(options[i].id); + if (fc == NULL || fc->init == NULL) + return LZMA_HEADER_ERROR; + + filters[i].init = fc->init; + filters[i].options = options[i].options; + } + } + + // Terminate the array. + filters[count].init = NULL; + + // Initialize the filters. + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); + if (ret != LZMA_OK) + lzma_next_end(next, allocator); + + return ret; +} + + +extern uint64_t +lzma_memusage_coder(lzma_filter_find coder_find, + const lzma_filter *filters) +{ + // The chain has to have at least one filter. + if (filters[0].id == LZMA_VLI_VALUE_UNKNOWN) + return UINT64_MAX; + + uint64_t total = 0; + size_t i = 0; + + do { + const lzma_filter_coder *const fc + = coder_find(filters[i].id); + if (fc == NULL) + return UINT64_MAX; // Unsupported Filter ID + + if (fc->memusage == NULL) { + // This filter doesn't have a function to calculate + // the memory usage. Such filters need only little + // memory, so we use 1 KiB as a good estimate. + total += 1024; + } else { + // Call the filter-specific memory usage calculation + // function. + const uint64_t usage + = fc->memusage(filters[i].options); + if (usage == UINT64_MAX) + return UINT64_MAX; // Invalid options + + total += usage; + } + } while (filters[++i].id != LZMA_VLI_VALUE_UNKNOWN); + + // Add some fixed amount of extra. It's to compensate memory usage + // of Stream, Block etc. coders, malloc() overhead, stack etc. + return total + (1U << 15); +} diff --git a/src/liblzma/common/filter_common.h b/src/liblzma/common/filter_common.h new file mode 100644 index 00000000..9def50b9 --- /dev/null +++ b/src/liblzma/common/filter_common.h @@ -0,0 +1,52 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_common.c +/// \brief Filter-specific stuff common for both encoder and decoder +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_COMMON_H +#define LZMA_FILTER_COMMON_H + +#include "common.h" + + +/// Both lzma_filter_encoder and lzma_filter_decoder begin with these members. +typedef struct { + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + +} lzma_filter_coder; + + +typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); + + +extern lzma_ret lzma_raw_coder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, + lzma_filter_find coder_find, bool is_encoder); + + +extern uint64_t lzma_memusage_coder(lzma_filter_find coder_find, + const lzma_filter *filters); + + +#endif diff --git a/src/liblzma/common/filter_decoder.c b/src/liblzma/common/filter_decoder.c new file mode 100644 index 00000000..9fe94f7b --- /dev/null +++ b/src/liblzma/common/filter_decoder.c @@ -0,0 +1,236 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_decoder.h" +#include "filter_common.h" +#include "lzma_decoder.h" +#include "lzma2_decoder.h" +#include "subblock_decoder.h" +#include "subblock_decoder_helper.h" +#include "simple_decoder.h" +#include "delta_decoder.h" + + +typedef struct { + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + + /// Decodes Filter Properties. + /// + /// \return - LZMA_OK: Properties decoded successfully. + /// - LZMA_HEADER_ERROR: Unsupported properties + /// - LZMA_MEM_ERROR: Memory allocation failed. + lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + +} lzma_filter_decoder; + + +static const lzma_vli ids[] = { +#ifdef HAVE_DECODER_LZMA + LZMA_FILTER_LZMA, +#endif + +#ifdef HAVE_DECODER_LZMA2 + LZMA_FILTER_LZMA2, +#endif + +#ifdef HAVE_DECODER_SUBBLOCK + LZMA_FILTER_SUBBLOCK, + LZMA_FILTER_SUBBLOCK_HELPER, +#endif + +#ifdef HAVE_DECODER_X86 + LZMA_FILTER_X86, +#endif + +#ifdef HAVE_DECODER_POWERPC + LZMA_FILTER_POWERPC, +#endif + +#ifdef HAVE_DECODER_IA64 + LZMA_FILTER_IA64, +#endif + +#ifdef HAVE_DECODER_ARM + LZMA_FILTER_ARM, +#endif + +#ifdef HAVE_DECODER_ARMTHUMB + LZMA_FILTER_ARMTHUMB, +#endif + +#ifdef HAVE_DECODER_SPARC + LZMA_FILTER_SPARC, +#endif + +#ifdef HAVE_DECODER_DELTA + LZMA_FILTER_DELTA, +#endif + + LZMA_VLI_VALUE_UNKNOWN +}; + + +// Using a pointer to avoid putting the size of the array to API/ABI. +LZMA_API const lzma_vli *const lzma_filter_decoders = ids; + + +// These must be in the same order as ids[]. +static const lzma_filter_decoder funcs[] = { +#ifdef HAVE_DECODER_LZMA + { + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, + }, +#endif +#ifdef HAVE_DECODER_LZMA2 + { + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, + }, +#endif +#ifdef HAVE_DECODER_SUBBLOCK + { + .init = &lzma_subblock_decoder_init, +// .memusage = &lzma_subblock_decoder_memusage, + .props_decode = NULL, + }, + { + .init = &lzma_subblock_decoder_helper_init, + .memusage = NULL, + .props_decode = NULL, + }, +#endif +#ifdef HAVE_DECODER_X86 + { + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_POWERPC + { + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_IA64 + { + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_ARM + { + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_ARMTHUMB + { + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_SPARC + { + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_DELTA + { + .init = &lzma_delta_decoder_init, + .memusage = NULL, + .props_decode = &lzma_delta_props_decode, + }, +#endif +}; + + +static const lzma_filter_decoder * +decoder_find(lzma_vli id) +{ + for (size_t i = 0; ids[i] != LZMA_VLI_VALUE_UNKNOWN; ++i) + if (ids[i] == id) + return funcs + i; + + return NULL; +} + + +extern lzma_ret +lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options) +{ + return lzma_raw_coder_init(next, allocator, + options, (lzma_filter_find)(&decoder_find), false); +} + + +extern LZMA_API lzma_ret +lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) +{ + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + + return LZMA_OK; +} + + +extern LZMA_API uint64_t +lzma_memusage_decoder(const lzma_filter *filters) +{ + return lzma_memusage_coder( + (lzma_filter_find)(&decoder_find), filters); +} + + +extern LZMA_API lzma_ret +lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + // Make it always NULL so that the caller can always safely free() it. + filter->options = NULL; + + const lzma_filter_decoder *const fd = decoder_find(filter->id); + if (fd == NULL) + return LZMA_HEADER_ERROR; + + if (fd->props_decode == NULL) + return props_size == 0 ? LZMA_OK : LZMA_HEADER_ERROR; + + return fd->props_decode( + &filter->options, allocator, props, props_size); +} diff --git a/src/liblzma/common/filter_decoder.h b/src/liblzma/common/filter_decoder.h new file mode 100644 index 00000000..33e491d1 --- /dev/null +++ b/src/liblzma/common/filter_decoder.h @@ -0,0 +1,35 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_DECODER_H +#define LZMA_FILTER_DECODER_H + +#include "common.h" + +// FIXME !!! Public API +extern lzma_ret lzma_properties_decode( + lzma_filter *filter, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + + +extern lzma_ret lzma_raw_decoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options); + +#endif diff --git a/src/liblzma/common/filter_encoder.c b/src/liblzma/common/filter_encoder.c new file mode 100644 index 00000000..55862e18 --- /dev/null +++ b/src/liblzma/common/filter_encoder.c @@ -0,0 +1,308 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "filter_common.h" +#include "lzma_encoder.h" +#include "lzma2_encoder.h" +#include "subblock_encoder.h" +#include "simple_encoder.h" +#include "delta_encoder.h" + + +typedef struct { + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + + /// Calculates the minimum sane size for Blocks (or other types of + /// chunks) to which the input data can be splitted to make + /// multithreaded encoding possible. If this is NULL, it is assumed + /// that the encoder is fast enough with single thread. + lzma_vli (*chunk_size)(const void *options); + + /// Tells the size of the Filter Properties field. If options are + /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed + /// is used. + lzma_ret (*props_size_get)(uint32_t *size, const void *options); + uint32_t props_size_fixed; + + /// Encodes Filter Properties. + /// + /// \return - LZMA_OK: Properties encoded sucessfully. + /// - LZMA_HEADER_ERROR: Unsupported options + /// - LZMA_PROG_ERROR: Invalid options or not enough + /// output space + lzma_ret (*props_encode)(const void *options, uint8_t *out); + +} lzma_filter_encoder; + + +static const lzma_vli ids[] = { +#ifdef HAVE_ENCODER_LZMA + LZMA_FILTER_LZMA, +#endif + +#ifdef HAVE_ENCODER_LZMA2 + LZMA_FILTER_LZMA2, +#endif + +#ifdef HAVE_ENCODER_SUBBLOCK + LZMA_FILTER_SUBBLOCK, +#endif + +#ifdef HAVE_ENCODER_X86 + LZMA_FILTER_X86, +#endif + +#ifdef HAVE_ENCODER_POWERPC + LZMA_FILTER_POWERPC, +#endif + +#ifdef HAVE_ENCODER_IA64 + LZMA_FILTER_IA64, +#endif + +#ifdef HAVE_ENCODER_ARM + LZMA_FILTER_ARM, +#endif + +#ifdef HAVE_ENCODER_ARMTHUMB + LZMA_FILTER_ARMTHUMB, +#endif + +#ifdef HAVE_ENCODER_SPARC + LZMA_FILTER_SPARC, +#endif + +#ifdef HAVE_ENCODER_DELTA + LZMA_FILTER_DELTA, +#endif + + LZMA_VLI_VALUE_UNKNOWN +}; + + +// Using a pointer to avoid putting the size of the array to API/ABI. +LZMA_API const lzma_vli *const lzma_filter_encoders = ids; + + +// These must be in the same order as ids[]. +static const lzma_filter_encoder funcs[] = { +#ifdef HAVE_ENCODER_LZMA + { + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .chunk_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_LZMA2 + { + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .chunk_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_SUBBLOCK + { + .init = &lzma_subblock_encoder_init, +// .memusage = &lzma_subblock_encoder_memusage, + .chunk_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 0, + .props_encode = NULL, + }, +#endif +#ifdef HAVE_ENCODER_X86 + { + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_POWERPC + { + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_IA64 + { + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_ARM + { + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_ARMTHUMB + { + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_SPARC + { + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_DELTA + { + .init = &lzma_delta_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, + }, +#endif +}; + + +static const lzma_filter_encoder * +encoder_find(lzma_vli id) +{ + for (size_t i = 0; ids[i] != LZMA_VLI_VALUE_UNKNOWN; ++i) + if (ids[i] == id) + return funcs + i; + + return NULL; +} + + +extern lzma_ret +lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options) +{ + return lzma_raw_coder_init(next, allocator, + options, (lzma_filter_find)(&encoder_find), true); +} + + +extern LZMA_API lzma_ret +lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) +{ + lzma_next_strm_init(lzma_raw_coder_init, strm, options, + (lzma_filter_find)(&encoder_find), true); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +extern LZMA_API uint64_t +lzma_memusage_encoder(const lzma_filter *filters) +{ + return lzma_memusage_coder( + (lzma_filter_find)(&encoder_find), filters); +} + + +extern LZMA_API lzma_vli +lzma_chunk_size(const lzma_filter *filters) +{ + uint64_t max = 0; + + for (size_t i = 0; filters[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { + const lzma_filter_encoder *const fe + = encoder_find(filters[i].id); + if (fe->chunk_size != NULL) { + const lzma_vli size + = fe->chunk_size(filters[i].options); + if (size == LZMA_VLI_VALUE_UNKNOWN) + return LZMA_VLI_VALUE_UNKNOWN; + + if (size > max) + max = size; + } + } + + return max; +} + + +extern LZMA_API lzma_ret +lzma_properties_size(uint32_t *size, const lzma_filter *filter) +{ + const lzma_filter_encoder *const fe = encoder_find(filter->id); + if (fe == NULL) { + // Unknown filter - if the Filter ID is a proper VLI, + // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR, + // because it's possible that we just don't have support + // compiled in for the requested filter. + return filter->id <= LZMA_VLI_VALUE_MAX + ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR; + } + + if (fe->props_size_get == NULL) { + // No props_size() function, use props_size_fixed. + *size = fe->props_size_fixed; + return LZMA_OK; + } + + return fe->props_size_get(size, filter->options); +} + + +extern LZMA_API lzma_ret +lzma_properties_encode(const lzma_filter *filter, uint8_t *props) +{ + const lzma_filter_encoder *const fe = encoder_find(filter->id); + if (fe == NULL) + return LZMA_PROG_ERROR; + + if (fe->props_encode == NULL) + return LZMA_OK; + + return fe->props_encode(filter->options, props); +} diff --git a/src/liblzma/common/filter_encoder.h b/src/liblzma/common/filter_encoder.h new file mode 100644 index 00000000..b2bf851d --- /dev/null +++ b/src/liblzma/common/filter_encoder.h @@ -0,0 +1,38 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_encoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_ENCODER_H +#define LZMA_FILTER_ENCODER_H + +#include "common.h" + + +// FIXME !!! Public API +extern lzma_vli lzma_chunk_size(const lzma_filter *filters); +extern lzma_ret lzma_properties_size( + uint32_t *size, const lzma_filter *filter); +extern lzma_ret lzma_properties_encode( + const lzma_filter *filter, uint8_t *props); + + +extern lzma_ret lzma_raw_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options); + +#endif diff --git a/src/liblzma/common/filter_flags_decoder.c b/src/liblzma/common/filter_flags_decoder.c index 498b2ad6..c2247312 100644 --- a/src/liblzma/common/filter_flags_decoder.c +++ b/src/liblzma/common/filter_flags_decoder.c @@ -17,192 +17,37 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "common.h" -#include "lzma_decoder.h" - - -#ifdef HAVE_FILTER_SUBBLOCK -static lzma_ret -properties_subblock(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props lzma_attribute((unused)), - size_t prop_size lzma_attribute((unused))) -{ - if (prop_size != 0) - return LZMA_HEADER_ERROR; - - options->options = lzma_alloc( - sizeof(lzma_options_subblock), allocator); - if (options->options == NULL) - return LZMA_MEM_ERROR; - - ((lzma_options_subblock *)(options->options))->allow_subfilters = true; - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_SIMPLE -static lzma_ret -properties_simple(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props, size_t prop_size) -{ - if (prop_size == 0) - return LZMA_OK; - - if (prop_size != 4) - return LZMA_HEADER_ERROR; - - lzma_options_simple *simple = lzma_alloc( - sizeof(lzma_options_simple), allocator); - if (simple == NULL) - return LZMA_MEM_ERROR; - - simple->start_offset = integer_read_32(props); - - // Don't leave an options structure allocated if start_offset is zero. - if (simple->start_offset == 0) - lzma_free(simple, allocator); - else - options->options = simple; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_DELTA -static lzma_ret -properties_delta(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props, size_t prop_size) -{ - if (prop_size != 1) - return LZMA_HEADER_ERROR; - - options->options = lzma_alloc(sizeof(lzma_options_delta), allocator); - if (options->options == NULL) - return LZMA_MEM_ERROR; - - ((lzma_options_delta *)(options->options))->distance - = (uint32_t)(props[0]) + 1; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_LZMA -static lzma_ret -properties_lzma(lzma_options_filter *options, lzma_allocator *allocator, - const uint8_t *props, size_t prop_size) -{ - // LZMA properties are always two bytes (at least for now). - if (prop_size != 2) - return LZMA_HEADER_ERROR; - - lzma_options_lzma *lzma = lzma_alloc( - sizeof(lzma_options_lzma), allocator); - if (lzma == NULL) - return LZMA_MEM_ERROR; - - // Decode lc, lp, and pb. - if (lzma_lzma_decode_properties(lzma, props[0])) - goto error; - - // Check that reserved bits are unset. - if (props[1] & 0xC0) - goto error; - - // Decode the dictionary size. - // FIXME The specification says that maximum is 4 GiB. - if (props[1] > 36) - goto error; -#if LZMA_DICTIONARY_SIZE_MAX != UINT32_C(1) << 30 -# error Update the if()-condition a few lines -# error above to match LZMA_DICTIONARY_SIZE_MAX. -#endif - - lzma->dictionary_size = 2 | (props[1] & 1); - lzma->dictionary_size <<= props[1] / 2 + 11; - - options->options = lzma; - return LZMA_OK; - -error: - lzma_free(lzma, allocator); - return LZMA_HEADER_ERROR; -} -#endif +#include "filter_decoder.h" extern LZMA_API lzma_ret lzma_filter_flags_decode( - lzma_options_filter *options, lzma_allocator *allocator, + lzma_filter *filter, lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { // Set the pointer to NULL so the caller can always safely free it. - options->options = NULL; + filter->options = NULL; // Filter ID - return_if_error(lzma_vli_decode(&options->id, NULL, + return_if_error(lzma_vli_decode(&filter->id, NULL, in, in_pos, in_size)); - // Size of Properties - lzma_vli prop_size; - return_if_error(lzma_vli_decode(&prop_size, NULL, - in, in_pos, in_size)); - - // Check that we have enough input. - if (prop_size > in_size - *in_pos) + if (filter->id >= LZMA_FILTER_RESERVED_START) return LZMA_DATA_ERROR; - // Determine the function to decode the properties. - lzma_ret (*get_properties)(lzma_options_filter *options, - lzma_allocator *allocator, const uint8_t *props, - size_t prop_size); + // Size of Properties + lzma_vli props_size; + return_if_error(lzma_vli_decode(&props_size, NULL, + in, in_pos, in_size)); - switch (options->id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - get_properties = &properties_subblock; - break; -#endif -#ifdef HAVE_FILTER_SIMPLE -# ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: -# endif -# ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: -# endif -# ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: -# endif -# ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: -# endif -# ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: -# endif -# ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: -# endif - get_properties = &properties_simple; - break; -#endif -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - get_properties = &properties_delta; - break; -#endif -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - get_properties = &properties_lzma; - break; -#endif - default: - return LZMA_HEADER_ERROR; - } + // Filter Properties + if (in_size - *in_pos < props_size) + return LZMA_DATA_ERROR; - const uint8_t *props = in + *in_pos; - *in_pos += prop_size; - return get_properties(options, allocator, props, prop_size); + const lzma_ret ret = lzma_properties_decode( + filter, allocator, in + *in_pos, props_size); + + *in_pos += props_size; + + return ret; } diff --git a/src/liblzma/common/filter_flags_encoder.c b/src/liblzma/common/filter_flags_encoder.c index 45fbbb00..46464c0f 100644 --- a/src/liblzma/common/filter_flags_encoder.c +++ b/src/liblzma/common/filter_flags_encoder.c @@ -17,267 +17,46 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "common.h" -#include "lzma_encoder.h" -#include "fastpos.h" +#include "filter_encoder.h" -/// Calculate the size of the Filter Properties field -static lzma_ret -get_properties_size(uint32_t *size, const lzma_options_filter *options) +extern LZMA_API lzma_ret +lzma_filter_flags_size(uint32_t *size, const lzma_filter *filter) { - lzma_ret ret = LZMA_OK; + return_if_error(lzma_properties_size(size, filter)); - switch (options->id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - *size = 0; - break; -#endif + // lzma_properties_size() validates the Filter ID as a side-effect, + // so we know that it is a valid VLI. + *size += lzma_vli_size(filter->id) + lzma_vli_size(*size); -#ifdef HAVE_FILTER_SIMPLE -# ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: -# endif -# ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: -# endif -# ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: -# endif -# ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: -# endif -# ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: -# endif -# ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: -# endif - if (options->options == NULL || ((const lzma_options_simple *)( - options->options))->start_offset == 0) - *size = 0; - else - *size = 4; - break; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - *size = 1; - break; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - *size = 2; - break; -#endif - - default: - // Unknown filter - if the Filter ID is a proper VLI, - // return LZMA_HEADER_ERROR instead of LZMA_PROG_ERROR, - // because it's possible that we just don't have support - // compiled in for the requested filter. - ret = options->id <= LZMA_VLI_VALUE_MAX - ? LZMA_HEADER_ERROR : LZMA_PROG_ERROR; - break; - } - - return ret; + return LZMA_OK; } extern LZMA_API lzma_ret -lzma_filter_flags_size(uint32_t *size, const lzma_options_filter *options) +lzma_filter_flags_encode(const lzma_filter *filter, + uint8_t *out, size_t *out_pos, size_t out_size) { - // Get size of Filter Properties. This also validates the Filter ID. - uint32_t prop_size; - return_if_error(get_properties_size(&prop_size, options)); - - // Calculate the size of the Filter ID and Size of Properties fields. - // These cannot fail since get_properties_size() already succeeded. - *size = lzma_vli_size(options->id) + lzma_vli_size(prop_size) - + prop_size; - - return LZMA_OK; -} - - -#ifdef HAVE_FILTER_SIMPLE -/// Encodes Filter Properties of the so called simple filters -static lzma_ret -properties_simple(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_simple *options) -{ - if (options == NULL || options->start_offset == 0) - return LZMA_OK; - - if (out_size - *out_pos < 4) - return LZMA_PROG_ERROR; - - integer_write_32(out + *out_pos, options->start_offset); - *out_pos += 4; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_DELTA -/// Encodes Filter Properties of the Delta filter -static lzma_ret -properties_delta(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_delta *options) -{ - if (options == NULL) - return LZMA_PROG_ERROR; - - // It's possible that newer liblzma versions will support larger - // distance values. - if (options->distance < LZMA_DELTA_DISTANCE_MIN - || options->distance > LZMA_DELTA_DISTANCE_MAX) - return LZMA_HEADER_ERROR; - - if (out_size - *out_pos < 1) - return LZMA_PROG_ERROR; - - out[*out_pos] = options->distance - LZMA_DELTA_DISTANCE_MIN; - ++*out_pos; - - return LZMA_OK; -} -#endif - - -#ifdef HAVE_FILTER_LZMA -/// Encodes LZMA Properties and Dictionary Flags (two bytes) -static lzma_ret -properties_lzma(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_lzma *options) -{ - if (options == NULL) - return LZMA_PROG_ERROR; - - if (out_size - *out_pos < 2) - return LZMA_PROG_ERROR; - - // LZMA Properties - if (lzma_lzma_encode_properties(options, out + *out_pos)) - return LZMA_HEADER_ERROR; - - ++*out_pos; - - // Dictionary flags - // - // Dictionary size is encoded using similar encoding that is used - // internally by LZMA. - // - // This won't work if dictionary size can be zero: -# if LZMA_DICTIONARY_SIZE_MIN < 1 -# error LZMA_DICTIONARY_SIZE_MIN cannot be zero. -# endif - - uint32_t d = options->dictionary_size; - - // Validate it: - if (d < LZMA_DICTIONARY_SIZE_MIN || d > LZMA_DICTIONARY_SIZE_MAX) - return LZMA_HEADER_ERROR; - - // Round up to to the next 2^n or 2^n + 2^(n - 1) depending on which - // one is the next: - --d; - d |= d >> 2; - d |= d >> 3; - d |= d >> 4; - d |= d >> 8; - d |= d >> 16; - ++d; - - // Get the highest two bits using the proper encoding: - out[*out_pos] = get_pos_slot(d) - 24; - ++*out_pos; - - return LZMA_OK; -} -#endif - - -extern LZMA_API lzma_ret -lzma_filter_flags_encode(uint8_t *out, size_t *out_pos, size_t out_size, - const lzma_options_filter *options) -{ - // Minimum output is one byte (everything fits into Misc). - // The caller should have checked that there is enough output space, - // so we return LZMA_PROG_ERROR instead of LZMA_BUF_ERROR. - if (*out_pos >= out_size) - return LZMA_PROG_ERROR; - - // Get size of Filter Properties. - uint32_t prop_size; - return_if_error(get_properties_size(&prop_size, options)); - // Filter ID - return_if_error(lzma_vli_encode(options->id, NULL, + if (filter->id >= LZMA_FILTER_RESERVED_START) + return LZMA_HEADER_ERROR; + + return_if_error(lzma_vli_encode(filter->id, NULL, out, out_pos, out_size)); // Size of Properties - return_if_error(lzma_vli_encode(prop_size, NULL, + uint32_t props_size; + return_if_error(lzma_properties_size(&props_size, filter)); + return_if_error(lzma_vli_encode(props_size, NULL, out, out_pos, out_size)); // Filter Properties - lzma_ret ret; - switch (options->id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - assert(prop_size == 0); - ret = LZMA_OK; - break; -#endif + if (out_size - *out_pos < props_size) + return LZMA_PROG_ERROR; -#ifdef HAVE_FILTER_SIMPLE -# ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: -# endif -# ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: -# endif -# ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: -# endif -# ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: -# endif -# ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: -# endif -# ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: -# endif - ret = properties_simple(out, out_pos, out_size, - options->options); - break; -#endif + return_if_error(lzma_properties_encode(filter, out + *out_pos)); -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - ret = properties_delta(out, out_pos, out_size, - options->options); - break; -#endif + *out_pos += props_size; -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - ret = properties_lzma(out, out_pos, out_size, - options->options); - break; -#endif - - default: - assert(0); - ret = LZMA_PROG_ERROR; - break; - } - - return ret; + return LZMA_OK; } diff --git a/src/liblzma/common/index_decoder.c b/src/liblzma/common/index_decoder.c index 1635948c..ae66595a 100644 --- a/src/liblzma/common/index_decoder.c +++ b/src/liblzma/common/index_decoder.c @@ -201,6 +201,8 @@ static lzma_ret index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_index **i) { + lzma_next_coder_init(index_decoder_init, next, allocator); + if (i == NULL) return LZMA_PROG_ERROR; @@ -231,20 +233,10 @@ index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -/* -extern lzma_ret -lzma_index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_index **i) -{ - lzma_next_coder_init(index_decoder_init, next, allocator, i); -} -*/ - - extern LZMA_API lzma_ret lzma_index_decoder(lzma_stream *strm, lzma_index **i) { - lzma_next_strm_init(strm, index_decoder_init, i); + lzma_next_strm_init(index_decoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; diff --git a/src/liblzma/common/index_encoder.c b/src/liblzma/common/index_encoder.c index 5a7d8c8c..3005f835 100644 --- a/src/liblzma/common/index_encoder.c +++ b/src/liblzma/common/index_encoder.c @@ -176,10 +176,12 @@ index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) } -static lzma_ret -index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, +extern lzma_ret +lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, lzma_index *i) { + lzma_next_coder_init(lzma_index_encoder_init, next, allocator); + if (i == NULL) return LZMA_PROG_ERROR; @@ -203,18 +205,10 @@ index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -extern lzma_ret -lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - lzma_index *i) -{ - lzma_next_coder_init(index_encoder_init, next, allocator, i); -} - - extern LZMA_API lzma_ret lzma_index_encoder(lzma_stream *strm, lzma_index *i) { - lzma_next_strm_init(strm, index_encoder_init, i); + lzma_next_strm_init(lzma_index_encoder_init, strm, i); strm->internal->supported_actions[LZMA_RUN] = true; diff --git a/src/liblzma/common/index_hash.c b/src/liblzma/common/index_hash.c index 35dea41f..dc533f9e 100644 --- a/src/liblzma/common/index_hash.c +++ b/src/liblzma/common/index_hash.c @@ -36,7 +36,7 @@ typedef struct { lzma_vli index_list_size; /// Check calculated from Total Sizes and Uncompressed Sizes. - lzma_check check; + lzma_check_state check; } lzma_index_hash_info; @@ -300,9 +300,9 @@ lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, // Finish the hashes and compare them. lzma_check_finish(&index_hash->blocks.check, LZMA_CHECK_BEST); lzma_check_finish(&index_hash->records.check, LZMA_CHECK_BEST); - if (memcmp(index_hash->blocks.check.buffer, - index_hash->records.check.buffer, - lzma_check_sizes[LZMA_CHECK_BEST]) != 0) + if (memcmp(index_hash->blocks.check.buffer.u8, + index_hash->records.check.buffer.u8, + lzma_check_size(LZMA_CHECK_BEST)) != 0) return LZMA_DATA_ERROR; // Finish the CRC32 calculation. diff --git a/src/liblzma/common/init_encoder.c b/src/liblzma/common/init_encoder.c index 8a1644be..c5f12a91 100644 --- a/src/liblzma/common/init_encoder.c +++ b/src/liblzma/common/init_encoder.c @@ -31,7 +31,7 @@ lzma_init_encoder(void) lzma_init_check(); -#if defined(HAVE_SMALL) && defined(HAVE_ENCODER) && defined(HAVE_FILTER_LZMA) +#if defined(HAVE_SMALL) && defined(HAVE_ENCODER_LZMA) lzma_rc_init(); #endif diff --git a/src/liblzma/common/memory_usage.c b/src/liblzma/common/memory_usage.c deleted file mode 100644 index 8244c404..00000000 --- a/src/liblzma/common/memory_usage.c +++ /dev/null @@ -1,112 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file memory_usage.c -/// \brief Calculate rough amount of memory required by filters -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" -#include "lz_encoder.h" -#include "lzma_literal.h" - - -static uint64_t -get_usage(const lzma_options_filter *filter, bool is_encoder) -{ - uint64_t ret; - - switch (filter->id) { - case LZMA_FILTER_X86: - case LZMA_FILTER_POWERPC: - case LZMA_FILTER_IA64: - case LZMA_FILTER_ARM: - case LZMA_FILTER_ARMTHUMB: - case LZMA_FILTER_SPARC: - case LZMA_FILTER_DELTA: - // These don't require any significant amount of memory. - ret = 0; - break; - - case LZMA_FILTER_SUBBLOCK: - if (is_encoder) { - const lzma_options_subblock *options = filter->options; - ret = options->subblock_data_size; - } else { - ret = 0; - } - break; - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: { - const lzma_options_lzma *options = filter->options; - - // Literal coder - this can be signficant if both values are - // big, or if sizeof(probability) is big. - ret = literal_states(options->literal_context_bits, - options->literal_pos_bits) * LIT_SIZE - * sizeof(probability); - - // Dictionary base size - ret += options->dictionary_size; - - if (is_encoder) { -# ifdef HAVE_ENCODER - // This is rough, but should be accurate enough - // in practice. - ret += options->dictionary_size / 2; - - uint32_t dummy1; - uint32_t dummy2; - uint32_t num_items; - if (lzma_lz_encoder_hash_properties( - options->match_finder, - options->dictionary_size, - &dummy1, &dummy2, &num_items)) - return UINT64_MAX; - - ret += (uint64_t)(num_items) * sizeof(uint32_t); -# else - return UINT64_MAX; -# endif - } - - break; - } -#endif - - default: - return UINT64_MAX; - } - - return ret; -} - - -extern LZMA_API uint32_t -lzma_memory_usage(const lzma_options_filter *filters, lzma_bool is_encoder) -{ - uint64_t usage = 0; - - for (size_t i = 0; filters[i].id != UINT64_MAX; ++i) { - const uint64_t ret = get_usage(filters + i, is_encoder); - if (ret == UINT64_MAX) - return UINT32_MAX; - - usage += ret; - } - - // Convert to mebibytes with rounding. - return usage / (1024 * 1024) + (usage % (1024 * 1024) >= 512 ? 1 : 0); -} diff --git a/src/liblzma/common/next_coder.c b/src/liblzma/common/next_coder.c deleted file mode 100644 index c10fe24d..00000000 --- a/src/liblzma/common/next_coder.c +++ /dev/null @@ -1,65 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file next_coder.c -/// \brief Initializing and freeing the next coder in the chain -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "common.h" - -extern lzma_ret -lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter_info *filters) -{ - lzma_ret ret = LZMA_OK; - - // Free the existing coder if it is different than the current one. - if ((uintptr_t)(filters[0].init) != next->init) - lzma_next_coder_end(next, allocator); - - if (filters[0].init != NULL) { - // Initialize the new coder. - ret = filters[0].init(next, allocator, filters); - - // Set the init function pointer if initialization was - // successful. next->code and next->end are set by the - // initialization function itself. - if (ret == LZMA_OK) { - next->init = (uintptr_t)(filters[0].init); - assert(next->code != NULL); - assert(next->end != NULL); - } else { - lzma_next_coder_end(next, allocator); - } - } - - return ret; -} - - -extern void -lzma_next_coder_end(lzma_next_coder *next, lzma_allocator *allocator) -{ - if (next != NULL) { - if (next->end != NULL) - next->end(next->coder, allocator); - - // Reset the variables so the we don't accidentally think - // that it is an already initialized coder. - *next = LZMA_NEXT_CODER_INIT; - } - - return; -} diff --git a/src/liblzma/common/raw_common.c b/src/liblzma/common/raw_common.c deleted file mode 100644 index 35252fc2..00000000 --- a/src/liblzma/common/raw_common.c +++ /dev/null @@ -1,127 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file raw_common.c -/// \brief Stuff shared between raw encoder and raw decoder -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "raw_common.h" - - -static lzma_ret -validate_options(const lzma_options_filter *options, size_t *count) -{ - if (options == NULL) - return LZMA_PROG_ERROR; - - // Number of non-last filters that may change the size of the data - // significantly (that is, more than 1-2 % or so). - size_t change = 0; - - // True if the last filter in the given chain is actually usable as - // the last filter. Only filters that support embedding End of Payload - // Marker can be used as the last filter in the chain. - bool last_ok = false; - - size_t i; - for (i = 0; options[i].id != LZMA_VLI_VALUE_UNKNOWN; ++i) { - switch (options[i].id) { - // Not #ifdeffing these for simplicity. - case LZMA_FILTER_X86: - case LZMA_FILTER_POWERPC: - case LZMA_FILTER_IA64: - case LZMA_FILTER_ARM: - case LZMA_FILTER_ARMTHUMB: - case LZMA_FILTER_SPARC: - case LZMA_FILTER_DELTA: - // These don't change the size of the data and cannot - // be used as the last filter in the chain. - last_ok = false; - break; - -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - last_ok = true; - ++change; - break; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - last_ok = true; - break; -#endif - - default: - return LZMA_HEADER_ERROR; - } - } - - // There must be 1-4 filters and the last filter must be usable as - // the last filter in the chain. - if (i == 0 || i > 4 || !last_ok) - return LZMA_HEADER_ERROR; - - // At maximum of two non-last filters are allowed to change the - // size of the data. - if (change > 2) - return LZMA_HEADER_ERROR; - - *count = i; - return LZMA_OK; -} - - -extern lzma_ret -lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *options, - lzma_init_function (*get_function)(lzma_vli id), - bool is_encoder) -{ - // Do some basic validation and get the number of filters. - size_t count; - return_if_error(validate_options(options, &count)); - - // Set the filter functions and copy the options pointer. - lzma_filter_info filters[count + 1]; - if (is_encoder) { - for (size_t i = 0; i < count; ++i) { - // The order of the filters is reversed in the - // encoder. It allows more efficient handling - // of the uncompressed data. - const size_t j = count - i - 1; - - filters[j].init = get_function(options[i].id); - if (filters[j].init == NULL) - return LZMA_HEADER_ERROR; - - filters[j].options = options[i].options; - } - } else { - for (size_t i = 0; i < count; ++i) { - filters[i].init = get_function(options[i].id); - if (filters[i].init == NULL) - return LZMA_HEADER_ERROR; - - filters[i].options = options[i].options; - } - } - - // Terminate the array. - filters[count].init = NULL; - - // Initialize the filters. - return lzma_next_filter_init(next, allocator, filters); -} diff --git a/src/liblzma/common/raw_decoder.c b/src/liblzma/common/raw_decoder.c deleted file mode 100644 index 4fb7111c..00000000 --- a/src/liblzma/common/raw_decoder.c +++ /dev/null @@ -1,116 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file raw_decoder.c -/// \brief Raw decoder initialization API -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "raw_decoder.h" -#include "simple_coder.h" -#include "subblock_decoder.h" -#include "subblock_decoder_helper.h" -#include "delta_decoder.h" -#include "lzma_decoder.h" - - -static lzma_init_function -get_function(lzma_vli id) -{ - switch (id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - return &lzma_subblock_decoder_init; -#endif - -#ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: - return &lzma_simple_x86_decoder_init; -#endif - -#ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: - return &lzma_simple_powerpc_decoder_init; -#endif - -#ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: - return &lzma_simple_ia64_decoder_init; -#endif - -#ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: - return &lzma_simple_arm_decoder_init; -#endif - -#ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: - return &lzma_simple_armthumb_decoder_init; -#endif - -#ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: - return &lzma_simple_sparc_decoder_init; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - return &lzma_delta_decoder_init; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - return &lzma_lzma_decoder_init; -#endif - -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK_HELPER: - return &lzma_subblock_decoder_helper_init; -#endif - } - - return NULL; -} - - -extern lzma_ret -lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *options) -{ - const lzma_ret ret = lzma_raw_coder_init(next, allocator, - options, &get_function, false); - - if (ret != LZMA_OK) - lzma_next_coder_end(next, allocator); - - return ret; -} - - -extern LZMA_API lzma_ret -lzma_raw_decoder(lzma_stream *strm, const lzma_options_filter *options) -{ - return_if_error(lzma_strm_init(strm)); - - strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; - - const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next, - strm->allocator, options, &get_function, false); - - if (ret != LZMA_OK) - lzma_end(strm); - - return ret; -} diff --git a/src/liblzma/common/raw_encoder.c b/src/liblzma/common/raw_encoder.c deleted file mode 100644 index 9b8cbfae..00000000 --- a/src/liblzma/common/raw_encoder.c +++ /dev/null @@ -1,111 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file raw_encoder.c -/// \brief Raw encoder initialization API -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "raw_encoder.h" -#include "simple_coder.h" -#include "subblock_encoder.h" -#include "delta_encoder.h" -#include "lzma_encoder.h" - - -static lzma_init_function -get_function(lzma_vli id) -{ - switch (id) { -#ifdef HAVE_FILTER_SUBBLOCK - case LZMA_FILTER_SUBBLOCK: - return &lzma_subblock_encoder_init; -#endif - -#ifdef HAVE_FILTER_X86 - case LZMA_FILTER_X86: - return &lzma_simple_x86_encoder_init; -#endif - -#ifdef HAVE_FILTER_POWERPC - case LZMA_FILTER_POWERPC: - return &lzma_simple_powerpc_encoder_init; -#endif - -#ifdef HAVE_FILTER_IA64 - case LZMA_FILTER_IA64: - return &lzma_simple_ia64_encoder_init; -#endif - -#ifdef HAVE_FILTER_ARM - case LZMA_FILTER_ARM: - return &lzma_simple_arm_encoder_init; -#endif - -#ifdef HAVE_FILTER_ARMTHUMB - case LZMA_FILTER_ARMTHUMB: - return &lzma_simple_armthumb_encoder_init; -#endif - -#ifdef HAVE_FILTER_SPARC - case LZMA_FILTER_SPARC: - return &lzma_simple_sparc_encoder_init; -#endif - -#ifdef HAVE_FILTER_DELTA - case LZMA_FILTER_DELTA: - return &lzma_delta_encoder_init; -#endif - -#ifdef HAVE_FILTER_LZMA - case LZMA_FILTER_LZMA: - return &lzma_lzma_encoder_init; -#endif - } - - return NULL; -} - - -extern lzma_ret -lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *options) -{ - const lzma_ret ret = lzma_raw_coder_init(next, allocator, - options, &get_function, true); - - if (ret != LZMA_OK) - lzma_next_coder_end(next, allocator); - - return ret; -} - - -extern LZMA_API lzma_ret -lzma_raw_encoder(lzma_stream *strm, const lzma_options_filter *options) -{ - return_if_error(lzma_strm_init(strm)); - - strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; - strm->internal->supported_actions[LZMA_FINISH] = true; - - const lzma_ret ret = lzma_raw_coder_init(&strm->internal->next, - strm->allocator, options, &get_function, true); - - if (ret != LZMA_OK) - lzma_end(strm); - - return ret; -} diff --git a/src/liblzma/common/stream_common.c b/src/liblzma/common/stream_common.c deleted file mode 100644 index 121a6674..00000000 --- a/src/liblzma/common/stream_common.c +++ /dev/null @@ -1,23 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file stream_common.c -/// \brief Common stuff for Stream coders -// -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "stream_common.h" - -const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 }; -const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A }; diff --git a/src/liblzma/common/stream_decoder.c b/src/liblzma/common/stream_decoder.c index 1bf7f1f8..5b46819d 100644 --- a/src/liblzma/common/stream_decoder.c +++ b/src/liblzma/common/stream_decoder.c @@ -17,8 +17,8 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" #include "stream_decoder.h" +#include "stream_flags_common.h" #include "check.h" #include "stream_flags_decoder.h" #include "block_decoder.h" @@ -31,6 +31,7 @@ struct lzma_coder_s { SEQ_BLOCK, SEQ_INDEX, SEQ_STREAM_FOOTER, + SEQ_STREAM_PADDING, } sequence; /// Block or Metadata decoder. This takes little memory and the same @@ -40,7 +41,7 @@ struct lzma_coder_s { /// Block options decoded by the Block Header decoder and used by /// the Block decoder. - lzma_options_block block_options; + lzma_block block_options; /// Stream Flags from Stream Header lzma_stream_flags stream_flags; @@ -49,8 +50,35 @@ struct lzma_coder_s { /// with O(1) memory usage. lzma_index_hash *index_hash; - /// Write position in buffer[] - size_t buffer_pos; + /// Memory usage limit + uint64_t memlimit; + + /// If true, LZMA_NO_CHECK is returned if the Stream has + /// no integrity check. + bool warn_no_check; + + /// If true, LZMA_UNSUPPORTED_CHECK is returned if the Stream has + /// an integrity check that isn't supported by this liblzma build. + bool warn_unsupported_check; + + /// If true, LZMA_SEE_CHECK is returned after decoding Stream Header. + bool tell_check; + + /// If true, we will decode concatenated Streams that possibly have + /// Stream Padding between or after them. LZMA_STREAM_END is returned + /// once the application isn't giving us any new input and we aren't + /// in the middle of a Stream and possible Stream Padding is a + /// multiple of four bytes. FIXME + bool concatenated; + + /// When decoding concatenated Streams, this is true as long as we + /// are decoding the first Stream. This is needed to avoid misleading + /// LZMA_FORMAT_ERROR in case the later Streams don't have valid magic + /// bytes. + bool first_stream; + + /// Write position in buffer[] and position in Stream Padding + size_t pos; /// Buffer to hold Stream Header, Block Header, and Stream Footer. /// Block Header has biggest maximum size. @@ -58,6 +86,23 @@ struct lzma_coder_s { }; +static lzma_ret +stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +{ + // Initialize the Index hash used to verify the Index. + coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); + if (coder->index_hash == NULL) + return LZMA_MEM_ERROR; + + // Reset the rest of the variables. + coder->sequence = SEQ_STREAM_HEADER; + coder->block_options.filters = NULL; + coder->pos = 0; + + return LZMA_OK; +} + + static lzma_ret stream_decode(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, @@ -66,43 +111,56 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, { // When decoding the actual Block, it may be able to produce more // output even if we don't give it any new input. - while (*out_pos < out_size && (*in_pos < in_size - || coder->sequence == SEQ_BLOCK)) + while (true) switch (coder->sequence) { case SEQ_STREAM_HEADER: { // Copy the Stream Header to the internal buffer. - bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos, + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); // Return if we didn't get the whole Stream Header yet. - if (coder->buffer_pos < LZMA_STREAM_HEADER_SIZE) + if (coder->pos < LZMA_STREAM_HEADER_SIZE) return LZMA_OK; - coder->buffer_pos = 0; + coder->pos = 0; // Decode the Stream Header. - return_if_error(lzma_stream_header_decode( - &coder->stream_flags, coder->buffer)); + const lzma_ret ret = lzma_stream_header_decode( + &coder->stream_flags, coder->buffer); + if (ret != LZMA_OK) + return ret == LZMA_FORMAT_ERROR && !coder->first_stream + ? LZMA_DATA_ERROR : ret; // Copy the type of the Check so that Block Header and Block // decoders see it. coder->block_options.check = coder->stream_flags.check; - // Even if we return LZMA_UNSUPPORTED_CHECK below, we want + // Even if we return LZMA_*_CHECK below, we want // to continue from Block Header decoding. coder->sequence = SEQ_BLOCK_HEADER; - // Detect if the Check type is supported and give appropriate - // warning if it isn't. We don't warn every time a new Block - // is started. - if (!lzma_available_checks[coder->block_options.check]) + // Detect if there's no integrity check or if it is + // unsupported if those were requested by the application. + if (coder->warn_no_check && coder->stream_flags.check + == LZMA_CHECK_NONE) + return LZMA_NO_CHECK; + + if (coder->warn_unsupported_check + && !lzma_check_is_supported( + coder->stream_flags.check)) return LZMA_UNSUPPORTED_CHECK; + if (coder->tell_check) + return LZMA_SEE_CHECK; + break; } case SEQ_BLOCK_HEADER: { - if (coder->buffer_pos == 0) { + if (*in_pos >= in_size) + return LZMA_OK; + + if (coder->pos == 0) { // Detect if it's Index. if (in[*in_pos] == 0x00) { coder->sequence = SEQ_INDEX; @@ -118,29 +176,41 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } // Copy the Block Header to the internal buffer. - bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos, + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, coder->block_options.header_size); // Return if we didn't get the whole Block Header yet. - if (coder->buffer_pos < coder->block_options.header_size) + if (coder->pos < coder->block_options.header_size) return LZMA_OK; - coder->buffer_pos = 0; + coder->pos = 0; // Set up a buffer to hold the filter chain. Block Header // decoder will initialize all members of this array so // we don't need to do it here. - lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; + lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; coder->block_options.filters = filters; // Decode the Block Header. return_if_error(lzma_block_header_decode(&coder->block_options, allocator, coder->buffer)); - // Initialize the Block decoder. - const lzma_ret ret = lzma_block_decoder_init( - &coder->block_decoder, - allocator, &coder->block_options); + // Check the memory usage limit. + const uint64_t memusage = lzma_memusage_decoder(filters); + lzma_ret ret; + + if (memusage == UINT64_MAX) { + // One or more unknown Filter IDs. + ret = LZMA_HEADER_ERROR; + } else if (memusage > coder->memlimit) { + // The chain would need too much memory. + ret = LZMA_MEMLIMIT_ERROR; + } else { + // Memory usage is OK. Initialize the Block decoder. + ret = lzma_block_decoder_init( + &coder->block_decoder, + allocator, &coder->block_options); + } // Free the allocated filter options since they are needed // only to initialize the Block decoder. @@ -149,10 +219,9 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, coder->block_options.filters = NULL; - // Check if Block enocoder initialization succeeded. Don't - // warn about unsupported check anymore since we did it - // earlier if it was needed. - if (ret != LZMA_OK && ret != LZMA_UNSUPPORTED_CHECK) + // Check if memory usage calculation and Block enocoder + // initialization succeeded. + if (ret != LZMA_OK) return ret; coder->sequence = SEQ_BLOCK; @@ -160,7 +229,7 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_BLOCK: { - lzma_ret ret = coder->block_decoder.code( + const lzma_ret ret = coder->block_decoder.code( coder->block_decoder.coder, allocator, in, in_pos, in_size, out, out_pos, out_size, action); @@ -180,6 +249,12 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, } case SEQ_INDEX: { + // If we don't have any input, don't call + // lzma_index_hash_decode() since it would return + // LZMA_BUF_ERROR, which we must not do here. + if (*in_pos >= in_size) + return LZMA_OK; + // Decode the Index and compare it to the hash calculated // from the sizes of the Blocks (if any). const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, @@ -193,14 +268,17 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, case SEQ_STREAM_FOOTER: // Copy the Stream Footer to the internal buffer. - bufcpy(in, in_pos, in_size, coder->buffer, &coder->buffer_pos, + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, LZMA_STREAM_HEADER_SIZE); // Return if we didn't get the whole Stream Footer yet. - if (coder->buffer_pos < LZMA_STREAM_HEADER_SIZE) + if (coder->pos < LZMA_STREAM_HEADER_SIZE) return LZMA_OK; + coder->pos = 0; + // Decode the Stream Footer. + // FIXME LZMA_FORMAT_ERROR doesn't make sense here. lzma_stream_flags footer_flags; return_if_error(lzma_stream_footer_decode( &footer_flags, coder->buffer)); @@ -217,7 +295,48 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, &footer_flags)) return LZMA_DATA_ERROR; - return LZMA_STREAM_END; + if (!coder->concatenated) + return LZMA_STREAM_END; + + coder->sequence = SEQ_STREAM_PADDING; + break; + + case SEQ_STREAM_PADDING: + assert(coder->concatenated); + + while (true) { + if (*in_pos >= in_size) { + // Unless LZMA_FINISH was used, we cannot + // know if there's more input coming later. + if (action != LZMA_FINISH) + return LZMA_OK; + + // Stream Padding must be a multiple of + // four bytes. + return coder->pos == 0 + ? LZMA_STREAM_END + : LZMA_DATA_ERROR; + } + + if (in[*in_pos] != 0x00) { + if (coder->pos != 0) { + // Stream Padding is not a multiple of + // four bytes. + ++*in_pos; + return LZMA_DATA_ERROR; + } + + // Prepare to decode the next Stream. + return_if_error(stream_decoder_reset( + coder, allocator)); + break; + } + + ++*in_pos; + coder->pos = (coder->pos + 1) & 3; + } + + break; default: assert(0); @@ -231,16 +350,29 @@ stream_decode(lzma_coder *coder, lzma_allocator *allocator, static void stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->block_decoder, allocator); + lzma_next_end(&coder->block_decoder, allocator); lzma_index_hash_end(coder->index_hash, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) +static lzma_check +stream_decoder_see_check(const lzma_coder *coder) { + return coder->stream_flags.check; +} + + +extern lzma_ret +lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) +{ + lzma_next_coder_init(lzma_stream_decoder_init, next, allocator); + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_HEADER_ERROR; + if (next->coder == NULL) { next->coder = lzma_alloc(sizeof(lzma_coder), allocator); if (next->coder == NULL) @@ -248,40 +380,32 @@ stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) next->code = &stream_decode; next->end = &stream_decoder_end; + next->see_check = &stream_decoder_see_check; next->coder->block_decoder = LZMA_NEXT_CODER_INIT; next->coder->index_hash = NULL; } - // Initialize the Index hash used to verify the Index. - next->coder->index_hash = lzma_index_hash_init( - next->coder->index_hash, allocator); - if (next->coder->index_hash == NULL) - return LZMA_MEM_ERROR; + next->coder->memlimit = memlimit; + next->coder->warn_no_check = (flags & LZMA_WARN_NO_CHECK) != 0; + next->coder->warn_unsupported_check + = (flags & LZMA_WARN_UNSUPPORTED_CHECK) != 0; + next->coder->tell_check = (flags & LZMA_TELL_CHECK) != 0; + next->coder->concatenated + = (flags & LZMA_CONCATENATED) != 0; - // Reset the rest of the variables. - next->coder->sequence = SEQ_STREAM_HEADER; - next->coder->block_options.filters = NULL; - next->coder->buffer_pos = 0; - - return LZMA_OK; -} - - -extern lzma_ret -lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator) -{ - lzma_next_coder_init0(stream_decoder_init, next, allocator); + return stream_decoder_reset(next->coder, allocator); } extern LZMA_API lzma_ret -lzma_stream_decoder(lzma_stream *strm) +lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) { - lzma_next_strm_init0(strm, stream_decoder_init); + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); strm->internal->supported_actions[LZMA_RUN] = true; - strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; +// strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; // FIXME + strm->internal->supported_actions[LZMA_FINISH] = true; return LZMA_OK; } diff --git a/src/liblzma/common/stream_decoder.h b/src/liblzma/common/stream_decoder.h index dcda387d..59d58c6f 100644 --- a/src/liblzma/common/stream_decoder.h +++ b/src/liblzma/common/stream_decoder.h @@ -22,7 +22,7 @@ #include "common.h" -extern lzma_ret lzma_stream_decoder_init( - lzma_next_coder *next, lzma_allocator *allocator); +extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); #endif diff --git a/src/liblzma/common/stream_encoder.c b/src/liblzma/common/stream_encoder.c index 767b8014..9d56c899 100644 --- a/src/liblzma/common/stream_encoder.c +++ b/src/liblzma/common/stream_encoder.c @@ -17,8 +17,8 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" #include "stream_encoder.h" +#include "stream_flags_common.h" #include "block_encoder.h" #include "index_encoder.h" @@ -37,7 +37,7 @@ struct lzma_coder_s { lzma_next_coder block_encoder; /// Options for the Block encoder - lzma_options_block block_options; + lzma_block block_options; /// Index encoder. This is separate from Block encoder, because this /// doesn't take much memory, and when encoding multiple Streams @@ -86,8 +86,8 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, case SEQ_STREAM_HEADER: case SEQ_BLOCK_HEADER: case SEQ_STREAM_FOOTER: - bufcpy(coder->buffer, &coder->buffer_pos, coder->buffer_size, - out, out_pos, out_size); + lzma_bufcpy(coder->buffer, &coder->buffer_pos, + coder->buffer_size, out, out_pos, out_size); if (coder->buffer_pos < coder->buffer_size) return LZMA_OK; @@ -202,18 +202,20 @@ stream_encode(lzma_coder *coder, lzma_allocator *allocator, static void stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->block_encoder, allocator); - lzma_next_coder_end(&coder->index_encoder, allocator); + lzma_next_end(&coder->block_encoder, allocator); + lzma_next_end(&coder->index_encoder, allocator); lzma_index_end(coder->index, allocator); lzma_free(coder, allocator); return; } -static lzma_ret -stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *filters, lzma_check_type check) +extern lzma_ret +lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, lzma_check check) { + lzma_next_coder_init(lzma_stream_encoder_init, next, allocator); + if (filters == NULL) return LZMA_PROG_ERROR; @@ -233,7 +235,7 @@ stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, // Basic initializations next->coder->sequence = SEQ_STREAM_HEADER; next->coder->block_options.check = check; - next->coder->block_options.filters = (lzma_options_filter *)(filters); + next->coder->block_options.filters = (lzma_filter *)(filters); // Initialize the Index next->coder->index = lzma_index_init(next->coder->index, allocator); @@ -258,20 +260,11 @@ stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, } -extern lzma_ret -lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *filters, lzma_check_type check) -{ - lzma_next_coder_init(stream_encoder_init, next, allocator, - filters, check); -} - - extern LZMA_API lzma_ret lzma_stream_encoder(lzma_stream *strm, - const lzma_options_filter *filters, lzma_check_type check) + const lzma_filter *filters, lzma_check check) { - lzma_next_strm_init(strm, stream_encoder_init, filters, check); + lzma_next_strm_init(lzma_stream_encoder_init, strm, filters, check); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; diff --git a/src/liblzma/common/stream_encoder.h b/src/liblzma/common/stream_encoder.h index 3ce29561..cec2e5b5 100644 --- a/src/liblzma/common/stream_encoder.h +++ b/src/liblzma/common/stream_encoder.h @@ -25,6 +25,6 @@ extern lzma_ret lzma_stream_encoder_init( lzma_next_coder *next, lzma_allocator *allocator, - const lzma_options_filter *filters, lzma_check_type check); + const lzma_filter *filters, lzma_check check); #endif diff --git a/src/liblzma/common/stream_flags_equal.c b/src/liblzma/common/stream_flags_common.c similarity index 72% rename from src/liblzma/common/stream_flags_equal.c rename to src/liblzma/common/stream_flags_common.c index db22567f..c44b3ff2 100644 --- a/src/liblzma/common/stream_flags_equal.c +++ b/src/liblzma/common/stream_flags_common.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_flags_equal.c -/// \brief Compare Stream Header and Stream Footer +/// \file stream_flags_common.c +/// \brief Common stuff for Stream flags coders // -// Copyright (C) 2008 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,11 +17,15 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "common.h" +#include "stream_flags_common.h" + + +const uint8_t lzma_header_magic[6] = { 0xFF, 0x4C, 0x5A, 0x4D, 0x41, 0x00 }; +const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A }; extern LZMA_API lzma_bool -lzma_stream_flags_equal(const lzma_stream_flags *a, lzma_stream_flags *b) +lzma_stream_flags_equal(const lzma_stream_flags *a, const lzma_stream_flags *b) { if (a->check != b->check) return false; diff --git a/src/liblzma/common/stream_common.h b/src/liblzma/common/stream_flags_common.h similarity index 85% rename from src/liblzma/common/stream_common.h rename to src/liblzma/common/stream_flags_common.h index 4f83fc58..6e57857b 100644 --- a/src/liblzma/common/stream_common.h +++ b/src/liblzma/common/stream_flags_common.h @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file stream_common.h -/// \brief Common stuff for Stream coders +/// \file stream_flags_common.h +/// \brief Common stuff for Stream flags coders // // Copyright (C) 2007 Lasse Collin // @@ -17,8 +17,8 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_STREAM_COMMON_H -#define LZMA_STREAM_COMMON_H +#ifndef LZMA_STREAM_FLAGS_COMMON_H +#define LZMA_STREAM_FLAGS_COMMON_H #include "common.h" diff --git a/src/liblzma/common/stream_flags_decoder.c b/src/liblzma/common/stream_flags_decoder.c index 0270875a..ccc1539d 100644 --- a/src/liblzma/common/stream_flags_decoder.c +++ b/src/liblzma/common/stream_flags_decoder.c @@ -17,7 +17,7 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" +#include "stream_flags_common.h" static bool diff --git a/src/liblzma/common/stream_flags_encoder.c b/src/liblzma/common/stream_flags_encoder.c index 4efbb6f4..1d736a8a 100644 --- a/src/liblzma/common/stream_flags_encoder.c +++ b/src/liblzma/common/stream_flags_encoder.c @@ -17,7 +17,7 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "stream_common.h" +#include "stream_flags_common.h" static bool diff --git a/src/liblzma/common/vli_decoder.c b/src/liblzma/common/vli_decoder.c index faff6ccb..60874baa 100644 --- a/src/liblzma/common/vli_decoder.c +++ b/src/liblzma/common/vli_decoder.c @@ -27,17 +27,30 @@ lzma_vli_decode(lzma_vli *restrict vli, size_t *restrict vli_pos, { // If we haven't been given vli_pos, work in single-call mode. size_t vli_pos_internal = 0; - if (vli_pos == NULL) + if (vli_pos == NULL) { vli_pos = &vli_pos_internal; - - // Initialize *vli when starting to decode a new integer. - if (*vli_pos == 0) *vli = 0; - // Validate the arguments. - if (*vli_pos >= LZMA_VLI_BYTES_MAX || *in_pos >= in_size - || (*vli >> (*vli_pos * 7)) != 0) - return LZMA_PROG_ERROR;; + // If there's no input, use LZMA_DATA_ERROR. This way it is + // easy to decode VLIs from buffers that have known size, + // and get the correct error code in case the buffer is + // too short. + if (*in_pos >= in_size) + return LZMA_DATA_ERROR; + + } else { + // Initialize *vli when starting to decode a new integer. + if (*vli_pos == 0) + *vli = 0; + + // Validate the arguments. + if (*vli_pos >= LZMA_VLI_BYTES_MAX + || (*vli >> (*vli_pos * 7)) != 0) + return LZMA_PROG_ERROR;; + + if (*in_pos >= in_size) + return LZMA_BUF_ERROR; + } do { // Read the next byte. diff --git a/src/liblzma/common/vli_encoder.c b/src/liblzma/common/vli_encoder.c index c48d6474..53022f16 100644 --- a/src/liblzma/common/vli_encoder.c +++ b/src/liblzma/common/vli_encoder.c @@ -31,10 +31,12 @@ lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos, vli_pos = &vli_pos_internal; // Validate the arguments. - if (*vli_pos >= LZMA_VLI_BYTES_MAX || *out_pos >= out_size - || vli > LZMA_VLI_VALUE_MAX) + if (*vli_pos >= LZMA_VLI_BYTES_MAX || vli > LZMA_VLI_VALUE_MAX) return LZMA_PROG_ERROR; + if (*out_pos >= out_size) + return LZMA_BUF_ERROR; + // Write the non-last bytes in a loop. while ((vli >> (*vli_pos * 7)) >= 0x80) { out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7)) | 0x80; @@ -55,20 +57,3 @@ lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos, return vli_pos == &vli_pos_internal ? LZMA_OK : LZMA_STREAM_END; } - - -extern LZMA_API uint32_t -lzma_vli_size(lzma_vli vli) -{ - if (vli > LZMA_VLI_VALUE_MAX) - return 0; - - uint32_t i = 0; - do { - vli >>= 7; - ++i; - } while (vli != 0); - - assert(i <= LZMA_VLI_BYTES_MAX); - return i; -} diff --git a/src/liblzma/common/version.c b/src/liblzma/common/vli_size.c similarity index 67% rename from src/liblzma/common/version.c rename to src/liblzma/common/vli_size.c index dffec7ff..547bba0b 100644 --- a/src/liblzma/common/version.c +++ b/src/liblzma/common/vli_size.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file version.c -/// \brief liblzma version number +/// \file vli_size.c +/// \brief Calculates the encoded size of a variable-length integer // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -20,6 +20,18 @@ #include "common.h" -LZMA_API const uint32_t lzma_version_number = LZMA_VERSION; +extern LZMA_API uint32_t +lzma_vli_size(lzma_vli vli) +{ + if (vli > LZMA_VLI_VALUE_MAX) + return 0; -LZMA_API const char *const lzma_version_string = PACKAGE_VERSION; + uint32_t i = 0; + do { + vli >>= 7; + ++i; + } while (vli != 0); + + assert(i <= LZMA_VLI_BYTES_MAX); + return i; +} diff --git a/src/liblzma/delta/Makefile.am b/src/liblzma/delta/Makefile.am new file mode 100644 index 00000000..fc09f5b8 --- /dev/null +++ b/src/liblzma/delta/Makefile.am @@ -0,0 +1,34 @@ +## +## Copyright (C) 2008 Lasse Collin +## +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## + +noinst_LTLIBRARIES = libdelta.la +libdelta_la_CPPFLAGS = \ + -I@top_srcdir@/src/liblzma/api \ + -I@top_srcdir@/src/liblzma/common + +libdelta_la_SOURCES = \ + delta_common.c \ + delta_common.h + +if COND_ENCODER_DELTA +libdelta_la_SOURCES += \ + delta_encoder.c \ + delta_encoder.h +endif + +if COND_DECODER_DELTA +libdelta_la_SOURCES += \ + delta_decoder.c \ + delta_decoder.h +endif diff --git a/src/liblzma/common/delta_common.c b/src/liblzma/delta/delta_common.c similarity index 97% rename from src/liblzma/common/delta_common.c rename to src/liblzma/delta/delta_common.c index acd31e14..d40e0c7f 100644 --- a/src/liblzma/common/delta_common.c +++ b/src/liblzma/delta/delta_common.c @@ -23,7 +23,7 @@ static void delta_coder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder, allocator); return; } diff --git a/src/liblzma/common/delta_common.h b/src/liblzma/delta/delta_common.h similarity index 100% rename from src/liblzma/common/delta_common.h rename to src/liblzma/delta/delta_common.h diff --git a/src/liblzma/common/delta_decoder.c b/src/liblzma/delta/delta_decoder.c similarity index 81% rename from src/liblzma/common/delta_decoder.c rename to src/liblzma/delta/delta_decoder.c index 8f5a4cbf..ee22ba02 100644 --- a/src/liblzma/common/delta_decoder.c +++ b/src/liblzma/delta/delta_decoder.c @@ -59,3 +59,24 @@ lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, { return lzma_delta_coder_init(next, allocator, filters, &delta_decode); } + + +extern lzma_ret +lzma_delta_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size != 1) + return LZMA_HEADER_ERROR; + + lzma_options_delta *opt + = lzma_alloc(sizeof(lzma_options_delta), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + opt->type = LZMA_DELTA_TYPE_BYTE; + opt->distance = props[0] + 1; + + *options = opt; + + return LZMA_OK; +} diff --git a/src/liblzma/common/delta_decoder.h b/src/liblzma/delta/delta_decoder.h similarity index 88% rename from src/liblzma/common/delta_decoder.h rename to src/liblzma/delta/delta_decoder.h index bef9f58a..84852bf3 100644 --- a/src/liblzma/common/delta_decoder.h +++ b/src/liblzma/delta/delta_decoder.h @@ -25,4 +25,8 @@ extern lzma_ret lzma_delta_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); +extern lzma_ret lzma_delta_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + #endif diff --git a/src/liblzma/common/delta_encoder.c b/src/liblzma/delta/delta_encoder.c similarity index 85% rename from src/liblzma/common/delta_encoder.c rename to src/liblzma/delta/delta_encoder.c index a8bb9341..d8f40287 100644 --- a/src/liblzma/common/delta_encoder.c +++ b/src/liblzma/delta/delta_encoder.c @@ -96,3 +96,24 @@ lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, { return lzma_delta_coder_init(next, allocator, filters, &delta_encode); } + + +extern lzma_ret +lzma_delta_props_encode(const void *options, uint8_t *out) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + const lzma_options_delta *opt = options; + + // It's possible that newer liblzma versions will support larger + // distance values. + if (opt->type != LZMA_DELTA_TYPE_BYTE + || opt->distance < LZMA_DELTA_DISTANCE_MIN + || opt->distance > LZMA_DELTA_DISTANCE_MAX) + return LZMA_HEADER_ERROR; + + out[0] = opt->distance - LZMA_DELTA_DISTANCE_MIN; + + return LZMA_OK; +} diff --git a/src/liblzma/common/delta_encoder.h b/src/liblzma/delta/delta_encoder.h similarity index 92% rename from src/liblzma/common/delta_encoder.h rename to src/liblzma/delta/delta_encoder.h index c669458d..b8b29c61 100644 --- a/src/liblzma/common/delta_encoder.h +++ b/src/liblzma/delta/delta_encoder.h @@ -25,4 +25,6 @@ extern lzma_ret lzma_delta_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); +extern lzma_ret lzma_delta_props_encode(const void *options, uint8_t *out); + #endif diff --git a/src/liblzma/lz/Makefile.am b/src/liblzma/lz/Makefile.am index 5c27e2f2..bf41d8e6 100644 --- a/src/liblzma/lz/Makefile.am +++ b/src/liblzma/lz/Makefile.am @@ -20,43 +20,16 @@ liblz_la_CPPFLAGS = \ liblz_la_SOURCES = -if COND_MAIN_ENCODER +if COND_ENCODER_LZ liblz_la_SOURCES += \ lz_encoder.c \ lz_encoder.h \ - lz_encoder_private.h \ - match_c.h \ - match_h.h - -if COND_MF_HC3 -liblz_la_SOURCES += hc3.c hc3.h -liblz_la_CPPFLAGS += -DHAVE_HC3 -endif - -if COND_MF_HC4 -liblz_la_SOURCES += hc4.c hc4.h -liblz_la_CPPFLAGS += -DHAVE_HC4 -endif - -if COND_MF_BT2 -liblz_la_SOURCES += bt2.c bt2.h -liblz_la_CPPFLAGS += -DHAVE_BT2 -endif - -if COND_MF_BT3 -liblz_la_SOURCES += bt3.c bt3.h -liblz_la_CPPFLAGS += -DHAVE_BT3 -endif - -if COND_MF_BT4 -liblz_la_SOURCES += bt4.c bt4.h -liblz_la_CPPFLAGS += -DHAVE_BT4 -endif - + lz_encoder_hash.h \ + lz_encoder_mf.c endif -if COND_MAIN_DECODER +if COND_DECODER_LZ liblz_la_SOURCES += \ lz_decoder.c \ lz_decoder.h diff --git a/src/liblzma/lz/bt2.c b/src/liblzma/lz/bt2.c deleted file mode 100644 index 7dc4cb80..00000000 --- a/src/liblzma/lz/bt2.c +++ /dev/null @@ -1,27 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt2.c -/// \brief Binary Tree 2 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "bt2.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#include "match_c.h" diff --git a/src/liblzma/lz/bt2.h b/src/liblzma/lz/bt2.h deleted file mode 100644 index 33cb52cd..00000000 --- a/src/liblzma/lz/bt2.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt2.h -/// \brief Binary Tree 2 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_BT2_H -#define LZMA_BT2_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER bt2 -#define LZMA_MATCH_FINDER_NAME_UPPER BT2 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/bt3.c b/src/liblzma/lz/bt3.c deleted file mode 100644 index d44310f3..00000000 --- a/src/liblzma/lz/bt3.c +++ /dev/null @@ -1,29 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt3.c -/// \brief Binary Tree 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "bt3.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define HASH_ARRAY_2 - -#include "match_c.h" diff --git a/src/liblzma/lz/bt3.h b/src/liblzma/lz/bt3.h deleted file mode 100644 index 247c7e5f..00000000 --- a/src/liblzma/lz/bt3.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt3.h -/// \brief Binary Tree 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_BT3_H -#define LZMA_BT3_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER bt3 -#define LZMA_MATCH_FINDER_NAME_UPPER BT3 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/bt4.c b/src/liblzma/lz/bt4.c deleted file mode 100644 index 6e1042c9..00000000 --- a/src/liblzma/lz/bt4.c +++ /dev/null @@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt4.c -/// \brief Binary Tree 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "bt4.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define HASH_ARRAY_2 -#define HASH_ARRAY_3 - -#include "match_c.h" diff --git a/src/liblzma/lz/bt4.h b/src/liblzma/lz/bt4.h deleted file mode 100644 index e3fcf6ac..00000000 --- a/src/liblzma/lz/bt4.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file bt4.h -/// \brief Binary Tree 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_BT4_H -#define LZMA_BT4_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER bt4 -#define LZMA_MATCH_FINDER_NAME_UPPER BT4 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/hc3.c b/src/liblzma/lz/hc3.c deleted file mode 100644 index 22b5689b..00000000 --- a/src/liblzma/lz/hc3.c +++ /dev/null @@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc3.c -/// \brief Hash Chain 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "hc3.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define IS_HASH_CHAIN -#define HASH_ARRAY_2 - -#include "match_c.h" diff --git a/src/liblzma/lz/hc3.h b/src/liblzma/lz/hc3.h deleted file mode 100644 index 97be0b1d..00000000 --- a/src/liblzma/lz/hc3.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc3.h -/// \brief Hash Chain 3 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_HC3_H -#define LZMA_HC3_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER hc3 -#define LZMA_MATCH_FINDER_NAME_UPPER HC3 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/hc4.c b/src/liblzma/lz/hc4.c deleted file mode 100644 index a55cfd09..00000000 --- a/src/liblzma/lz/hc4.c +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc4.c -/// \brief Hash Chain 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "hc4.h" - -#undef IS_HASH_CHAIN -#undef HASH_ARRAY_2 -#undef HASH_ARRAY_3 - -#define IS_HASH_CHAIN -#define HASH_ARRAY_2 -#define HASH_ARRAY_3 - -#include "match_c.h" diff --git a/src/liblzma/lz/hc4.h b/src/liblzma/lz/hc4.h deleted file mode 100644 index dc072e2f..00000000 --- a/src/liblzma/lz/hc4.h +++ /dev/null @@ -1,31 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file hc4.h -/// \brief Hash Chain 4 -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_HC4_H -#define LZMA_HC4_H - -#undef LZMA_MATCH_FINDER_NAME_LOWER -#undef LZMA_MATCH_FINDER_NAME_UPPER -#define LZMA_MATCH_FINDER_NAME_LOWER hc4 -#define LZMA_MATCH_FINDER_NAME_UPPER HC4 - -#include "match_h.h" - -#endif diff --git a/src/liblzma/lz/lz_decoder.c b/src/liblzma/lz/lz_decoder.c index ae969d62..5c3f1d18 100644 --- a/src/liblzma/lz/lz_decoder.c +++ b/src/liblzma/lz/lz_decoder.c @@ -18,351 +18,142 @@ // /////////////////////////////////////////////////////////////////////////////// +// liblzma supports multiple LZ77-based filters. The LZ part is shared +// between these filters. The LZ code takes care of dictionary handling +// and passing the data between filters in the chain. The filter-specific +// part decodes from the input buffer to the dictionary. + + #include "lz_decoder.h" -/// Minimum size of allocated dictionary -#define DICT_SIZE_MIN 8192 - -/// When there is less than this amount of data available for decoding, -/// it is moved to the temporary buffer which -/// - protects from reads past the end of the buffer; and -/// - stored the incomplete data between lzma_code() calls. -/// -/// \note TEMP_LIMIT must be at least as much as -/// REQUIRED_IN_BUFFER_SIZE defined in lzma_decoder.c. -#define TEMP_LIMIT 32 - -// lzma_lz_decoder.dict[] must be three times the size of TEMP_LIMIT. -// 2 * TEMP_LIMIT is used for the actual data, and the third TEMP_LIMIT -// bytes is needed for safety to allow decode_dummy() in lzma_decoder.c -// to read past end of the buffer. This way it should be both fast and simple. -#if LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT -# error LZMA_BUFFER_SIZE < 3 * TEMP_LIMIT -#endif - - struct lzma_coder_s { - lzma_next_coder next; + /// Dictionary (history buffer) + lzma_dict dict; + + /// The actual LZ-based decoder e.g. LZMA lzma_lz_decoder lz; - // There are more members in this structure but they are not - // visible in LZ coder. + /// Next filter in the chain, if any. Note that LZMA and LZMA2 are + /// only allowed as the last filter, but the long-range filter in + /// future can be in the middle of the chain. + lzma_next_coder next; + + /// True if the next filter in the chain has returned LZMA_STREAM_END. + bool next_finished; + + /// True if the LZ decoder (e.g. LZMA) has detected end of payload + /// marker. This may become true before next_finished becomes true. + bool this_finished; + + /// Temporary buffer needed when the LZ-based filter is not the last + /// filter in the chain. The output of the next filter is first + /// decoded into buffer[], which is then used as input for the actual + /// LZ-based decoder. + struct { + size_t pos; + size_t size; + uint8_t buffer[LZMA_BUFFER_SIZE]; + } temp; }; -/// - Copy as much data as possible from lz->dict[] to out[]. -/// - Update *out_pos, lz->start, and lz->end accordingly. -/// - Wrap lz-pos to the beginning of lz->dict[] if there is a danger that -/// it may go past the end of the buffer (lz->pos >= lz->must_flush_pos). -static inline bool -flush(lzma_lz_decoder *restrict lz, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size) -{ - // Flush uncompressed data from the history buffer to - // the output buffer. This is done in two phases. - - assert(lz->start <= lz->end); - - // Flush if pos < start < end. - if (lz->pos < lz->start && lz->start < lz->end) { - bufcpy(lz->dict, &lz->start, lz->end, out, out_pos, out_size); - - // If we reached end of the data in history buffer, - // wrap to the beginning. - if (lz->start == lz->end) - lz->start = 0; - } - - // Flush if start start < pos <= end. This is not as `else' for - // previous `if' because the previous one may make this one true. - if (lz->start < lz->pos) { - bufcpy(lz->dict, &lz->start, - lz->pos, out, out_pos, out_size); - - if (lz->pos >= lz->must_flush_pos) { - // Wrap the flushing position if we have - // flushed the whole history buffer. - if (lz->pos == lz->start) - lz->start = 0; - - // Wrap the write position and store to lz.end - // how much there is new data available. - lz->end = lz->pos; - lz->pos = 0; - lz->is_full = true; - } - } - - assert(lz->pos < lz->must_flush_pos); - - return *out_pos == out_size; -} - - -/// Calculate safe value for lz->limit. If no safe value can be found, -/// set lz->limit to zero. When flushing, only as little data will be -/// decoded as is needed to fill the output buffer (lowers both latency -/// and throughput). -/// -/// \return true if there is no space for new uncompressed data. -/// -static inline bool -set_limit(lzma_lz_decoder *lz, size_t out_avail, bool flushing) -{ - // Set the limit so that writing to dict[limit + match_max_len - 1] - // doesn't overwrite any unflushed data and doesn't write past the - // end of the dict buffer. - if (lz->start <= lz->pos) { - // We can fill the buffer from pos till the end - // of the dict buffer. - lz->limit = lz->must_flush_pos; - } else if (lz->pos + lz->match_max_len < lz->start) { - // There's some unflushed data between pos and end of the - // buffer. Limit so that we don't overwrite the unflushed data. - lz->limit = lz->start - lz->match_max_len; - } else { - // Buffer is too full. - lz->limit = 0; - return true; - } - - // Finetune the limit a bit if it isn't zero. - - assert(lz->limit > lz->pos); - const size_t dict_avail = lz->limit - lz->pos; - - if (lz->uncompressed_size < dict_avail) { - // Finishing a stream that doesn't have - // an end of stream marker. - lz->limit = lz->pos + lz->uncompressed_size; - - } else if (flushing && out_avail < dict_avail) { - // Flushing enabled, decoding only as little as needed to - // fill the out buffer (if there's enough input, of course). - lz->limit = lz->pos + out_avail; - } - - return lz->limit == lz->pos; -} - - -/// Takes care of wrapping the data into temporary buffer when needed, -/// and calls the actual decoder. -/// -/// \return true if error occurred -/// -static inline bool -call_process(lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t in_size) -{ - // It would be nice and simple if we could just give in[] to the - // decoder, but the requirement of zlib-like API forces us to be - // able to make *in_pos == in_size whenever there is enough output - // space. If needed, we will append a few bytes from in[] to - // a temporary buffer and decode enough to reach the part that - // was copied from in[]. Then we can continue with the real in[]. - - bool error; - const size_t dict_old_pos = coder->lz.pos; - const size_t in_avail = in_size - *in_pos; - - if (coder->lz.temp_size + in_avail < 2 * TEMP_LIMIT) { - // Copy all the available input from in[] to temp[]. - memcpy(coder->lz.temp + coder->lz.temp_size, - in + *in_pos, in_avail); - coder->lz.temp_size += in_avail; - *in_pos += in_avail; - assert(*in_pos == in_size); - - // Decode as much as possible. - size_t temp_used = 0; - error = coder->lz.process(coder, coder->lz.temp, &temp_used, - coder->lz.temp_size, true); - assert(temp_used <= coder->lz.temp_size); - - // Move the remaining data to the beginning of temp[]. - coder->lz.temp_size -= temp_used; - memmove(coder->lz.temp, coder->lz.temp + temp_used, - coder->lz.temp_size); - - } else if (coder->lz.temp_size > 0) { - // Fill temp[] unless it is already full because we aren't - // the last filter in the chain. - size_t copy_size = 0; - if (coder->lz.temp_size < 2 * TEMP_LIMIT) { - assert(*in_pos < in_size); - copy_size = 2 * TEMP_LIMIT - coder->lz.temp_size; - memcpy(coder->lz.temp + coder->lz.temp_size, - in + *in_pos, copy_size); - // NOTE: We don't update lz.temp_size or *in_pos yet. - } - - size_t temp_used = 0; - error = coder->lz.process(coder, coder->lz.temp, &temp_used, - coder->lz.temp_size + copy_size, false); - - if (temp_used < coder->lz.temp_size) { - // Only very little input data was consumed. Move - // the unprocessed data to the beginning temp[]. - coder->lz.temp_size += copy_size - temp_used; - memmove(coder->lz.temp, coder->lz.temp + temp_used, - coder->lz.temp_size); - *in_pos += copy_size; - assert(*in_pos <= in_size); - - } else { - // We were able to decode so much data that next time - // we can decode directly from in[]. That is, we can - // consider temp[] to be empty now. - *in_pos += temp_used - coder->lz.temp_size; - coder->lz.temp_size = 0; - assert(*in_pos <= in_size); - } - - } else { - // Decode directly from in[]. - error = coder->lz.process(coder, in, in_pos, in_size, false); - assert(*in_pos <= in_size); - } - - assert(coder->lz.pos >= dict_old_pos); - if (coder->lz.uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { - // Update uncompressed size. - coder->lz.uncompressed_size -= coder->lz.pos - dict_old_pos; - - // Check that End of Payload Marker hasn't been detected - // since it must not be present because uncompressed size - // is known. - if (coder->lz.eopm_detected) - error = true; - } - - return error; -} - - static lzma_ret decode_buffer(lzma_coder *coder, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, - bool flushing) + size_t *restrict out_pos, size_t out_size) { - bool stop = false; - while (true) { - // Flush from coder->lz.dict to out[]. - flush(&coder->lz, out, out_pos, out_size); + // Wrap the dictionary if needed. + if (coder->dict.pos == coder->dict.size) + coder->dict.pos = 0; - // All done? - if (*out_pos == out_size - || stop - || coder->lz.eopm_detected - || coder->lz.uncompressed_size == 0) - break; + // Store the current dictionary position. It is needed to know + // where to start copying to the out[] buffer. + const size_t dict_start = coder->dict.pos; - // Set write limit in the dictionary. - if (set_limit(&coder->lz, out_size - *out_pos, flushing)) - break; + // Calculate how much we allow the process() function to + // decode. It must not decode past the end of the dictionary + // buffer, and we don't want it to decode more than is + // actually needed to fill the out[] buffer. + coder->dict.limit = coder->dict.pos + MIN(out_size - *out_pos, + coder->dict.size - coder->dict.pos); - // Decode more data. - if (call_process(coder, in, in_pos, in_size)) - return LZMA_DATA_ERROR; + // Call the process() function to do the actual decoding. + const lzma_ret ret = coder->lz.code( + coder->lz.coder, &coder->dict, + in, in_pos, in_size); - // Set stop to true if we must not call call_process() again - // during this function call. - // FIXME: Can this make the loop exist too early? It wouldn't - // cause data corruption so not a critical problem. It can - // happen if dictionary gets full and lz.temp still contains - // a few bytes data that we could decode right now. - if (*in_pos == in_size && coder->lz.temp_size <= TEMP_LIMIT - && coder->lz.pos < coder->lz.limit) - stop = true; + // Copy the decoded data from the dictionary to the out[] + // buffer. + const size_t copy_size = coder->dict.pos - dict_start; + assert(copy_size <= out_size - *out_pos); + memcpy(out + *out_pos, coder->dict.buf + dict_start, + copy_size); + *out_pos += copy_size; + + // Return if everything got decoded or an error occurred, or + // if there's no more data to decode. + if (ret != LZMA_OK || *out_pos == out_size + || coder->dict.pos < coder->dict.size) + return ret; } - - // If we have decoded everything (EOPM detected or uncompressed_size - // bytes were processed) to the history buffer, and also flushed - // everything from the history buffer, our job is done. - if ((coder->lz.eopm_detected - || coder->lz.uncompressed_size == 0) - && coder->lz.start == coder->lz.pos) - return LZMA_STREAM_END; - - return LZMA_OK; } -extern lzma_ret -lzma_lz_decode(lzma_coder *coder, +static lzma_ret +lz_decode(lzma_coder *coder, lzma_allocator *allocator lzma_attribute((unused)), const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, size_t *restrict out_pos, size_t out_size, lzma_action action) { - if (coder->next.code == NULL) { - const lzma_ret ret = decode_buffer(coder, in, in_pos, in_size, - out, out_pos, out_size, - action == LZMA_SYNC_FLUSH); - - if (*out_pos == out_size || ret == LZMA_STREAM_END) { - // Unread to make coder->temp[] empty. This is easy, - // because we know that all the data currently in - // coder->temp[] has been copied form in[] during this - // call to the decoder. - // - // If we didn't do this, we could have data left in - // coder->temp[] when end of stream is reached. That - // data could be left there from *previous* call to - // the decoder; in that case we wouldn't know where - // to put that data. - assert(*in_pos >= coder->lz.temp_size); - *in_pos -= coder->lz.temp_size; - coder->lz.temp_size = 0; - } - - return ret; - } + if (coder->next.code == NULL) + return decode_buffer(coder, in, in_pos, in_size, + out, out_pos, out_size); // We aren't the last coder in the chain, we need to decode // our input to a temporary buffer. - const bool flushing = action == LZMA_SYNC_FLUSH; while (*out_pos < out_size) { - if (!coder->lz.next_finished - && coder->lz.temp_size < LZMA_BUFFER_SIZE) { + // Fill the temporary buffer if it is empty. + if (!coder->next_finished + && coder->temp.pos == coder->temp.size) { + coder->temp.pos = 0; + coder->temp.size = 0; + const lzma_ret ret = coder->next.code( coder->next.coder, allocator, in, in_pos, in_size, - coder->lz.temp, &coder->lz.temp_size, + coder->temp.buffer, &coder->temp.size, LZMA_BUFFER_SIZE, action); if (ret == LZMA_STREAM_END) - coder->lz.next_finished = true; - else if (coder->lz.temp_size < LZMA_BUFFER_SIZE - || ret != LZMA_OK) + coder->next_finished = true; + else if (ret != LZMA_OK || coder->temp.size == 0) return ret; } - if (coder->lz.this_finished) { - if (coder->lz.temp_size != 0) + if (coder->this_finished) { + if (coder->temp.size != 0) return LZMA_DATA_ERROR; - if (coder->lz.next_finished) + if (coder->next_finished) return LZMA_STREAM_END; return LZMA_OK; } - size_t dummy = 0; - const lzma_ret ret = decode_buffer(coder, NULL, &dummy, 0, - out, out_pos, out_size, flushing); + const lzma_ret ret = decode_buffer(coder, coder->temp.buffer, + &coder->temp.pos, coder->temp.size, + out, out_pos, out_size); if (ret == LZMA_STREAM_END) - coder->lz.this_finished = true; + coder->this_finished = true; else if (ret != LZMA_OK) return ret; - else if (coder->lz.next_finished && *out_pos < out_size) + else if (coder->next_finished && *out_pos < out_size) return LZMA_DATA_ERROR; } @@ -370,94 +161,104 @@ lzma_lz_decode(lzma_coder *coder, } -/// \brief Initializes LZ part of the LZMA decoder or Inflate -/// -/// \param history_size Number of bytes the LZ out window is -/// supposed keep available from the output -/// history. -/// \param match_max_len Number of bytes a single decoding loop -/// can advance the write position (lz->pos) -/// in the history buffer (lz->dict). -/// -/// \note This function is called by LZMA decoder and Inflate init()s. -/// It's up to those functions allocate *lz and initialize it -/// with LZMA_LZ_DECODER_INIT. -extern lzma_ret -lzma_lz_decoder_reset(lzma_lz_decoder *lz, lzma_allocator *allocator, - bool (*process)(lzma_coder *restrict coder, - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, bool has_safe_buffer), - size_t history_size, size_t match_max_len) +static void +lz_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - // Known uncompressed size is used only with LZMA_Alone files so we - // set it always to unknown by default. - lz->uncompressed_size = LZMA_VLI_VALUE_UNKNOWN; + lzma_next_end(&coder->next, allocator); + lzma_free(coder->dict.buf, allocator); - // Limit the history size to roughly sane values. This is primarily - // to prevent integer overflows. - if (history_size > UINT32_MAX / 2) - return LZMA_HEADER_ERROR; + if (coder->lz.end != NULL) + coder->lz.end(coder->lz.coder, allocator); + else + lzma_free(coder->lz.coder, allocator); - // Store the value actually requested. We use it for sanity checks - // when repeating data from the history buffer. - lz->requested_size = history_size; + lzma_free(coder, allocator); + return; +} - // Avoid tiny history buffer sizes for performance reasons. - // TODO: Test if this actually helps... - if (history_size < DICT_SIZE_MIN) - history_size = DICT_SIZE_MIN; - // The real size of the history buffer is a bit bigger than - // requested by our caller. This allows us to do some optimizations, - // which help not only speed but simplicity of the code; specifically, - // we can make sure that there is always at least match_max_len - // bytes immediatelly available for writing without a need to wrap - // the history buffer. - const size_t dict_real_size = history_size + 2 * match_max_len + 1; - - // Reallocate memory if needed. - if (history_size != lz->size || match_max_len != lz->match_max_len) { - // Destroy the old buffer. - lzma_lz_decoder_end(lz, allocator); - - lz->size = history_size; - lz->match_max_len = match_max_len; - lz->must_flush_pos = history_size + match_max_len + 1; - - lz->dict = lzma_alloc(dict_real_size, allocator); - if (lz->dict == NULL) +extern lzma_ret +lzma_lz_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_decoder *lz, + lzma_allocator *allocator, const void *options, + size_t *dict_size)) +{ + // Allocate the base structure if it isn't already allocated. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) return LZMA_MEM_ERROR; + + next->code = &lz_decode; + next->end = &lz_decoder_end; + + next->coder->dict.buf = NULL; + next->coder->dict.size = 0; + next->coder->lz = LZMA_LZ_DECODER_INIT; + next->coder->next = LZMA_NEXT_CODER_INIT; } - // Reset the variables so that lz_get_byte(lz, 0) will return '\0'. - lz->pos = 0; - lz->start = 0; - lz->end = dict_real_size; - lz->dict[dict_real_size - 1] = 0; - lz->is_full = false; - lz->eopm_detected = false; - lz->next_finished = false; - lz->this_finished = false; - lz->temp_size = 0; + // Allocate and initialize the LZ-based decoder. It will also give + // us the dictionary size. + size_t dict_size; + return_if_error(lz_init(&next->coder->lz, allocator, + filters[0].options, &dict_size)); - // Clean up the temporary buffer to make it very sure that there are - // no information leaks when multiple steams are decoded with the - // same decoder structures. - memzero(lz->temp, LZMA_BUFFER_SIZE); + // If the dictionary size is very small, increase it to 4096 bytes. + // This is to prevent constant wrapping of the dictionary, which + // would slow things down. The downside is that since we don't check + // separately for the real dictionary size, we may happily accept + // corrupt files. + if (dict_size < 4096) + dict_size = 4096; - // Set the process function pointer. - lz->process = process; + // Make dictionary size a multipe of 16. Some LZ-based decoders like + // LZMA use the lowest bits lzma_dict.pos to know the alignment of the + // data. Aligned buffer is also good when memcpying from the + // dictionary to the output buffer, since applications are + // recommended to give aligned buffers to liblzma. + // + // Avoid integer overflow. FIXME Should the return value be + // LZMA_HEADER_ERROR or LZMA_MEM_ERROR? + if (dict_size > SIZE_MAX - 15) + return LZMA_MEM_ERROR; - return LZMA_OK; + dict_size = (dict_size + 15) & (SIZE_MAX - 15); + + // Allocate and initialize the dictionary. + if (next->coder->dict.size != dict_size) { + lzma_free(next->coder->dict.buf, allocator); + next->coder->dict.buf = lzma_alloc(dict_size, allocator); + if (next->coder->dict.buf == NULL) + return LZMA_MEM_ERROR; + + next->coder->dict.size = dict_size; + } + + dict_reset(&next->coder->dict); + + // Miscellaneous initializations + next->coder->next_finished = false; + next->coder->this_finished = false; + next->coder->temp.pos = 0; + next->coder->temp.size = 0; + + // Initialize the next filter in the chain, if any. + return lzma_next_filter_init(&next->coder->next, allocator, + filters + 1); +} + + +extern uint64_t +lzma_lz_decoder_memusage(size_t dictionary_size) +{ + return sizeof(lzma_coder) + (uint64_t)(dictionary_size); } extern void -lzma_lz_decoder_end(lzma_lz_decoder *lz, lzma_allocator *allocator) +lzma_lz_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) { - lzma_free(lz->dict, allocator); - lz->dict = NULL; - lz->size = 0; - lz->match_max_len = 0; - return; + coder->lz.set_uncompressed(coder->lz.coder, uncompressed_size); } diff --git a/src/liblzma/lz/lz_decoder.h b/src/liblzma/lz/lz_decoder.h index 1acf9831..d2a77ba4 100644 --- a/src/liblzma/lz/lz_decoder.h +++ b/src/liblzma/lz/lz_decoder.h @@ -18,201 +18,215 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_LZ_OUT_H -#define LZMA_LZ_OUT_H +#ifndef LZMA_LZ_DECODER_H +#define LZMA_LZ_DECODER_H #include "common.h" -/// Get a byte from the history buffer. -#define lz_get_byte(lz, distance) \ - ((distance) < (lz).pos \ - ? (lz).dict[(lz).pos - (distance) - 1] \ - : (lz).dict[(lz).pos - (distance) - 1 + (lz).end]) +typedef struct { + /// Pointer to the dictionary buffer. It can be an allocated buffer + /// internal to liblzma, or it can a be a buffer given by the + /// application when in single-call mode (not implemented yet). + uint8_t *buf; + /// Write position in dictionary. The next byte will be written to + /// buf[pos]. + size_t pos; -/// Test if dictionary is empty. -#define lz_is_empty(lz) \ - ((lz).pos == 0 && !(lz).is_full) + /// Indicates how full the dictionary is. This is used by + /// dict_is_distance_valid() to detect corrupt files that would + /// read beyond the beginning of the dictionary. + size_t full; + /// Write limit + size_t limit; -#define LZMA_LZ_DECODER_INIT \ - (lzma_lz_decoder){ .dict = NULL, .size = 0, .match_max_len = 0 } + /// Size of the dictionary + size_t size; + +} lzma_dict; typedef struct { - /// Function to do the actual decoding (LZMA or Inflate) - bool (*process)(lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t size_in, - bool has_safe_buffer); + /// Data specific to the LZ-based decoder + lzma_coder *coder; - /// Pointer to dictionary (history) buffer. - /// \note Not 'restrict' because can alias next_out. - uint8_t *dict; + /// Function to decode from in[] to *dict + lzma_ret (*code)(lzma_coder *restrict coder, + lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size); - /// Next write goes to dict[pos]. - size_t pos; + void (*reset)(lzma_coder *coder, const void *options); - /// Next byte to flush is buffer[start]. - size_t start; + /// Set the uncompressed size + void (*set_uncompressed)(lzma_coder *coder, + lzma_vli uncompressed_size); - /// First byte to not flush is buffer[end]. - size_t end; - - /// First position to which data must not be written. - size_t limit; - - /// True if dictionary has needed wrapping. - bool is_full; - - /// True if process() has detected End of Payload Marker. - bool eopm_detected; - - /// True if the next coder in the chain has returned LZMA_STREAM_END. - bool next_finished; - - /// True if the LZ decoder (e.g. LZMA) has detected End of Payload - /// Marker. This may become true before next_finished becomes true. - bool this_finished; - - /// When pos >= must_flush_pos, we must not call process(). - size_t must_flush_pos; - - /// Maximum number of bytes that a single decoding loop inside - /// process() can produce data into dict. This amount is kept - /// always available at dict + pos i.e. it is safe to write a byte - /// to dict[pos + match_max_len - 1]. - size_t match_max_len; - - /// Number of bytes allocated to dict. - size_t size; - - /// Requested size of the dictionary. This is needed because we avoid - /// using extremely tiny history buffers. - size_t requested_size; - - /// Uncompressed Size or LZMA_VLI_VALUE_UNKNOWN if unknown. - lzma_vli uncompressed_size; - - /// Number of bytes currently in temp[]. - size_t temp_size; - - /// Temporary buffer needed when - /// 1) we cannot make the input buffer completely empty; or - /// 2) we are not the last filter in the chain. - uint8_t temp[LZMA_BUFFER_SIZE]; + /// Free allocated resources + void (*end)(lzma_coder *coder, lzma_allocator *allocator); } lzma_lz_decoder; -///////////////////////// -// Function prototypes // -///////////////////////// +#define LZMA_LZ_DECODER_INIT \ + (lzma_lz_decoder){ \ + .coder = NULL, \ + .code = NULL, \ + .reset = NULL, \ + .set_uncompressed = NULL, \ + .end = NULL, \ + } -extern lzma_ret lzma_lz_decoder_reset(lzma_lz_decoder *lz, - lzma_allocator *allocator, bool (*process)( - lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t in_size, - bool has_safe_buffer), - size_t history_size, size_t match_max_len); -extern lzma_ret lzma_lz_decode(lzma_coder *coder, lzma_allocator *allocator, - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, - lzma_action action); +extern lzma_ret lzma_lz_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_decoder *lz, + lzma_allocator *allocator, const void *options, + size_t *dict_size)); + +extern uint64_t lzma_lz_decoder_memusage(size_t dictionary_size); + +extern void lzma_lz_decoder_uncompressed( + lzma_coder *coder, lzma_vli uncompressed_size); -/// Deallocates the history buffer if one exists. -extern void lzma_lz_decoder_end( - lzma_lz_decoder *lz, lzma_allocator *allocator); ////////////////////// // Inline functions // ////////////////////// -// Repeat a block of data from the history. Because memcpy() is faster -// than copying byte by byte in a loop, the copying process gets split -// into three cases: -// 1. distance < length -// Source and target areas overlap, thus we can't use memcpy() -// (nor memmove()) safely. -// TODO: If this is common enough, it might be worth optimizing this -// more e.g. by checking if distance > sizeof(uint8_t*) and using -// memcpy in small chunks. -// 2. distance < pos -// This is the easiest and the fastest case. The block being copied -// is a contiguous piece in the history buffer. The buffer offset -// doesn't need wrapping. -// 3. distance >= pos -// We need to wrap the position, because otherwise we would try copying -// behind the first byte of the allocated buffer. It is possible that -// the block is fragmeneted into two pieces, thus we might need to call -// memcpy() twice. -// NOTE: The function using this macro must ensure that length is positive -// and that distance is FIXME -static inline bool -lzma_lz_out_repeat(lzma_lz_decoder *lz, size_t distance, size_t length) +/// Get a byte from the history buffer. +static inline uint8_t +dict_get(const lzma_dict *const dict, const uint32_t distance) { - // Validate offset of the block to be repeated. It doesn't - // make sense to copy data behind the beginning of the stream. - // Leaving this check away would lead to a security problem, - // in which e.g. the data of the previously decoded file(s) - // would be leaked (or whatever happens to be in unused - // part of the dictionary buffer). - if (unlikely(distance >= lz->pos && !lz->is_full)) - return false; + return dict->buf[dict->pos - distance - 1 + + (distance < dict->pos ? 0 : dict->size)]; +} - // It also doesn't make sense to copy data farer than - // the dictionary size. - if (unlikely(distance >= lz->requested_size)) - return false; - // The caller must have checked these! - assert(distance <= lz->size); - assert(length > 0); - assert(length <= lz->match_max_len); +/// Test if dictionary is empty. +static inline bool +dict_is_empty(const lzma_dict *const dict) +{ + return dict->full == 0; +} - // Copy the amount of data requested by the decoder. - if (distance < length) { + +/// Validate the match distance +static inline bool +dict_is_distance_valid(const lzma_dict *const dict, const size_t distance) +{ + return dict->full >= distance; +} + + +/// Repeat *len bytes at distance. +static inline bool +dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) +{ + // Don't write past the end of the dictionary. + const size_t dict_avail = dict->limit - dict->pos; + uint32_t left = MIN(dict_avail, *len); + *len -= left; + + // Repeat a block of data from the history. Because memcpy() is faster + // than copying byte by byte in a loop, the copying process gets split + // into three cases. + if (distance < left) { // Source and target areas overlap, thus we can't use - // memcpy() nor even memmove() safely. :-( - // TODO: Copying byte by byte is slow. It might be - // worth optimizing this more if this case is common. + // memcpy() nor even memmove() safely. do { - lz->dict[lz->pos] = lz_get_byte(*lz, distance); - ++lz->pos; - } while (--length > 0); + dict->buf[dict->pos] = dict_get(dict, distance); + ++dict->pos; + } while (--left > 0); - } else if (distance < lz->pos) { + } else if (distance < dict->pos) { // The easiest and fastest case - memcpy(lz->dict + lz->pos, - lz->dict + lz->pos - distance - 1, - length); - lz->pos += length; + memcpy(dict->buf + dict->pos, + dict->buf + dict->pos - distance - 1, + left); + dict->pos += left; } else { // The bigger the dictionary, the more rare this // case occurs. We need to "wrap" the dict, thus // we might need two memcpy() to copy all the data. - assert(lz->is_full); - const uint32_t copy_pos = lz->pos - distance - 1 + lz->end; - uint32_t copy_size = lz->end - copy_pos; + assert(dict->full == dict->size); + const uint32_t copy_pos + = dict->pos - distance - 1 + dict->size; + uint32_t copy_size = dict->size - copy_pos; - if (copy_size < length) { - memcpy(lz->dict + lz->pos, lz->dict + copy_pos, + if (copy_size < left) { + memcpy(dict->buf + dict->pos, dict->buf + copy_pos, copy_size); - lz->pos += copy_size; - copy_size = length - copy_size; - memcpy(lz->dict + lz->pos, lz->dict, copy_size); - lz->pos += copy_size; + dict->pos += copy_size; + copy_size = left - copy_size; + memcpy(dict->buf + dict->pos, dict->buf, copy_size); + dict->pos += copy_size; } else { - memcpy(lz->dict + lz->pos, lz->dict + copy_pos, - length); - lz->pos += length; + memcpy(dict->buf + dict->pos, dict->buf + copy_pos, + left); + dict->pos += left; } } - return true; + // Update how full the dictionary is. + if (dict->full < dict->pos) + dict->full = dict->pos; + + return unlikely(*len != 0); +} + + +/// Puts one byte into the dictionary. Returns true if the dictionary was +/// already full and the byte couldn't be added. +static inline bool +dict_put(lzma_dict *dict, uint8_t byte) +{ + if (unlikely(dict->pos == dict->limit)) + return true; + + dict->buf[dict->pos++] = byte; + + if (dict->pos > dict->full) + dict->full = dict->pos; + + return false; +} + + +/// Copies arbitrary amount of data into the dictionary. +static inline void +dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size, + size_t *restrict left) +{ + // NOTE: If we are being given more data than the size of the + // dictionary, it could be possible to optimize the LZ decoder + // so that not everything needs to go through the dictionary. + // This shouldn't be very common thing in practice though, and + // the slowdown of one extra memcpy() isn't bad compared to how + // much time it would have taken if the data were compressed. + + if (in_size - *in_pos > *left) + in_size = *in_pos + *left; + + *left -= lzma_bufcpy(in, in_pos, in_size, + dict->buf, &dict->pos, dict->limit); + + if (dict->pos > dict->full) + dict->full = dict->pos; + + return; +} + + +static inline void +dict_reset(lzma_dict *dict) +{ + dict->pos = 0; + dict->full = 0; + dict->buf[dict->size - 1] = '\0'; } #endif diff --git a/src/liblzma/lz/lz_encoder.c b/src/liblzma/lz/lz_encoder.c index 82b9103f..d5f84826 100644 --- a/src/liblzma/lz/lz_encoder.c +++ b/src/liblzma/lz/lz_encoder.c @@ -3,8 +3,8 @@ /// \file lz_encoder.c /// \brief LZ in window // -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -18,346 +18,61 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "lz_encoder_private.h" - -// Hash Chains -#ifdef HAVE_HC3 -# include "hc3.h" -#endif -#ifdef HAVE_HC4 -# include "hc4.h" -#endif - -// Binary Trees -#ifdef HAVE_BT2 -# include "bt2.h" -#endif -#ifdef HAVE_BT3 -# include "bt3.h" -#endif -#ifdef HAVE_BT4 -# include "bt4.h" -#endif +#include "lz_encoder.h" +#include "lz_encoder_hash.h" -/// This is needed in two places so provide a macro. -#define get_cyclic_buffer_size(history_size) ((history_size) + 1) +struct lzma_coder_s { + /// LZ-based encoder e.g. LZMA + lzma_lz_encoder lz; + /// History buffer and match finder + lzma_mf mf; -/// Calculate certain match finder properties and validate the calculated -/// values. This is as its own function, because *num_items is needed to -/// calculate memory requirements in common/memory.c. -extern bool -lzma_lz_encoder_hash_properties(lzma_match_finder match_finder, - uint32_t history_size, uint32_t *restrict hash_mask, - uint32_t *restrict hash_size_sum, uint32_t *restrict num_items) -{ - uint32_t fix_hash_size; - uint32_t sons; - - switch (match_finder) { -#ifdef HAVE_HC3 - case LZMA_MF_HC3: - fix_hash_size = LZMA_HC3_FIX_HASH_SIZE; - sons = 1; - break; -#endif -#ifdef HAVE_HC4 - case LZMA_MF_HC4: - fix_hash_size = LZMA_HC4_FIX_HASH_SIZE; - sons = 1; - break; -#endif -#ifdef HAVE_BT2 - case LZMA_MF_BT2: - fix_hash_size = LZMA_BT2_FIX_HASH_SIZE; - sons = 2; - break; -#endif -#ifdef HAVE_BT3 - case LZMA_MF_BT3: - fix_hash_size = LZMA_BT3_FIX_HASH_SIZE; - sons = 2; - break; -#endif -#ifdef HAVE_BT4 - case LZMA_MF_BT4: - fix_hash_size = LZMA_BT4_FIX_HASH_SIZE; - sons = 2; - break; -#endif - default: - return true; - } - - uint32_t hs; - -#ifdef HAVE_LZMA_BT2 - if (match_finder == LZMA_BT2) { - // NOTE: hash_mask is not used by the BT2 match finder, - // but it is initialized just in case. - hs = LZMA_BT2_HASH_SIZE; - *hash_mask = 0; - } else -#endif - { - hs = history_size - 1; - hs |= (hs >> 1); - hs |= (hs >> 2); - hs |= (hs >> 4); - hs |= (hs >> 8); - hs >>= 1; - hs |= 0xFFFF; - - if (hs > (UINT32_C(1) << 24)) { - if (match_finder == LZMA_MF_HC4 - || match_finder == LZMA_MF_BT4) - hs >>= 1; - else - hs = (1 << 24) - 1; - } - - *hash_mask = hs; - ++hs; - } - - *hash_size_sum = hs + fix_hash_size; - - *num_items = *hash_size_sum - + get_cyclic_buffer_size(history_size) * sons; - - return false; -} - - -extern lzma_ret -lzma_lz_encoder_reset(lzma_lz_encoder *lz, lzma_allocator *allocator, - bool (*process)(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size), - size_t history_size, size_t additional_buffer_before, - size_t match_max_len, size_t additional_buffer_after, - lzma_match_finder match_finder, uint32_t match_finder_cycles, - const uint8_t *preset_dictionary, - size_t preset_dictionary_size) -{ - lz->sequence = SEQ_RUN; - - /////////////// - // In Window // - /////////////// - - // Validate history size. - if (history_size < LZMA_DICTIONARY_SIZE_MIN - || history_size > LZMA_DICTIONARY_SIZE_MAX) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; - } - - assert(history_size <= MAX_VAL_FOR_NORMALIZE - 256); - assert(LZMA_DICTIONARY_SIZE_MAX <= MAX_VAL_FOR_NORMALIZE - 256); - - // Calculate the size of the history buffer to allocate. - // TODO: Get a reason for magic constant of 256. - const size_t size_reserv = (history_size + additional_buffer_before - + match_max_len + additional_buffer_after) / 2 + 256; - - lz->keep_size_before = history_size + additional_buffer_before; - lz->keep_size_after = match_max_len + additional_buffer_after; - - const size_t buffer_size = lz->keep_size_before + lz->keep_size_after - + size_reserv; - - // Allocate history buffer if its size has changed. - if (buffer_size != lz->size) { - lzma_free(lz->buffer, allocator); - lz->buffer = lzma_alloc(buffer_size, allocator); - if (lz->buffer == NULL) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_MEM_ERROR; - } - } - - // Allocation successful. Store the new size. - lz->size = buffer_size; - - // Reset in window variables. - lz->offset = 0; - lz->read_pos = 0; - lz->read_limit = 0; - lz->write_pos = 0; - lz->pending = 0; - - - ////////////////// - // Match Finder // - ////////////////// - - // Validate match_finder, set function pointers and a few match - // finder specific variables. - switch (match_finder) { -#ifdef HAVE_HC3 - case LZMA_MF_HC3: - lz->get_matches = &lzma_hc3_get_matches; - lz->skip = &lzma_hc3_skip; - lz->cut_value = 8 + (match_max_len >> 2); - break; -#endif -#ifdef HAVE_HC4 - case LZMA_MF_HC4: - lz->get_matches = &lzma_hc4_get_matches; - lz->skip = &lzma_hc4_skip; - lz->cut_value = 8 + (match_max_len >> 2); - break; -#endif -#ifdef HAVE_BT2 - case LZMA_MF_BT2: - lz->get_matches = &lzma_bt2_get_matches; - lz->skip = &lzma_bt2_skip; - lz->cut_value = 16 + (match_max_len >> 1); - break; -#endif -#ifdef HAVE_BT3 - case LZMA_MF_BT3: - lz->get_matches = &lzma_bt3_get_matches; - lz->skip = &lzma_bt3_skip; - lz->cut_value = 16 + (match_max_len >> 1); - break; -#endif -#ifdef HAVE_BT4 - case LZMA_MF_BT4: - lz->get_matches = &lzma_bt4_get_matches; - lz->skip = &lzma_bt4_skip; - lz->cut_value = 16 + (match_max_len >> 1); - break; -#endif - default: - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; - } - - // Check if we have been requested to use a non-default cut_value. - if (match_finder_cycles > 0) - lz->cut_value = match_finder_cycles; - - lz->match_max_len = match_max_len; - lz->cyclic_buffer_size = get_cyclic_buffer_size(history_size); - - uint32_t hash_size_sum; - uint32_t num_items; - if (lzma_lz_encoder_hash_properties(match_finder, history_size, - &lz->hash_mask, &hash_size_sum, &num_items)) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; - } - - if (num_items != lz->num_items) { -#if UINT32_MAX >= SIZE_MAX / 4 - // Check for integer overflow. (Huge dictionaries are not - // possible on 32-bit CPU.) - if (num_items > SIZE_MAX / sizeof(uint32_t)) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_MEM_ERROR; - } -#endif - - const size_t size_in_bytes - = (size_t)(num_items) * sizeof(uint32_t); - - lzma_free(lz->hash, allocator); - lz->hash = lzma_alloc(size_in_bytes, allocator); - if (lz->hash == NULL) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_MEM_ERROR; - } - - lz->num_items = num_items; - } - - lz->son = lz->hash + hash_size_sum; - - // Reset the hash table to empty hash values. - { - uint32_t *restrict items = lz->hash; - - for (uint32_t i = 0; i < hash_size_sum; ++i) - items[i] = EMPTY_HASH_VALUE; - } - - lz->cyclic_buffer_pos = 0; - - // Because zero is used as empty hash value, make the first byte - // appear at buffer[1 - offset]. - ++lz->offset; - - // If we are using a preset dictionary, read it now. - // TODO: This isn't implemented yet so return LZMA_HEADER_ERROR. - if (preset_dictionary != NULL && preset_dictionary_size > 0) { - lzma_lz_encoder_end(lz, allocator); - return LZMA_HEADER_ERROR; - } - - // Set the process function pointer. - lz->process = process; - - return LZMA_OK; -} - - -extern void -lzma_lz_encoder_end(lzma_lz_encoder *lz, lzma_allocator *allocator) -{ - lzma_free(lz->hash, allocator); - lz->hash = NULL; - lz->num_items = 0; - - lzma_free(lz->buffer, allocator); - lz->buffer = NULL; - lz->size = 0; - - return; -} + /// Next coder in the chain + lzma_next_coder next; +}; /// \brief Moves the data in the input window to free space for new data /// -/// lz->buffer is a sliding input window, which keeps lz->keep_size_before +/// mf->buffer is a sliding input window, which keeps mf->keep_size_before /// bytes of input history available all the time. Now and then we need to /// "slide" the buffer to make space for the new data to the end of the /// buffer. At the same time, data older than keep_size_before is dropped. /// static void -move_window(lzma_lz_encoder *lz) +move_window(lzma_mf *mf) { - // buffer[move_offset] will become buffer[0]. - assert(lz->read_pos > lz->keep_size_after); - size_t move_offset = lz->read_pos - lz->keep_size_before; + // Align the move to a multiple of 16 bytes. Some LZ-based encoders + // like LZMA use the lowest bits of mf->read_pos to know the + // alignment of the uncompressed data. We also get better speed + // for memmove() with aligned buffers. + assert(mf->read_pos > mf->keep_size_before); + const uint32_t move_offset + = (mf->read_pos - mf->keep_size_before) & ~UINT32_C(15); - // We need one additional byte, since move_pos() moves on 1 byte. - // TODO: Clean up? At least document more. - if (move_offset > 0) - --move_offset; + assert(mf->write_pos > move_offset); + const size_t move_size = mf->write_pos - move_offset; - assert(lz->write_pos > move_offset); - const size_t move_size = lz->write_pos - move_offset; + assert(move_offset + move_size <= mf->size); - assert(move_offset + move_size <= lz->size); + memmove(mf->buffer, mf->buffer + move_offset, move_size); - memmove(lz->buffer, lz->buffer + move_offset, move_size); - - lz->offset += move_offset; - lz->read_pos -= move_offset; - lz->read_limit -= move_offset; - lz->write_pos -= move_offset; + mf->offset += move_offset; + mf->read_pos -= move_offset; + mf->read_limit -= move_offset; + mf->write_pos -= move_offset; return; } -/// \brief Tries to fill the input window (lz->buffer) +/// \brief Tries to fill the input window (mf->buffer) /// /// If we are the last encoder in the chain, our input data is in in[]. /// Otherwise we call the next filter in the chain to process in[] and -/// write its output to lz->buffer. +/// write its output to mf->buffer. /// /// This function must not be called once it has returned LZMA_STREAM_END. /// @@ -365,30 +80,28 @@ static lzma_ret fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size, lzma_action action) { - assert(coder->lz.read_pos <= coder->lz.write_pos); + assert(coder->mf.read_pos <= coder->mf.write_pos); // Move the sliding window if needed. - if (coder->lz.read_pos >= coder->lz.size - coder->lz.keep_size_after) - move_window(&coder->lz); + if (coder->mf.read_pos >= coder->mf.size - coder->mf.keep_size_after) + move_window(&coder->mf); size_t in_used; lzma_ret ret; if (coder->next.code == NULL) { // Not using a filter, simply memcpy() as much as possible. - in_used = bufcpy(in, in_pos, in_size, coder->lz.buffer, - &coder->lz.write_pos, coder->lz.size); + in_used = lzma_bufcpy(in, in_pos, in_size, coder->mf.buffer, + &coder->mf.write_pos, coder->mf.size); - if (action != LZMA_RUN && *in_pos == in_size) - ret = LZMA_STREAM_END; - else - ret = LZMA_OK; + ret = action != LZMA_RUN && *in_pos == in_size + ? LZMA_STREAM_END : LZMA_OK; } else { const size_t in_start = *in_pos; ret = coder->next.code(coder->next.coder, allocator, in, in_pos, in_size, - coder->lz.buffer, &coder->lz.write_pos, - coder->lz.size, action); + coder->mf.buffer, &coder->mf.write_pos, + coder->mf.size, action); in_used = *in_pos - in_start; } @@ -398,53 +111,43 @@ fill_window(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *in, // available as prebuffer. if (ret == LZMA_STREAM_END) { assert(*in_pos == in_size); - coder->lz.read_limit = coder->lz.write_pos; ret = LZMA_OK; + coder->mf.action = action; + coder->mf.read_limit = coder->mf.write_pos; - switch (action) { - case LZMA_SYNC_FLUSH: - coder->lz.sequence = SEQ_FLUSH; - break; - - case LZMA_FINISH: - coder->lz.sequence = SEQ_FINISH; - break; - - default: - assert(0); - ret = LZMA_PROG_ERROR; - break; - } - - } else if (coder->lz.write_pos > coder->lz.keep_size_after) { + } else if (coder->mf.write_pos > coder->mf.keep_size_after) { // This needs to be done conditionally, because if we got // only little new input, there may be too little input // to do any encoding yet. - coder->lz.read_limit = coder->lz.write_pos - - coder->lz.keep_size_after; + coder->mf.read_limit = coder->mf.write_pos + - coder->mf.keep_size_after; } // Restart the match finder after finished LZMA_SYNC_FLUSH. - if (coder->lz.pending > 0 - && coder->lz.read_pos < coder->lz.read_limit) { + if (coder->mf.pending > 0 + && coder->mf.read_pos < coder->mf.read_limit) { // Match finder may update coder->pending and expects it to // start from zero, so use a temporary variable. - const size_t pending = coder->lz.pending; - coder->lz.pending = 0; + const size_t pending = coder->mf.pending; + coder->mf.pending = 0; // Rewind read_pos so that the match finder can hash // the pending bytes. - assert(coder->lz.read_pos >= pending); - coder->lz.read_pos -= pending; - coder->lz.skip(&coder->lz, pending); + assert(coder->mf.read_pos >= pending); + coder->mf.read_pos -= pending; + + // Call the skip function directly instead of using + // lz_dict_skip(), since we don't want to touch + // mf->read_ahead. + coder->mf.skip(&coder->mf, pending); } return ret; } -extern lzma_ret -lzma_lz_encode(lzma_coder *coder, lzma_allocator *allocator, +static lzma_ret +lz_encode(lzma_coder *coder, lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size, uint8_t *restrict out, size_t *restrict out_pos, @@ -452,18 +155,21 @@ lzma_lz_encode(lzma_coder *coder, lzma_allocator *allocator, { while (*out_pos < out_size && (*in_pos < in_size || action != LZMA_RUN)) { - // Read more data to coder->lz.buffer if needed. - if (coder->lz.sequence == SEQ_RUN - && coder->lz.read_pos >= coder->lz.read_limit) + // Read more data to coder->mf.buffer if needed. + if (coder->mf.action == LZMA_RUN && coder->mf.read_pos + >= coder->mf.read_limit) return_if_error(fill_window(coder, allocator, in, in_pos, in_size, action)); // Encode - if (coder->lz.process(coder, out, out_pos, out_size)) { - // Setting this to SEQ_RUN for cases when we are - // flushing. It doesn't matter when finishing. - coder->lz.sequence = SEQ_RUN; - return action != LZMA_RUN ? LZMA_STREAM_END : LZMA_OK; + const lzma_ret ret = coder->lz.code(coder->lz.coder, + &coder->mf, out, out_pos, out_size); + if (ret != LZMA_OK) { + // Setting this to LZMA_RUN for cases when we are + // flushing. It doesn't matter when finishing or if + // an error occurred. + coder->mf.action = LZMA_RUN; + return ret; } } @@ -471,43 +177,333 @@ lzma_lz_encode(lzma_coder *coder, lzma_allocator *allocator, } -/// \brief Normalizes hash values -/// -/// lzma_lz_normalize is called when lz->pos hits MAX_VAL_FOR_NORMALIZE, -/// which currently happens once every 2 GiB of input data (to be exact, -/// after the first 2 GiB it happens once every 2 GiB minus dictionary_size -/// bytes). lz->pos is incremented by lzma_lz_move_pos(). -/// -/// lz->hash contains big amount of offsets relative to lz->buffer. -/// The offsets are stored as uint32_t, which is the only reasonable -/// datatype for these offsets; uint64_t would waste far too much RAM -/// and uint16_t would limit the dictionary to 64 KiB (far too small). -/// -/// When compressing files over 2 GiB, lz->buffer needs to be moved forward -/// to avoid integer overflows. We scan the lz->hash array and fix every -/// value to match the updated lz->buffer. -extern void -lzma_lz_encoder_normalize(lzma_lz_encoder *lz) +static bool +lz_encoder_prepare(lzma_mf *mf, lzma_allocator *allocator, + const lzma_lz_options *lz_options) { - const uint32_t subvalue = lz->read_pos - lz->cyclic_buffer_size; - assert(subvalue <= INT32_MAX); + if (lz_options->dictionary_size < LZMA_DICTIONARY_SIZE_MIN + || lz_options->dictionary_size + > LZMA_DICTIONARY_SIZE_MAX + || lz_options->find_len_max + > lz_options->match_len_max) + return true; - { - const uint32_t num_items = lz->num_items; - uint32_t *restrict items = lz->hash; + mf->keep_size_before = lz_options->before_size + + lz_options->dictionary_size; - for (uint32_t i = 0; i < num_items; ++i) { - // If the distance is greater than the dictionary - // size, we can simply mark the item as empty. - if (items[i] <= subvalue) - items[i] = EMPTY_HASH_VALUE; + mf->keep_size_after = lz_options->after_size + + lz_options->match_len_max; + + // To avoid constant memmove()s, allocate some extra space. Since + // memmove()s become more expensive when the size of the buffer + // increases, we reserve more space when a large dictionary is + // used to make the memmove() calls rarer. + uint32_t reserve = lz_options->dictionary_size / 2; + if (reserve > (UINT32_C(1) << 30)) + reserve /= 2; + + reserve += (lz_options->before_size + lz_options->match_len_max + + lz_options->after_size) / 2 + (UINT32_C(1) << 19); + + const uint32_t old_size = mf->size; + mf->size = mf->keep_size_before + reserve + mf->keep_size_after; + + // FIXME Integer overflows + + // Deallocate the old history buffer if it exists but has different + // size than what is needed now. + if (mf->buffer != NULL && old_size != mf->size) { + lzma_free(mf->buffer, allocator); + mf->buffer = NULL; + } + + // Match finder options + mf->match_len_max = lz_options->match_len_max; + mf->find_len_max = lz_options->find_len_max; + mf->cyclic_buffer_size = lz_options->dictionary_size + 1; + + // Validate the match finder ID and setup the function pointers. + switch (lz_options->match_finder) { +#ifdef HAVE_MF_HC3 + case LZMA_MF_HC3: + mf->find = &lzma_mf_hc3_find; + mf->skip = &lzma_mf_hc3_skip; + break; +#endif +#ifdef HAVE_MF_HC4 + case LZMA_MF_HC4: + mf->find = &lzma_mf_hc4_find; + mf->skip = &lzma_mf_hc4_skip; + break; +#endif +#ifdef HAVE_MF_BT2 + case LZMA_MF_BT2: + mf->find = &lzma_mf_bt2_find; + mf->skip = &lzma_mf_bt2_skip; + break; +#endif +#ifdef HAVE_MF_BT3 + case LZMA_MF_BT3: + mf->find = &lzma_mf_bt3_find; + mf->skip = &lzma_mf_bt3_skip; + break; +#endif +#ifdef HAVE_MF_BT4 + case LZMA_MF_BT4: + mf->find = &lzma_mf_bt4_find; + mf->skip = &lzma_mf_bt4_skip; + break; +#endif + + default: + return true; + } + + // Calculate the sizes of mf->hash and mf->son. + const uint32_t hash_bytes = lz_options->match_finder & 0x0F; + const bool is_bt = (lz_options->match_finder & 0x10) != 0; + uint32_t hs; + + if (hash_bytes == 2) { + hs = 0xFFFF; + } else { + // Round dictionary size up to the next 2^n - 1 so it can + // be used as a hash mask. + hs = lz_options->dictionary_size - 1; + hs |= hs >> 1; + hs |= hs >> 2; + hs |= hs >> 4; + hs |= hs >> 8; + hs >>= 1; + hs |= 0xFFFF; + + if (hs > (UINT32_C(1) << 24)) { + if (hash_bytes == 3) + hs = (UINT32_C(1) << 24) - 1; else - items[i] -= subvalue; + hs >>= 1; } } - // Update offset to match the new locations. - lz->offset -= subvalue; + mf->hash_mask = hs; + ++hs; + if (hash_bytes > 2) + hs += HASH_2_SIZE; + if (hash_bytes > 3) + hs += HASH_3_SIZE; +/* + No match finder uses this at the moment. + if (mf->hash_bytes > 4) + hs += HASH_4_SIZE; +*/ + + const uint32_t old_count = mf->hash_size_sum + mf->sons_count; + mf->hash_size_sum = hs; + mf->sons_count = mf->cyclic_buffer_size; + if (is_bt) + mf->sons_count *= 2; + + const uint32_t new_count = mf->hash_size_sum + mf->sons_count; + + // Deallocate the old hash array if it exists and has different size + // than what is needed now. + if (mf->hash != NULL && old_count != new_count) { + lzma_free(mf->hash, allocator); + mf->hash = NULL; + } + + // Maximum number of match finder cycles + mf->loops = lz_options->match_finder_cycles; + if (mf->loops == 0) { + mf->loops = 16 + (lz_options->find_len_max / 2); + if (!is_bt) + mf->loops /= 2; + } + + return false; +} + + +static bool +lz_encoder_init(lzma_mf *mf, lzma_allocator *allocator) +{ + // Allocate the history buffer. + if (mf->buffer == NULL) { + mf->buffer = lzma_alloc(mf->size, allocator); + if (mf->buffer == NULL) + return true; + } + + // Use cyclic_buffer_size as initial mf->offset. This allows + // avoiding a few branches in the match finders. The downside is + // that match finder needs to be normalized more often, which may + // hurt performance with huge dictionaries. + mf->offset = mf->cyclic_buffer_size; + mf->read_pos = 0; + mf->read_ahead = 0; + mf->read_limit = 0; + mf->write_pos = 0; + mf->pending = 0; + + // Allocate match finder's hash array. + const size_t alloc_count = mf->hash_size_sum + mf->sons_count; + +#if UINT32_MAX >= SIZE_MAX / 4 + // Check for integer overflow. (Huge dictionaries are not + // possible on 32-bit CPU.) + if (alloc_count > SIZE_MAX / sizeof(uint32_t)) + return true; +#endif + + if (mf->hash == NULL) { + mf->hash = lzma_alloc(alloc_count * sizeof(uint32_t), + allocator); + if (mf->hash == NULL) + return true; + } + + mf->son = mf->hash + mf->hash_size_sum; + mf->cyclic_buffer_pos = 0; + + // Initialize the hash table. Since EMPTY_HASH_VALUE is zero, we + // can use memset(). +/* + for (uint32_t i = 0; i < hash_size_sum; ++i) + mf->hash[i] = EMPTY_HASH_VALUE; +*/ + memzero(mf->hash, (size_t)(mf->hash_size_sum) * sizeof(uint32_t)); + + // We don't need to initialize mf->son, but not doing that will + // make Valgrind complain in normalization (see normalize() in + // lz_encoder_mf.c). + // + // Skipping this initialization is *very* good when big dictionary is + // used but only small amount of data gets actually compressed: most + // of the mf->hash won't get actually allocated by the kernel, so + // we avoid wasting RAM and improve initialization speed a lot. + //memzero(mf->son, (size_t)(mf->sons_count) * sizeof(uint32_t)); + + mf->action = LZMA_RUN; + + return false; +} + + +extern uint64_t +lzma_lz_encoder_memusage(const lzma_lz_options *lz_options) +{ + // Old buffers must not exist when calling lz_encoder_prepare(). + lzma_mf mf = { + .buffer = NULL, + .hash = NULL, + }; + + // Setup the size information into mf. + if (lz_encoder_prepare(&mf, NULL, lz_options)) + return UINT64_MAX; + + // Calculate the memory usage. + return (uint64_t)(mf.hash_size_sum + mf.sons_count) + * sizeof(uint32_t) + + (uint64_t)(mf.size) + sizeof(lzma_coder); +} + + +static void +lz_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + + lzma_free(coder->mf.hash, allocator); + lzma_free(coder->mf.buffer, allocator); + + if (coder->lz.end != NULL) + coder->lz.end(coder->lz.coder, allocator); + else + lzma_free(coder->lz.coder, allocator); + + lzma_free(coder, allocator); return; } + + +extern lzma_ret +lzma_lz_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_encoder *lz, + lzma_allocator *allocator, const void *options, + lzma_lz_options *lz_options)) +{ + // Allocate and initialize the base data structure. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &lz_encode; + next->end = &lz_encoder_end; + + next->coder->lz.coder = NULL; + next->coder->lz.code = NULL; + next->coder->lz.end = NULL; + + next->coder->mf.buffer = NULL; + next->coder->mf.hash = NULL; + + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Initialize the LZ-based encoder. + lzma_lz_options lz_options; + return_if_error(lz_init(&next->coder->lz, allocator, + filters[0].options, &lz_options)); + + // Setup the size information into next->coder->mf and deallocate + // old buffers if they have wrong size. + if (lz_encoder_prepare(&next->coder->mf, allocator, &lz_options)) + return LZMA_HEADER_ERROR; + + // Allocate new buffers if needed, and do the rest of + // the initialization. + if (lz_encoder_init(&next->coder->mf, allocator)) + return LZMA_MEM_ERROR; + + // Initialize the next filter in the chain, if any. + return lzma_next_filter_init(&next->coder->next, allocator, + filters + 1); +} + + +extern LZMA_API lzma_bool +lzma_mf_is_supported(lzma_match_finder mf) +{ + bool ret = false; + +#ifdef HAVE_MF_HC3 + if (mf == LZMA_MF_HC3) + ret = true; +#endif + +#ifdef HAVE_MF_HC4 + if (mf == LZMA_MF_HC4) + ret = true; +#endif + +#ifdef HAVE_MF_BT2 + if (mf == LZMA_MF_BT2) + ret = true; +#endif + +#ifdef HAVE_MF_BT3 + if (mf == LZMA_MF_BT3) + ret = true; +#endif + +#ifdef HAVE_MF_BT4 + if (mf == LZMA_MF_BT4) + ret = true; +#endif + + return ret; +} diff --git a/src/liblzma/lz/lz_encoder.h b/src/liblzma/lz/lz_encoder.h index da0e0804..45bb8462 100644 --- a/src/liblzma/lz/lz_encoder.h +++ b/src/liblzma/lz/lz_encoder.h @@ -3,8 +3,8 @@ /// \file lz_encoder.h /// \brief LZ in window and match finder API // -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -24,19 +24,16 @@ #include "common.h" -typedef struct lzma_lz_encoder_s lzma_lz_encoder; -struct lzma_lz_encoder_s { - enum { - SEQ_RUN, - SEQ_FLUSH, - SEQ_FINISH, - } sequence; +/// A table of these is used by the LZ-based encoder to hold +/// the length-distance pairs found by the match finder. +typedef struct { + uint32_t len; + uint32_t dist; +} lzma_match; - /// Function to do the actual encoding from the sliding input window - /// to the output stream. - bool (*process)(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size); +typedef struct lzma_mf_s lzma_mf; +struct lzma_mf_s { /////////////// // In Window // /////////////// @@ -46,17 +43,33 @@ struct lzma_lz_encoder_s { /// Total size of the allocated buffer (that is, including all /// the extra space) - size_t size; + uint32_t size; + + /// Number of bytes that must be kept available in our input history. + /// That is, once keep_size_before bytes have been processed, + /// buffer[read_pos - keep_size_before] is the oldest byte that + /// must be available for reading. + uint32_t keep_size_before; + + /// Number of bytes that must be kept in buffer after read_pos. + /// That is, read_pos <= write_pos - keep_size_after as long as + /// stream_end_was_reached is false (once it is true, read_pos + /// is allowed to reach write_pos). + uint32_t keep_size_after; /// Match finders store locations of matches using 32-bit integers. /// To avoid adjusting several megabytes of integers every time the /// input window is moved with move_window(), we only adjust the /// offset of the buffer. Thus, buffer[match_finder_pos - offset] /// is the byte pointed by match_finder_pos. - size_t offset; + uint32_t offset; /// buffer[read_pos] is the current byte. - size_t read_pos; + uint32_t read_pos; + + /// Number of bytes that have been ran through the match finder, but + /// which haven't been encoded by the LZ-based encoder yet. + uint32_t read_ahead; /// As long as read_pos is less than read_limit, there is enough /// input available in buffer for at least one encoding loop. @@ -64,92 +77,253 @@ struct lzma_lz_encoder_s { /// Because of the stateful API, read_limit may and will get greater /// than read_pos quite often. This is taken into account when /// calculating the value for keep_size_after. - size_t read_limit; + uint32_t read_limit; /// buffer[write_pos] is the first byte that doesn't contain valid /// uncompressed data; that is, the next input byte will be copied /// to buffer[write_pos]. - size_t write_pos; + uint32_t write_pos; /// Number of bytes not hashed before read_pos. This is needed to /// restart the match finder after LZMA_SYNC_FLUSH. - size_t pending; - - /// Number of bytes that must be kept available in our input history. - /// That is, once keep_size_before bytes have been processed, - /// buffer[read_pos - keep_size_before] is the oldest byte that - /// must be available for reading. - size_t keep_size_before; - - /// Number of bytes that must be kept in buffer after read_pos. - /// That is, read_pos <= write_pos - keep_size_after as long as - /// stream_end_was_reached is false (once it is true, read_pos - /// is allowed to reach write_pos). - size_t keep_size_after; + uint32_t pending; ////////////////// // Match Finder // ////////////////// - // Pointers to match finder functions - void (*get_matches)(lzma_lz_encoder *restrict lz, - uint32_t *restrict distances); - void (*skip)(lzma_lz_encoder *restrict lz, uint32_t num); + /// Find matches. Returns the number of distance-length pairs written + /// to the matches array. This is called only via lzma_mf_find. + uint32_t (*find)(lzma_mf *mf, lzma_match *matches); - // Match finder data - uint32_t *hash; // TODO: Check if hash aliases son - uint32_t *son; // and add 'restrict' if possible. + /// Skips num bytes. This is like find() but doesn't make the + /// distance-length pairs available, thus being a little faster. + /// This is called only via mf_skip function. + void (*skip)(lzma_mf *mf, uint32_t num); + + uint32_t *hash; + uint32_t *son; uint32_t cyclic_buffer_pos; uint32_t cyclic_buffer_size; // Must be dictionary_size + 1. uint32_t hash_mask; - uint32_t cut_value; + + /// Maximum number of loops in the match finder + uint32_t loops; + + /// Maximum length of a match that the match finder will try to find. + uint32_t find_len_max; + + /// Maximum length of a match supported by the LZ-based encoder. + /// If the longest match found by the match finder is find_len_max, + /// lz_dict_find() tries to expand it up to match_len_max bytes. + uint32_t match_len_max; + + /// When running out of input, binary tree match finders need to know + /// if it is due to flushing or finishing. The action is used also + /// by the LZ-based encoders themselves. + lzma_action action; + + /// Number of elements in hash[] uint32_t hash_size_sum; - uint32_t num_items; - uint32_t match_max_len; + + /// Number of elements in son[] + uint32_t sons_count; }; -#define LZMA_LZ_ENCODER_INIT \ - (lzma_lz_encoder){ \ - .buffer = NULL, \ - .size = 0, \ - .hash = NULL, \ - .num_items = 0, \ +typedef struct { + /// Extra amount of data to keep available before the "actual" + /// dictionary. + size_t before_size; + + /// Size of the history buffer + size_t dictionary_size; + + /// Extra amount of data to keep available after the "actual" + /// dictionary. + size_t after_size; + + /// Maximum length of a match that the LZ-based encoder can accept. + /// This is used to extend matches of length find_len_max to the + /// maximum possible length. + size_t match_len_max; + + /// Match finder will search matches of at maximum of this length. + /// This must be less than or equal to match_len_max. + size_t find_len_max; + + /// Type of the match finder to use + lzma_match_finder match_finder; + + /// TODO: Comment + uint32_t match_finder_cycles; + + /// TODO: Comment + const uint8_t *preset_dictionary; + + uint32_t preset_dictionary_size; + +} lzma_lz_options; + + +// The total usable buffer space at any moment outside the match finder: +// before_size + dictionary_size + after_size + match_len_max +// +// In reality, there's some extra space allocated to prevent the number of +// memmove() calls reasonable. The bigger the dictionary_size is, the bigger +// this extra buffer will be since with bigger dictionaries memmove() would +// also take longer. +// +// A single encoder loop in the LZ-based encoder may call the match finder +// (lz_dict_find() or lz_dict_skip()) at maximum of after_size times. +// In other words, a single encoder loop may advance lz_dict.read_pos at +// maximum of after_size times. Since matches are looked up to +// lz_dict.buffer[lz_dict.read_pos + match_len_max - 1], the total +// amount of extra buffer needed after dictionary_size becomes +// after_size + match_len_max. +// +// before_size has two uses. The first one is to keep literals available +// in cases when the LZ-based encoder has made some read ahead. +// TODO: Maybe this could be changed by making the LZ-based encoders to +// store the actual literals as they do with length-distance pairs. +// +// Alrogithms such as LZMA2 first try to compress a chunk, and then check +// if the encoded result is smaller than the uncompressed one. If the chunk +// was uncompressible, it is better to store it in uncompressed form in +// the output stream. To do this, the whole uncompressed chunk has to be +// still available in the history buffer. before_size achieves that. + + +typedef struct { + /// Data specific to the LZ-based encoder + lzma_coder *coder; + + /// Function to encode from *dict to out[] + lzma_ret (*code)(lzma_coder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); + + /// Free allocated resources + void (*end)(lzma_coder *coder, lzma_allocator *allocator); + +} lzma_lz_encoder; + + +// Basic steps: +// 1. Input gets copied into the dictionary. +// 2. Data in dictionary gets run through the match finder byte by byte. +// 3. The literals and matches are encoded using e.g. LZMA. +// +// The bytes that have been ran through the match finder, but not encoded yet, +// are called `read ahead'. + + +/// Get pointer to the first byte not ran through the match finder +static inline const uint8_t * +mf_ptr(const lzma_mf *mf) +{ + return mf->buffer + mf->read_pos; +} + + +/// Get the number of bytes that haven't been ran through the match finder yet. +static inline uint32_t +mf_avail(const lzma_mf *mf) +{ + return mf->write_pos - mf->read_pos; +} + + +/// Get the number of bytes that haven't been encoded yet (some of these +/// bytes may have been ran through the match finder though). +static inline uint32_t +mf_unencoded(const lzma_mf *mf) +{ + return mf->write_pos - mf->read_pos - mf->read_ahead; +} + + +/// Calculate the absolute offset from the beginning of the most recent +/// dictionary reset. Only the lowest four bits are important, so there's no +/// problem that we don't know the 64-bit size of the data encoded so far. +/// +/// NOTE: When moving the input window, we need to do it so that the lowest +/// bits of dict->read_pos are not modified to keep this macro working +/// as intended. +static inline uint32_t +mf_position(const lzma_mf *mf) +{ + return mf->read_pos - mf->read_ahead; +} + + +/// Since everything else begins with mf_, use it also for lzma_mf_find(). +#define mf_find lzma_mf_find + + +/// Skip the given number of bytes. This is used when a good match was found. +/// For example, if mf_find() finds a match of 200 bytes long, the first byte +/// of that match was already consumed by mf_find(), and the rest 199 bytes +/// have to be skipped with mf_skip(mf, 199). +static inline void +mf_skip(lzma_mf *mf, uint32_t amount) +{ + if (amount != 0) { + mf->skip(mf, amount); + mf->read_ahead += amount; } +} -/// Calculates -extern bool lzma_lz_encoder_hash_properties(lzma_match_finder match_finder, - uint32_t history_size, uint32_t *restrict hash_mask, - uint32_t *restrict hash_size_sum, - uint32_t *restrict num_items); +/// Copies at maximum of *left amount of bytes from the history buffer +/// to out[]. This is needed by LZMA2 to encode uncompressed chunks. +static inline void +mf_read(lzma_mf *mf, uint8_t *out, size_t *out_pos, size_t out_size, + size_t *left) +{ + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(out_avail, *left); -// NOTE: liblzma doesn't use callback API like LZMA SDK does. The caller -// must make sure that keep_size_after is big enough for single encoding pass -// i.e. keep_size_after >= maximum number of bytes possibly needed after -// the current position between calls to lzma_lz_read(). -extern lzma_ret lzma_lz_encoder_reset(lzma_lz_encoder *lz, - lzma_allocator *allocator, - bool (*process)(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size), - size_t history_size, size_t additional_buffer_before, - size_t match_max_len, size_t additional_buffer_after, - lzma_match_finder match_finder, uint32_t match_finder_cycles, - const uint8_t *preset_dictionary, - size_t preset_dictionary_size); + assert(mf->read_ahead == 0); + assert(mf->read_pos >= *left); -/// Frees memory allocated for in window and match finder buffers. -extern void lzma_lz_encoder_end( - lzma_lz_encoder *lz, lzma_allocator *allocator); + memcpy(out + *out_pos, mf->buffer + mf->read_pos - *left, + copy_size); -extern lzma_ret lzma_lz_encode(lzma_coder *coder, - lzma_allocator *allocator lzma_attribute((unused)), - const uint8_t *restrict in, size_t *restrict in_pos, - size_t in_size, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size, - lzma_action action); + *out_pos += copy_size; + *left -= copy_size; + return; +} -/// This should not be called directly, but only via move_pos() macro. -extern void lzma_lz_encoder_normalize(lzma_lz_encoder *lz); + +extern lzma_ret lzma_lz_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters, + lzma_ret (*lz_init)(lzma_lz_encoder *lz, + lzma_allocator *allocator, const void *options, + lzma_lz_options *lz_options)); + + +extern uint64_t lzma_lz_encoder_memusage(const lzma_lz_options *lz_options); + + +// These are only for LZ encoder's internal use. +extern uint32_t lzma_mf_find( + lzma_mf *mf, uint32_t *count, lzma_match *matches); + +extern uint32_t lzma_mf_hc3_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_hc3_skip(lzma_mf *dict, uint32_t amount); + +extern uint32_t lzma_mf_hc4_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_hc4_skip(lzma_mf *dict, uint32_t amount); + +extern uint32_t lzma_mf_bt2_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_bt2_skip(lzma_mf *dict, uint32_t amount); + +extern uint32_t lzma_mf_bt3_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_bt3_skip(lzma_mf *dict, uint32_t amount); + +extern uint32_t lzma_mf_bt4_find(lzma_mf *dict, lzma_match *matches); +extern void lzma_mf_bt4_skip(lzma_mf *dict, uint32_t amount); #endif diff --git a/src/liblzma/lz/lz_encoder_hash.h b/src/liblzma/lz/lz_encoder_hash.h new file mode 100644 index 00000000..0841c38f --- /dev/null +++ b/src/liblzma/lz/lz_encoder_hash.h @@ -0,0 +1,104 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder_hash.h +/// \brief Hash macros for match finders +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZ_ENCODER_HASH_H +#define LZMA_LZ_ENCODER_HASH_H + +#define HASH_2_SIZE (UINT32_C(1) << 10) +#define HASH_3_SIZE (UINT32_C(1) << 16) +#define HASH_4_SIZE (UINT32_C(1) << 20) + +#define HASH_2_MASK (HASH_2_SIZE - 1) +#define HASH_3_MASK (HASH_3_SIZE - 1) +#define HASH_4_MASK (HASH_4_SIZE - 1) + +#define FIX_3_HASH_SIZE (HASH_2_SIZE) +#define FIX_4_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE) +#define FIX_5_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE + HASH_4_SIZE) + +// TODO Benchmark, and probably doesn't need to be endian dependent. +#if !defined(WORDS_BIGENDIAN) && defined(HAVE_FAST_UNALIGNED_ACCESS) +# define hash_2_calc() \ + const uint32_t hash_value = *(const uint16_t *)(cur); +#else +# define hash_2_calc() \ + const uint32_t hash_value \ + = (uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8) +#endif + +#define hash_3_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & mf->hash_mask + +#define hash_4_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ + ^ (lzma_crc32_table[0][cur[3]] << 5)) & mf->hash_mask + + +// The following are not currently used. + +#define hash_5_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + uint32_t hash_4_value = (temp ^ ((uint32_t)(cur[2]) << 8) ^ \ + ^ lzma_crc32_table[0][cur[3]] << 5); \ + const uint32_t hash_value \ + = (hash_4_value ^ (lzma_crc32_table[0][cur[4]] << 3)) \ + & mf->hash_mask; \ + hash_4_value &= HASH_4_MASK + +/* +#define hash_zip_calc() \ + const uint32_t hash_value \ + = (((uint32_t)(cur[0]) | ((uint32_t)(cur[1]) << 8)) \ + ^ lzma_crc32_table[0][cur[2]]) & 0xFFFF +*/ + +#define hash_zip_calc() \ + const uint32_t hash_value \ + = (((uint32_t)(cur[2]) | ((uint32_t)(cur[0]) << 8)) \ + ^ lzma_crc32_table[0][cur[1]]) & 0xFFFF + +#define mt_hash_2_calc() \ + const uint32_t hash_2_value \ + = (lzma_crc32_table[0][cur[0]] ^ cur[1]) & HASH_2_MASK + +#define mt_hash_3_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK + +#define mt_hash_4_calc() \ + const uint32_t temp = lzma_crc32_table[0][cur[0]] ^ cur[1]; \ + const uint32_t hash_2_value = temp & HASH_2_MASK; \ + const uint32_t hash_3_value \ + = (temp ^ ((uint32_t)(cur[2]) << 8)) & HASH_3_MASK; \ + const uint32_t hash_4_value = (temp ^ ((uint32_t)(cur[2]) << 8) ^ \ + (lzma_crc32_table[0][cur[3]] << 5)) & HASH_4_MASK + +#endif diff --git a/src/liblzma/lz/lz_encoder_mf.c b/src/liblzma/lz/lz_encoder_mf.c new file mode 100644 index 00000000..b1c20f50 --- /dev/null +++ b/src/liblzma/lz/lz_encoder_mf.c @@ -0,0 +1,780 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lz_encoder_mf.c +/// \brief Match finders +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_encoder.h" +#include "lz_encoder_hash.h" +#include "check.h" + + +/// \brief Find matches starting from the current byte +/// +/// \return The length of the longest match found +extern uint32_t +lzma_mf_find(lzma_mf *mf, uint32_t *count_ptr, lzma_match *matches) +{ + // Call the match finder. It returns the number of length-distance + // pairs found. + // FIXME: Minimum count is zero, what _exactly_ is the maximum? + const uint32_t count = mf->find(mf, matches); + + // Length of the longest match; assume that no matches were found + // and thus the maximum length is zero. + uint32_t len_best = 0; + + if (count > 0) { +#ifndef NDEBUG + // Validate the matches. + for (uint32_t i = 0; i < count; ++i) { + assert(matches[i].len <= mf->find_len_max); + assert(matches[i].dist < mf->read_pos); + assert(memcmp(mf_ptr(mf) - 1, + mf_ptr(mf) - matches[i].dist - 2, + matches[i].len) == 0); + } +#endif + + // The last used element in the array contains + // the longest match. + len_best = matches[count - 1].len; + + // If a match of maximum search length was found, try to + // extend the match to maximum possible length. + if (len_best == mf->find_len_max) { + // The limit for the match length is either the + // maximum match length supported by the LZ-based + // encoder or the number of bytes left in the + // dictionary, whichever is smaller. + uint32_t limit = mf_avail(mf) + 1; + if (limit > mf->match_len_max) + limit = mf->match_len_max; + + // Pointer to the byte we just ran through + // the match finder. + const uint8_t *p1 = mf_ptr(mf) - 1; + + // Pointer to the beginning of the match. We need -1 + // here because the match distances are zero based. + const uint8_t *p2 = p1 - matches[count - 1].dist - 1; + + while (len_best < limit + && p1[len_best] == p2[len_best]) + ++len_best; + } + } + + *count_ptr = count; + + // Finally update the read position to indicate that match finder was + // run for this dictionary offset. + ++mf->read_ahead; + + return len_best; +} + + +/// Hash value to indicate unused element in the hash. Since we start the +/// positions from dictionary_size + 1, zero is always too far to qualify +/// as usable match position. +#define EMPTY_HASH_VALUE 0 + + +/// Normalization must be done when lzma_mf.offset + lzma_mf.read_pos +/// reaches MUST_NORMALIZE_POS. +#define MUST_NORMALIZE_POS UINT32_MAX + + +/// \brief Normalizes hash values +/// +/// The hash arrays store positions of match candidates. The positions are +/// relative to an arbitrary offset that is not the same as the absolute +/// offset in the input stream. The relative position of the current byte +/// is lzma_mf.offset + lzma_mf.read_pos. The distances of the matches are +/// the differences of the current read position and the position found from +/// the hash. +/// +/// To prevent integer overflows of the offsets stored in the hash arrays, +/// we need to "normalize" the stored values now and then. During the +/// normalization, we drop values that indicate distance greater than the +/// dictionary size, thus making space for new values. +static void +normalize(lzma_mf *mf) +{ + assert(mf->read_pos + mf->offset == MUST_NORMALIZE_POS); + + // In future we may not want to touch the lowest bits, because there + // may be match finders that use larger resolution than one byte. + const uint32_t subvalue + = (MUST_NORMALIZE_POS - mf->cyclic_buffer_size); + // & (~(UINT32_C(1) << 10) - 1); + + const uint32_t count = mf->hash_size_sum + mf->sons_count; + uint32_t *hash = mf->hash; + + for (uint32_t i = 0; i < count; ++i) { + // If the distance is greater than the dictionary size, + // we can simply mark the hash element as empty. + // + // NOTE: Only the first mf->hash_size_sum elements are + // initialized for sure. There may be uninitialized elements + // in mf->son. Since we go through both mf->hash and + // mf->son here in normalization, Valgrind may complain + // that the "if" below depends on uninitialized value. In + // this case it is safe to ignore the warning. See also the + // comments in lz_encoder_init() in lz_encoder.c. + if (hash[i] <= subvalue) + hash[i] = EMPTY_HASH_VALUE; + else + hash[i] -= subvalue; + } + + // Update offset to match the new locations. + mf->offset -= subvalue; + + return; +} + + +/// Mark the current byte as processed from point of view of the match finder. +static void +move_pos(lzma_mf *mf) +{ + if (++mf->cyclic_buffer_pos == mf->cyclic_buffer_size) + mf->cyclic_buffer_pos = 0; + + ++mf->read_pos; + assert(mf->read_pos <= mf->write_pos); + + if (unlikely(mf->read_pos + mf->offset == UINT32_MAX)) + normalize(mf); +} + + +/// When flushing, we cannot run the match finder unless there is find_len_max +/// bytes available in the dictionary. Instead, we skip running the match +/// finder (indicating that no match was found), and count how many bytes we +/// have ignored this way. +/// +/// When new data is given after the flushing was completed, the match finder +/// is restarted by rewinding mf->read_pos backwards by mf->pending. Then +/// the missed bytes are added to the hash using the match finder's skip +/// function (with small amount of input, it may start using mf->pending +/// again if flushing). +/// +/// Due to this rewinding, we don't touch cyclic_buffer_pos or test for +/// normalization. It will be done when the match finder's skip function +/// catches up after a flush. +static void +move_pending(lzma_mf *mf) +{ + ++mf->read_pos; + assert(mf->read_pos <= mf->write_pos); + ++mf->pending; +} + + +/// Calculate len_limit and determine if there is enough input to run +/// the actual match finder code. Sets up "cur" and "pos". This macro +/// is used by all find functions and binary tree skip functions. Hash +/// chain skip function doesn't need len_limit so a simpler code is used +/// in them. +#define header(is_bt, len_min, ret_op) \ + uint32_t len_limit = mf_avail(mf); \ + if (mf->find_len_max <= len_limit) { \ + len_limit = mf->find_len_max; \ + } else if (len_limit < (len_min) \ + || (is_bt && mf->action == LZMA_SYNC_FLUSH)) { \ + assert(mf->action != LZMA_RUN); \ + move_pending(mf); \ + ret_op; \ + } \ + const uint8_t *cur = mf_ptr(mf); \ + const uint32_t pos = mf->read_pos + mf->offset + + +/// Header for find functions. "return 0" indicates that zero matches +/// were found. +#define header_find(is_bt, len_min) \ + header(is_bt, len_min, return 0); \ + uint32_t matches_count = 0 + + +/// Header for a loop in a skip function. "continue" tells to skip the rest +/// of the code in the loop. +#define header_skip(is_bt, len_min) \ + header(is_bt, len_min, continue) + + +/// Calls hc_find_func() or bt_find_func() and calculates the total number +/// of matches found. Updates the dictionary position and returns the number +/// of matches found. +#define call_find(func, len_best) \ +do { \ + matches_count = func(len_limit, pos, cur, cur_match, mf->loops, \ + mf->son, mf->cyclic_buffer_pos, \ + mf->cyclic_buffer_size, \ + matches + matches_count, len_best) \ + - matches; \ + move_pos(mf); \ + return matches_count; \ +} while (0) + + +//////////////// +// Hash Chain // +//////////////// + +#if defined(HAVE_MF_HC3) || defined(HAVE_MF_HC4) +/// +/// +/// \param len_limit Don't look for matches longer than len_limit. +/// \param pos lzma_mf.read_pos + lzma_mf.offset +/// \param cur Pointer to current byte (lzma_dict_ptr(mf)) +/// \param cur_match Start position of the current match candidate +/// \param loops Maximum length of the hash chain +/// \param son lzma_mf.son (contains the hash chain) +/// \param cyclic_buffer_pos +/// \param cyclic_buffer_size +/// \param matches Array to hold the matches. +/// \param len_best The length of the longest match found so far. +static lzma_match * +hc_find_func( + const uint32_t len_limit, + const uint32_t pos, + const uint8_t *const cur, + uint32_t cur_match, + uint32_t loops, + uint32_t *const son, + const uint32_t cyclic_buffer_pos, + const uint32_t cyclic_buffer_size, + lzma_match *matches, + uint32_t len_best) +{ + son[cyclic_buffer_pos] = cur_match; + + while (true) { + const uint32_t delta = pos - cur_match; + if (loops-- == 0 || delta >= cyclic_buffer_size) + return matches; + + const uint8_t *const pb = cur - delta; + cur_match = son[cyclic_buffer_pos - delta + + (delta > cyclic_buffer_pos + ? cyclic_buffer_size : 0)]; + + if (pb[len_best] == cur[len_best] && pb[0] == cur[0]) { + uint32_t len = 0; + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len_best < len) { + len_best = len; + matches->len = len; + matches->dist = delta - 1; + ++matches; + + if (len == len_limit) + return matches; + } + } + } +} + +/* +#define hc_header_find(len_min, ret_op) \ + uint32_t len_limit = mf_avail(mf); \ + if (mf->find_len_max <= len_limit) { \ + len_limit = mf->find_len_max; \ + } else if (len_limit < (len_min)) { \ + move_pending(mf); \ + ret_op; \ + } \ +#define header_hc(len_min, ret_op) \ +do { \ + if (mf_avail(mf) < (len_min)) { \ + move_pending(mf); \ + ret_op; \ + } \ +} while (0) +*/ + +#define hc_find(len_best) \ + call_find(hc_find_func, len_best) + + +#define hc_skip() \ +do { \ + mf->son[mf->cyclic_buffer_pos] = cur_match; \ + move_pos(mf); \ +} while (0) + +#endif + + +#ifdef HAVE_MF_HC3 +extern uint32_t +lzma_mf_hc3_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(false, 3); + + hash_3_calc(); + + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 2; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[0].len = len_best; + matches[0].dist = delta2 - 1; + matches_count = 1; + + if (len_best == len_limit) { + hc_skip(); + return 1; // matches_count + } + } + + hc_find(len_best); +} + + +extern void +lzma_mf_hc3_skip(lzma_mf *mf, uint32_t amount) +{ + do { + if (mf_avail(mf) < 3) { + move_pending(mf); + continue; + } + + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; + + hash_3_calc(); + + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + hc_skip(); + + } while (--amount != 0); +} +#endif + + +#ifdef HAVE_MF_HC4 +extern uint32_t +lzma_mf_hc4_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(false, 4); + + hash_4_calc(); + + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value ] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 1; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + len_best = 2; + matches[0].len = 2; + matches[0].dist = delta2 - 1; + matches_count = 1; + } + + if (delta2 != delta3 && delta3 < mf->cyclic_buffer_size + && *(cur - delta3) == *cur) { + len_best = 3; + matches[matches_count++].dist = delta3 - 1; + delta2 = delta3; + } + + if (matches_count != 0) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[matches_count - 1].len = len_best; + + if (len_best == len_limit) { + hc_skip(); + return matches_count; + } + } + + if (len_best < 3) + len_best = 3; + + hc_find(len_best); +} + + +extern void +lzma_mf_hc4_skip(lzma_mf *mf, uint32_t amount) +{ + do { + if (mf_avail(mf) < 4) { + move_pending(mf); + continue; + } + + const uint8_t *cur = mf_ptr(mf); + const uint32_t pos = mf->read_pos + mf->offset; + + hash_4_calc(); + + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + hc_skip(); + + } while (--amount != 0); +} +#endif + + +///////////////// +// Binary Tree // +///////////////// + +#if defined(HAVE_MF_BT2) || defined(HAVE_MF_BT3) || defined(HAVE_MF_BT4) +static lzma_match * +bt_find_func( + const uint32_t len_limit, + const uint32_t pos, + const uint8_t *const cur, + uint32_t cur_match, + uint32_t loops, + uint32_t *const son, + const uint32_t cyclic_buffer_pos, + const uint32_t cyclic_buffer_size, + lzma_match *matches, + uint32_t len_best) +{ + uint32_t *ptr0 = son + (cyclic_buffer_pos << 1) + 1; + uint32_t *ptr1 = son + (cyclic_buffer_pos << 1); + + uint32_t len0 = 0; + uint32_t len1 = 0; + + while (true) { + const uint32_t delta = pos - cur_match; + if (loops-- == 0 || delta >= cyclic_buffer_size) { + *ptr0 = EMPTY_HASH_VALUE; + *ptr1 = EMPTY_HASH_VALUE; + return matches; + } + + uint32_t *const pair = son + ((cyclic_buffer_pos - delta + + (delta > cyclic_buffer_pos + ? cyclic_buffer_size : 0)) << 1); + + const uint8_t *const pb = cur - delta; + uint32_t len = MIN(len0, len1); + + if (pb[len] == cur[len]) { + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len_best < len) { + len_best = len; + matches->len = len; + matches->dist = delta - 1; + ++matches; + + if (len == len_limit) { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + return matches; + } + } + } + + if (pb[len] < cur[len]) { + *ptr1 = cur_match; + ptr1 = pair + 1; + cur_match = *ptr1; + len1 = len; + } else { + *ptr0 = cur_match; + ptr0 = pair; + cur_match = *ptr0; + len0 = len; + } + } +} + + +static void +bt_skip_func( + const uint32_t len_limit, + const uint32_t pos, + const uint8_t *const cur, + uint32_t cur_match, + uint32_t loops, + uint32_t *const son, + const uint32_t cyclic_buffer_pos, + const uint32_t cyclic_buffer_size) +{ + uint32_t *ptr0 = son + (cyclic_buffer_pos << 1) + 1; + uint32_t *ptr1 = son + (cyclic_buffer_pos << 1); + + uint32_t len0 = 0; + uint32_t len1 = 0; + + while (true) { + const uint32_t delta = pos - cur_match; + if (loops-- == 0 || delta >= cyclic_buffer_size) { + *ptr0 = EMPTY_HASH_VALUE; + *ptr1 = EMPTY_HASH_VALUE; + return; + } + + uint32_t *pair = son + ((cyclic_buffer_pos - delta + + (delta > cyclic_buffer_pos + ? cyclic_buffer_size : 0)) << 1); + const uint8_t *pb = cur - delta; + uint32_t len = MIN(len0, len1); + + if (pb[len] == cur[len]) { + while (++len != len_limit) + if (pb[len] != cur[len]) + break; + + if (len == len_limit) { + *ptr1 = pair[0]; + *ptr0 = pair[1]; + return; + } + } + + if (pb[len] < cur[len]) { + *ptr1 = cur_match; + ptr1 = pair + 1; + cur_match = *ptr1; + len1 = len; + } else { + *ptr0 = cur_match; + ptr0 = pair; + cur_match = *ptr0; + len0 = len; + } + } +} + + +#define bt_find(len_best) \ + call_find(bt_find_func, len_best) + +#define bt_skip() \ +do { \ + bt_skip_func(len_limit, pos, cur, cur_match, mf->loops, \ + mf->son, mf->cyclic_buffer_pos, \ + mf->cyclic_buffer_size); \ + move_pos(mf); \ +} while (0) + +#endif + + +#ifdef HAVE_MF_BT2 +extern uint32_t +lzma_mf_bt2_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(true, 2); + + hash_2_calc(); + + const uint32_t cur_match = mf->hash[hash_value]; + mf->hash[hash_value] = pos; + + bt_find(1); +} + + +extern void +lzma_mf_bt2_skip(lzma_mf *mf, uint32_t amount) +{ + do { + header_skip(true, 2); + + hash_2_calc(); + + const uint32_t cur_match = mf->hash[hash_value]; + mf->hash[hash_value] = pos; + + bt_skip(); + + } while (--amount != 0); +} +#endif + + +#ifdef HAVE_MF_BT3 +extern uint32_t +lzma_mf_bt3_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(true, 3); + + hash_3_calc(); + + const uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t cur_match = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 2; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[0].len = len_best; + matches[0].dist = delta2 - 1; + matches_count = 1; + + if (len_best == len_limit) { + bt_skip(); + return 1; // matches_count + } + } + + bt_find(len_best); +} + + +extern void +lzma_mf_bt3_skip(lzma_mf *mf, uint32_t amount) +{ + do { + header_skip(true, 3); + + hash_3_calc(); + + const uint32_t cur_match + = mf->hash[FIX_3_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_value] = pos; + + bt_skip(); + + } while (--amount != 0); +} +#endif + + +#ifdef HAVE_MF_BT4 +extern uint32_t +lzma_mf_bt4_find(lzma_mf *mf, lzma_match *matches) +{ + header_find(true, 4); + + hash_4_calc(); + + uint32_t delta2 = pos - mf->hash[hash_2_value]; + const uint32_t delta3 + = pos - mf->hash[FIX_3_HASH_SIZE + hash_3_value]; + const uint32_t cur_match = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + uint32_t len_best = 1; + + if (delta2 < mf->cyclic_buffer_size && *(cur - delta2) == *cur) { + len_best = 2; + matches[0].len = 2; + matches[0].dist = delta2 - 1; + matches_count = 1; + } + + if (delta2 != delta3 && delta3 < mf->cyclic_buffer_size + && *(cur - delta3) == *cur) { + len_best = 3; + matches[matches_count++].dist = delta3 - 1; + delta2 = delta3; + } + + if (matches_count != 0) { + for ( ; len_best != len_limit; ++len_best) + if (*(cur + len_best - delta2) != cur[len_best]) + break; + + matches[matches_count - 1].len = len_best; + + if (len_best == len_limit) { + bt_skip(); + return matches_count; + } + } + + if (len_best < 3) + len_best = 3; + + bt_find(len_best); +} + + +extern void +lzma_mf_bt4_skip(lzma_mf *mf, uint32_t amount) +{ + do { + header_skip(true, 4); + + hash_4_calc(); + + const uint32_t cur_match + = mf->hash[FIX_4_HASH_SIZE + hash_value]; + + mf->hash[hash_2_value] = pos; + mf->hash[FIX_3_HASH_SIZE + hash_3_value] = pos; + mf->hash[FIX_4_HASH_SIZE + hash_value] = pos; + + bt_skip(); + + } while (--amount != 0); +} +#endif diff --git a/src/liblzma/lz/match_c.h b/src/liblzma/lz/match_c.h deleted file mode 100644 index 664db290..00000000 --- a/src/liblzma/lz/match_c.h +++ /dev/null @@ -1,412 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file match_c.h -/// \brief Template for different match finders -/// -/// This file is included by hc3.c, hc4, bt2.c, bt3.c and bt4.c. Each file -/// sets slighly different #defines, resulting the different match finders. -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -////////////// -// Includes // -////////////// - -#include "check.h" - - -/////////////// -// Constants // -/////////////// - -#define START_MAX_LEN 1 - -#ifdef HASH_ARRAY_2 -# define NUM_HASH_DIRECT_BYTES 0 -# define HASH_2_SIZE (1 << 10) -# ifdef HASH_ARRAY_3 -# define NUM_HASH_BYTES 4 -# define HASH_3_SIZE (1 << 16) -# define HASH_3_OFFSET HASH_2_SIZE -# define FIX_HASH_SIZE (HASH_2_SIZE + HASH_3_SIZE) -# else -# define NUM_HASH_BYTES 3 -# define FIX_HASH_SIZE HASH_2_SIZE -# endif -# define HASH_SIZE 0 -# define MIN_MATCH_CHECK NUM_HASH_BYTES -#else -# define NUM_HASH_DIRECT_BYTES 2 -# define NUM_HASH_BYTES 2 -# define HASH_SIZE (1 << (8 * NUM_HASH_BYTES)) -# define MIN_MATCH_CHECK (NUM_HASH_BYTES + 1) -# define FIX_HASH_SIZE 0 -#endif - - -//////////// -// Macros // -//////////// - -#ifdef HASH_ARRAY_2 -# ifdef HASH_ARRAY_3 -# define HASH_CALC() \ - do { \ - const uint32_t temp = lzma_crc32_table[0][ \ - cur[0]] ^ cur[1]; \ - hash_2_value = temp & (HASH_2_SIZE - 1); \ - hash_3_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \ - & (HASH_3_SIZE - 1); \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8) \ - ^ (lzma_crc32_table[0][cur[3]] << 5)) \ - & lz->hash_mask; \ - } while (0) -# else -# define HASH_CALC() \ - do { \ - const uint32_t temp = lzma_crc32_table[0][ \ - cur[0]] ^ cur[1]; \ - hash_2_value = temp & (HASH_2_SIZE - 1); \ - hash_value = (temp ^ ((uint32_t)(cur[2]) << 8)) \ - & lz->hash_mask; \ - } while (0) -# endif -#else -# define HASH_CALC() hash_value = cur[0] ^ ((uint32_t)(cur[1]) << 8) -#endif - - -// Moves the current read position forward by one byte. In LZMA SDK, -// CLZInWindow::MovePos() can read more input data if needed, because of -// the callback style API. In liblzma we must have ensured earlier, that -// there is enough data available in lz->buffer. -#define move_pos() \ -do { \ - if (++lz->cyclic_buffer_pos == lz->cyclic_buffer_size) \ - lz->cyclic_buffer_pos = 0; \ - ++lz->read_pos; \ - assert(lz->read_pos <= lz->write_pos); \ - if (lz->read_pos == MAX_VAL_FOR_NORMALIZE) \ - lzma_lz_encoder_normalize(lz); \ -} while (0) - - -#define move_pending() \ -do { \ - ++lz->read_pos; \ - assert(lz->read_pos <= lz->write_pos); \ - ++lz->pending; \ -} while (0) - - -////////////////////// -// Global constants // -////////////////////// - -LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = HASH_SIZE; -LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER) = FIX_HASH_SIZE; - - -/////////////////// -// API functions // -/////////////////// - -LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER) -{ - uint32_t len_limit; - if (lz->read_pos + lz->match_max_len <= lz->write_pos) { - len_limit = lz->match_max_len; - } else { - len_limit = lz->write_pos - lz->read_pos; - if (len_limit < MIN_MATCH_CHECK || lz->sequence == SEQ_FLUSH) { - distances[0] = 0; - move_pending(); - return; - } - } - - assert(lz->pending == 0); - - int32_t offset = 1; - const uint32_t match_min_pos - = lz->read_pos + lz->offset > lz->cyclic_buffer_size - ? lz->read_pos + lz->offset - lz->cyclic_buffer_size - : 0; - const uint8_t *cur = lz->buffer + lz->read_pos; - uint32_t max_len = START_MAX_LEN; // to avoid items for len < hash_size - -#ifdef HASH_ARRAY_2 - uint32_t hash_2_value; -# ifdef HASH_ARRAY_3 - uint32_t hash_3_value; -# endif -#endif - uint32_t hash_value; - HASH_CALC(); - - uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value]; -#ifdef HASH_ARRAY_2 - uint32_t cur_match2 = lz->hash[hash_2_value]; -# ifdef HASH_ARRAY_3 - uint32_t cur_match3 = lz->hash[HASH_3_OFFSET + hash_3_value]; -# endif - lz->hash[hash_2_value] = lz->read_pos + lz->offset; - - if (cur_match2 > match_min_pos) { - if (lz->buffer[cur_match2 - lz->offset] == cur[0]) { - max_len = 2; - distances[offset++] = 2; - distances[offset++] = lz->read_pos + lz->offset - - cur_match2 - 1; - } - } - -# ifdef HASH_ARRAY_3 - lz->hash[HASH_3_OFFSET + hash_3_value] = lz->read_pos + lz->offset; - if (cur_match3 > match_min_pos) { - if (lz->buffer[cur_match3 - lz->offset] == cur[0]) { - if (cur_match3 == cur_match2) - offset -= 2; - - max_len = 3; - distances[offset++] = 3; - distances[offset++] = lz->read_pos + lz->offset - - cur_match3 - 1; - cur_match2 = cur_match3; - } - } -# endif - - if (offset != 1 && cur_match2 == cur_match) { - offset -= 2; - max_len = START_MAX_LEN; - } -#endif - - lz->hash[FIX_HASH_SIZE + hash_value] = lz->read_pos + lz->offset; - -#ifdef IS_HASH_CHAIN - lz->son[lz->cyclic_buffer_pos] = cur_match; -#else - uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1; - uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1); - - uint32_t len0 = NUM_HASH_DIRECT_BYTES; - uint32_t len1 = NUM_HASH_DIRECT_BYTES; -#endif - -#if NUM_HASH_DIRECT_BYTES != 0 - if (cur_match > match_min_pos) { - if (lz->buffer[cur_match + NUM_HASH_DIRECT_BYTES - lz->offset] - != cur[NUM_HASH_DIRECT_BYTES]) { - max_len = NUM_HASH_DIRECT_BYTES; - distances[offset++] = NUM_HASH_DIRECT_BYTES; - distances[offset++] = lz->read_pos + lz->offset - - cur_match - 1; - } - } -#endif - - uint32_t count = lz->cut_value; - - while (true) { - if (cur_match <= match_min_pos || count-- == 0) { -#ifndef IS_HASH_CHAIN - *ptr0 = EMPTY_HASH_VALUE; - *ptr1 = EMPTY_HASH_VALUE; -#endif - break; - } - - const uint32_t delta = lz->read_pos + lz->offset - cur_match; - const uint32_t cyclic_pos = delta <= lz->cyclic_buffer_pos - ? lz->cyclic_buffer_pos - delta - : lz->cyclic_buffer_pos - delta - + lz->cyclic_buffer_size; - uint32_t *pair = lz->son + -#ifdef IS_HASH_CHAIN - cyclic_pos; -#else - (cyclic_pos << 1); -#endif - - const uint8_t *pb = lz->buffer + cur_match - lz->offset; - uint32_t len = -#ifdef IS_HASH_CHAIN - NUM_HASH_DIRECT_BYTES; - if (pb[max_len] == cur[max_len]) -#else - MIN(len0, len1); -#endif - - if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; - - if (max_len < len) { - max_len = len; - distances[offset++] = len; - distances[offset++] = delta - 1; - if (len == len_limit) { -#ifndef IS_HASH_CHAIN - *ptr1 = pair[0]; - *ptr0 = pair[1]; -#endif - break; - } - } - } - -#ifdef IS_HASH_CHAIN - cur_match = *pair; -#else - if (pb[len] < cur[len]) { - *ptr1 = cur_match; - ptr1 = pair + 1; - cur_match = *ptr1; - len1 = len; - } else { - *ptr0 = cur_match; - ptr0 = pair; - cur_match = *ptr0; - len0 = len; - } -#endif - } - - distances[0] = offset - 1; - - move_pos(); - - return; -} - - -LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER) -{ - do { -#ifdef IS_HASH_CHAIN - if (lz->write_pos - lz->read_pos < NUM_HASH_BYTES) { - move_pending(); - continue; - } -#else - uint32_t len_limit; - if (lz->read_pos + lz->match_max_len <= lz->write_pos) { - len_limit = lz->match_max_len; - } else { - len_limit = lz->write_pos - lz->read_pos; - if (len_limit < MIN_MATCH_CHECK - || lz->sequence == SEQ_FLUSH) { - move_pending(); - continue; - } - } - const uint32_t match_min_pos - = lz->read_pos + lz->offset > lz->cyclic_buffer_size - ? lz->read_pos + lz->offset - lz->cyclic_buffer_size - : 0; -#endif - - assert(lz->pending == 0); - - const uint8_t *cur = lz->buffer + lz->read_pos; - -#ifdef HASH_ARRAY_2 - uint32_t hash_2_value; -# ifdef HASH_ARRAY_3 - uint32_t hash_3_value; - uint32_t hash_value; - HASH_CALC(); - lz->hash[HASH_3_OFFSET + hash_3_value] - = lz->read_pos + lz->offset; -# else - uint32_t hash_value; - HASH_CALC(); -# endif - lz->hash[hash_2_value] = lz->read_pos + lz->offset; -#else - uint32_t hash_value; - HASH_CALC(); -#endif - - uint32_t cur_match = lz->hash[FIX_HASH_SIZE + hash_value]; - lz->hash[FIX_HASH_SIZE + hash_value] - = lz->read_pos + lz->offset; - -#ifdef IS_HASH_CHAIN - lz->son[lz->cyclic_buffer_pos] = cur_match; -#else - uint32_t *ptr0 = lz->son + (lz->cyclic_buffer_pos << 1) + 1; - uint32_t *ptr1 = lz->son + (lz->cyclic_buffer_pos << 1); - - uint32_t len0 = NUM_HASH_DIRECT_BYTES; - uint32_t len1 = NUM_HASH_DIRECT_BYTES; - uint32_t count = lz->cut_value; - - while (true) { - if (cur_match <= match_min_pos || count-- == 0) { - *ptr0 = EMPTY_HASH_VALUE; - *ptr1 = EMPTY_HASH_VALUE; - break; - } - - const uint32_t delta = lz->read_pos - + lz->offset - cur_match; - const uint32_t cyclic_pos - = delta <= lz->cyclic_buffer_pos - ? lz->cyclic_buffer_pos - delta - : lz->cyclic_buffer_pos - delta - + lz->cyclic_buffer_size; - uint32_t *pair = lz->son + (cyclic_pos << 1); - - const uint8_t *pb = lz->buffer + cur_match - - lz->offset; - uint32_t len = MIN(len0, len1); - - if (pb[len] == cur[len]) { - while (++len != len_limit) - if (pb[len] != cur[len]) - break; - - if (len == len_limit) { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - break; - } - } - - if (pb[len] < cur[len]) { - *ptr1 = cur_match; - ptr1 = pair + 1; - cur_match = *ptr1; - len1 = len; - } else { - *ptr0 = cur_match; - ptr0 = pair; - cur_match = *ptr0; - len0 = len; - } - } -#endif - - move_pos(); - - } while (--num != 0); - - return; -} diff --git a/src/liblzma/lz/match_h.h b/src/liblzma/lz/match_h.h deleted file mode 100644 index 2eae90ba..00000000 --- a/src/liblzma/lz/match_h.h +++ /dev/null @@ -1,69 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file match_h.h -/// \brief Header template for different match finders -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "lz_encoder_private.h" - - -////////////////////// -// Global constants // -////////////////////// - -#undef LZMA_HASH_SIZE -#undef LZMA_FIX_HASH_SIZE -#undef LZMA_HASH_SIZE_C -#undef LZMA_FIX_HASH_SIZE_C - -#define LZMA_HASH_SIZE(mf_name) LZMA_HASH_SIZE_C(mf_name) -#define LZMA_FIX_HASH_SIZE(mf_name) LZMA_FIX_HASH_SIZE_C(mf_name) - -#define LZMA_HASH_SIZE_C(mf_name) \ - const uint32_t LZMA_ ## mf_name ## _HASH_SIZE - -#define LZMA_FIX_HASH_SIZE_C(mf_name) \ - const uint32_t LZMA_ ## mf_name ## _FIX_HASH_SIZE - -extern LZMA_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER); -extern LZMA_FIX_HASH_SIZE(LZMA_MATCH_FINDER_NAME_UPPER); - - -/////////////// -// Functions // -/////////////// - -#undef LZMA_GET_MATCHES -#undef LZMA_SKIP -#undef LZMA_GET_MATCHES_C -#undef LZMA_SKIP_C - -#define LZMA_GET_MATCHES(mf_name) LZMA_GET_MATCHES_C(mf_name) -#define LZMA_SKIP(mf_name) LZMA_SKIP_C(mf_name) - -#define LZMA_GET_MATCHES_C(mf_name) \ - extern void lzma_ ## mf_name ## _get_matches( \ - lzma_lz_encoder *restrict lz, \ - uint32_t *restrict distances) - -#define LZMA_SKIP_C(mf_name) \ - extern void lzma_ ## mf_name ## _skip( \ - lzma_lz_encoder *lz, uint32_t num) - -LZMA_GET_MATCHES(LZMA_MATCH_FINDER_NAME_LOWER); - -LZMA_SKIP(LZMA_MATCH_FINDER_NAME_LOWER); diff --git a/src/liblzma/lzma/Makefile.am b/src/liblzma/lzma/Makefile.am index 59ded214..7aeceb63 100644 --- a/src/liblzma/lzma/Makefile.am +++ b/src/liblzma/lzma/Makefile.am @@ -14,37 +14,46 @@ EXTRA_DIST = fastpos_tablegen.c -noinst_LTLIBRARIES = liblzma4.la -liblzma4_la_CPPFLAGS = \ +## Using liblzma2 since liblzma is already used for the final library. +noinst_LTLIBRARIES = liblzma2.la +liblzma2_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/common \ -I@top_srcdir@/src/liblzma/lz \ -I@top_srcdir@/src/liblzma/rangecoder -liblzma4_la_SOURCES = \ - lzma_common.h \ - lzma_literal.c \ - lzma_literal.h +liblzma2_la_SOURCES = lzma_common.h -if COND_MAIN_ENCODER -liblzma4_la_SOURCES += \ +if COND_ENCODER_LZMA +liblzma2_la_SOURCES += \ fastpos.h \ lzma_encoder.h \ lzma_encoder.c \ lzma_encoder_presets.c \ lzma_encoder_private.h \ - lzma_encoder_init.c \ lzma_encoder_features.c \ - lzma_encoder_getoptimum.c \ - lzma_encoder_getoptimumfast.c + lzma_encoder_optimum_fast.c \ + lzma_encoder_optimum_normal.c if !COND_SMALL -liblzma4_la_SOURCES += fastpos_table.c +liblzma2_la_SOURCES += fastpos_table.c endif endif -if COND_MAIN_DECODER -liblzma4_la_SOURCES += \ +if COND_DECODER_LZMA +liblzma2_la_SOURCES += \ lzma_decoder.c \ lzma_decoder.h endif + +if COND_ENCODER_LZMA2 +liblzma2_la_SOURCES += \ + lzma2_encoder.c \ + lzma2_encoder.h +endif + +if COND_DECODER_LZMA2 +liblzma2_la_SOURCES += \ + lzma2_decoder.c \ + lzma2_decoder.h +endif diff --git a/src/liblzma/lzma/fastpos.h b/src/liblzma/lzma/fastpos.h index 57a94556..503be275 100644 --- a/src/liblzma/lzma/fastpos.h +++ b/src/liblzma/lzma/fastpos.h @@ -81,8 +81,6 @@ // I'm making the table version the default, because that has good speed // on all systems I have tried. The size optimized version is sometimes // slightly faster, but sometimes it is a lot slower. -// -// Finally, this code isn't a major bottle neck in LZMA encoding anyway. #ifdef HAVE_SMALL # include "bsr.h" @@ -135,11 +133,7 @@ get_pos_slot(uint32_t pos) static inline uint32_t get_pos_slot_2(uint32_t pos) { - // FIXME: This assert() cannot be enabled at the moment, because - // lzma_getoptimum.c calls this function so that this assertion - // fails; however, it ignores the result of this function when - // this assert() would have failed. - // assert(pos >= FULL_DISTANCES); + assert(pos >= FULL_DISTANCES); if (pos < fastpos_limit(FULL_DISTANCES_BITS - 1, 0)) return fastpos_result(pos, FULL_DISTANCES_BITS - 1, 0); diff --git a/src/liblzma/lzma/lzma2_decoder.c b/src/liblzma/lzma/lzma2_decoder.c new file mode 100644 index 00000000..b16c40ce --- /dev/null +++ b/src/liblzma/lzma/lzma2_decoder.c @@ -0,0 +1,318 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma2_decoder.c +/// \brief LZMA2 decoder +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma2_decoder.h" +#include "lz_decoder.h" +#include "lzma_decoder.h" + + +struct lzma_coder_s { + enum sequence { + SEQ_CONTROL, + SEQ_UNCOMPRESSED_1, + SEQ_UNCOMPRESSED_2, + SEQ_COMPRESSED_0, + SEQ_COMPRESSED_1, + SEQ_PROPERTIES, + SEQ_LZMA, + SEQ_COPY, + } sequence; + + /// Sequence after the size fields have been decoded. + enum sequence next_sequence; + + /// LZMA decoder + lzma_lz_decoder lzma; + + /// Uncompressed size of LZMA chunk + size_t uncompressed_size; + + /// Compressed size of the chunk (naturally equals to uncompressed + /// size of uncompressed chunk) + size_t compressed_size; + + /// True if properties are needed. This is false before the + /// first LZMA chunk. + bool need_properties; + + /// True if dictionary reset is needed. This is false before the + /// first chunk (LZMA or uncompressed). + bool need_dictionary_reset; + + lzma_options_lzma options; +}; + + +static lzma_ret +lzma2_decode(lzma_coder *restrict coder, lzma_dict *restrict dict, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size) +{ + // With SEQ_LZMA it is possible that no new input is needed to do + // some progress. The rest of the sequences assume that there is + // at least one byte of input. + while (*in_pos < in_size || coder->sequence == SEQ_LZMA) + switch (coder->sequence) { + case SEQ_CONTROL: + if (in[*in_pos] & 0x80) { + // Get the highest five bits of uncompressed size. + coder->uncompressed_size + = (uint32_t)(in[*in_pos] & 0x1F) << 16; + coder->sequence = SEQ_UNCOMPRESSED_1; + + // See if we need to reset dictionary or state. + switch ((in[(*in_pos)++] >> 5) & 3) { + case 3: + dict_reset(dict); + coder->need_dictionary_reset = false; + + // Fall through + + case 2: + if (coder->need_dictionary_reset) + return LZMA_DATA_ERROR; + + coder->need_properties = false; + coder->next_sequence = SEQ_PROPERTIES; + break; + + case 1: + if (coder->need_properties) + return LZMA_DATA_ERROR; + + coder->lzma.reset(coder->lzma.coder, + &coder->options); + + coder->next_sequence = SEQ_LZMA; + break; + + case 0: + if (coder->need_properties) + return LZMA_DATA_ERROR; + + coder->next_sequence = SEQ_LZMA; + break; + } + + } else { + switch (in[(*in_pos)++]) { + case 0: + // End of payload marker + return LZMA_STREAM_END; + + case 1: + // Dictionary reset + dict_reset(dict); + coder->need_dictionary_reset = false; + + // Fall through + + case 2: + if (coder->need_dictionary_reset) + return LZMA_DATA_ERROR; + + // Uncompressed chunk; we need to read total + // size first. + coder->sequence = SEQ_COMPRESSED_0; + coder->next_sequence = SEQ_COPY; + break; + + default: + return LZMA_DATA_ERROR; + } + } + + break; + + case SEQ_UNCOMPRESSED_1: + coder->uncompressed_size += (uint32_t)(in[(*in_pos)++]) << 8; + coder->sequence = SEQ_UNCOMPRESSED_2; + break; + + case SEQ_UNCOMPRESSED_2: + coder->uncompressed_size += in[(*in_pos)++] + 1; + coder->sequence = SEQ_COMPRESSED_0; + coder->lzma.set_uncompressed(coder->lzma.coder, + coder->uncompressed_size); + break; + + case SEQ_COMPRESSED_0: + coder->compressed_size = (uint32_t)(in[(*in_pos)++]) << 8; + coder->sequence = SEQ_COMPRESSED_1; + break; + + case SEQ_COMPRESSED_1: + coder->compressed_size += in[(*in_pos)++] + 1; + coder->sequence = coder->next_sequence; + break; + + case SEQ_PROPERTIES: + if (lzma_lzma_lclppb_decode(&coder->options, in[(*in_pos)++])) + return LZMA_DATA_ERROR; + + coder->lzma.reset(coder->lzma.coder, &coder->options); + + coder->sequence = SEQ_LZMA; + break; + + case SEQ_LZMA: { + // Store the start offset so that we can update + // coder->compressed_size later. + const size_t in_start = *in_pos; + + // Decode from in[] to *dict. + const lzma_ret ret = coder->lzma.code(coder->lzma.coder, + dict, in, in_pos, in_size); + + // Validate and update coder->compressed_size. + const size_t in_used = *in_pos - in_start; + if (in_used > coder->compressed_size) + return LZMA_DATA_ERROR; + + coder->compressed_size -= in_used; + + // Return if we didn't finish the chunk, or an error occurred. + if (ret != LZMA_STREAM_END) + return ret; + + // The LZMA decoder must have consumed the whole chunk now. + // We don't need to worry about uncompressed size since it + // is checked by the LZMA decoder. + if (coder->compressed_size != 0) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_CONTROL; + break; + } + + case SEQ_COPY: { + // Copy from input to the dictionary as is. + // FIXME Can copy too much? + dict_write(dict, in, in_pos, in_size, &coder->compressed_size); + if (coder->compressed_size != 0) + return LZMA_OK; + + coder->sequence = SEQ_CONTROL; + break; + } + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +lzma2_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + assert(coder->lzma.end == NULL); + lzma_free(coder->lzma.coder, allocator); + + lzma_free(coder, allocator); + + return; +} + + +static lzma_ret +lzma2_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *options, size_t *dict_size) +{ + if (lz->coder == NULL) { + lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (lz->coder == NULL) + return LZMA_MEM_ERROR; + + lz->code = &lzma2_decode; + lz->end = &lzma2_decoder_end; + + lz->coder->lzma = LZMA_LZ_DECODER_INIT; + } + + lz->coder->sequence = SEQ_CONTROL; + lz->coder->need_properties = true; + lz->coder->need_dictionary_reset = true; + + return lzma_lzma_decoder_create(&lz->coder->lzma, + allocator, options, dict_size); +} + + +extern lzma_ret +lzma_lzma2_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + // LZMA2 can only be the last filter in the chain. This is enforced + // by the raw_decoder initialization. + assert(filters[1].init == NULL); + + return lzma_lz_decoder_init(next, allocator, filters, + &lzma2_decoder_init); +} + + +extern uint64_t +lzma_lzma2_decoder_memusage(const void *options) +{ + const uint64_t lzma_memusage = lzma_lzma_decoder_memusage(options); + if (lzma_memusage == UINT64_MAX) + return UINT64_MAX; + + return sizeof(lzma_coder) + lzma_memusage; +} + + +extern lzma_ret +lzma_lzma2_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size != 1) + return LZMA_HEADER_ERROR; + + // Check that reserved bits are unset. + if (props[0] & 0xC0) + return LZMA_HEADER_ERROR; + + // Decode the dictionary size. + if (props[0] > 40) + return LZMA_HEADER_ERROR; + + lzma_options_lzma *opt = lzma_alloc( + sizeof(lzma_options_lzma), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + if (props[0] == 40) { + opt->dictionary_size = UINT32_MAX; + } else { + opt->dictionary_size = 2 | (props[0] & 1); + opt->dictionary_size <<= props[0] / 2 + 11; + } + + opt->preset_dictionary = NULL; + opt->preset_dictionary_size = 0; + + *options = opt; + + return LZMA_OK; +} diff --git a/src/liblzma/lzma/lzma2_decoder.h b/src/liblzma/lzma/lzma2_decoder.h new file mode 100644 index 00000000..a7504863 --- /dev/null +++ b/src/liblzma/lzma/lzma2_decoder.h @@ -0,0 +1,35 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma2_decoder.h +/// \brief LZMA2 decoder +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_LZMA2_DECODER_H +#define LZMA_LZMA2_DECODER_H + +#include "common.h" + +extern lzma_ret lzma_lzma2_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +extern uint64_t lzma_lzma2_decoder_memusage(const void *options); + +extern lzma_ret lzma_lzma2_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + +#endif diff --git a/src/liblzma/lzma/lzma2_encoder.c b/src/liblzma/lzma/lzma2_encoder.c new file mode 100644 index 00000000..b2cd176b --- /dev/null +++ b/src/liblzma/lzma/lzma2_encoder.c @@ -0,0 +1,406 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma2_encoder.c +/// \brief LZMA2 encoder +// +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lz_encoder.h" +#include "lzma_encoder.h" +#include "fastpos.h" +#include "lzma2_encoder.h" + + +/// Maximum number of bytes of actual data per chunk (no headers) +#define LZMA2_CHUNK_MAX (UINT32_C(1) << 16) + +/// Maximum uncompressed size of LZMA chunk (no headers) +#define LZMA2_UNCOMPRESSED_MAX (UINT32_C(1) << 21) + +/// Maximum size of LZMA2 headers +#define LZMA2_HEADER_MAX 6 + +/// Size of a header for uncompressed chunk +#define LZMA2_HEADER_UNCOMPRESSED 3 + + +struct lzma_coder_s { + enum { + SEQ_INIT, + SEQ_LZMA_ENCODE, + SEQ_LZMA_COPY, + SEQ_UNCOMPRESSED_HEADER, + SEQ_UNCOMPRESSED_COPY, + } sequence; + + /// LZMA encoder + lzma_coder *lzma; + + /// If this is not NULL, we will check new options from this + /// structure when starting a new chunk. + const lzma_options_lzma *opt_new; + + /// LZMA options currently in use. + lzma_options_lzma opt_cur; + + bool need_properties; + bool need_state_reset; + bool need_dictionary_reset; + + /// Uncompressed size of a chunk + size_t uncompressed_size; + + /// Compressed size of a chunk (excluding headers); this is also used + /// to indicate the end of buf[] in SEQ_LZMA_COPY. + size_t compressed_size; + + /// Read position in buf[] + size_t buf_pos; + + /// Buffer to hold the chunk header and LZMA compressed data + uint8_t buf[LZMA2_HEADER_MAX + LZMA2_CHUNK_MAX]; +}; + + +static void +lzma2_header_lzma(lzma_coder *coder) +{ + assert(coder->uncompressed_size > 0); + assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); + assert(coder->compressed_size > 0); + assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + + size_t pos; + + if (coder->need_properties) { + pos = 0; + + if (coder->need_dictionary_reset) + coder->buf[pos] = 0x80 + (3 << 5); + else + coder->buf[pos] = 0x80 + (2 << 5); + } else { + pos = 1; + + if (coder->need_state_reset) + coder->buf[pos] = 0x80 + (1 << 5); + else + coder->buf[pos] = 0x80; + } + + // Set the start position for copying. + coder->buf_pos = pos; + + // Uncompressed size + size_t size = coder->uncompressed_size - 1; + coder->buf[pos++] += size >> 16; + coder->buf[pos++] = (size >> 8) & 0xFF; + coder->buf[pos++] = size & 0xFF; + + // Compressed size + size = coder->compressed_size - 1; + coder->buf[pos++] = size >> 8; + coder->buf[pos++] = size & 0xFF; + + // Properties, if needed + if (coder->need_properties) + lzma_lzma_lclppb_encode(&coder->opt_cur, coder->buf + pos); + + coder->need_properties = false; + coder->need_state_reset = false; + coder->need_dictionary_reset = false; + + // The copying code uses coder->compressed_size to indicate the end + // of coder->buf[], so we need add the maximum size of the header here. + coder->compressed_size += LZMA2_HEADER_MAX; + + return; +} + + +static void +lzma2_header_uncompressed(lzma_coder *coder) +{ + assert(coder->uncompressed_size > 0); + assert(coder->uncompressed_size <= LZMA2_CHUNK_MAX); + + // If this is the first chunk, we need to include dictionary + // reset indicator. + if (coder->need_dictionary_reset) + coder->buf[0] = 1; + else + coder->buf[0] = 2; + + coder->need_dictionary_reset = false; + + // "Compressed" size + coder->buf[1] = (coder->uncompressed_size - 1) >> 8; + coder->buf[2] = (coder->uncompressed_size - 1) & 0xFF; + + // Set the start position for copying. + coder->buf_pos = 0; + return; +} + + +static lzma_ret +lzma2_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_INIT: + // If there's no input left and we are flushing or finishing, + // don't start a new chunk. + if (mf_unencoded(mf) == 0) { + // Write end of payload marker if finishing. + if (mf->action == LZMA_FINISH) + out[(*out_pos)++] = 0; + + return mf->action == LZMA_RUN + ? LZMA_OK : LZMA_STREAM_END; + } + + // Look if there are new options. At least for now, + // only lc/lp/pb can be changed. + if (coder->opt_new != NULL + && (coder->opt_cur.literal_context_bits + != coder->opt_new->literal_context_bits + || coder->opt_cur.literal_pos_bits + != coder->opt_new->literal_pos_bits + || coder->opt_cur.pos_bits + != coder->opt_new->pos_bits)) { + // Options have been changed, copy them to opt_cur. + coder->opt_cur.literal_context_bits + = coder->opt_new->literal_context_bits; + coder->opt_cur.literal_pos_bits + = coder->opt_new->literal_pos_bits; + coder->opt_cur.pos_bits + = coder->opt_new->pos_bits; + + // We need to write the new options and reset + // the encoder state. + coder->need_properties = true; + coder->need_state_reset = true; + } + + if (coder->need_state_reset) + lzma_lzma_encoder_reset(coder->lzma, &coder->opt_cur); + + coder->uncompressed_size = 0; + coder->compressed_size = 0; + coder->sequence = SEQ_LZMA_ENCODE; + + // Fall through + + case SEQ_LZMA_ENCODE: { + // Calculate how much more uncompressed data this chunk + // could accept. + const uint32_t left = LZMA2_UNCOMPRESSED_MAX + - coder->uncompressed_size; + uint32_t limit; + + if (left < mf->match_len_max) { + // Must flush immediatelly since the next LZMA symbol + // could make the uncompressed size of the chunk too + // big. + limit = 0; + } else { + // Calculate maximum read_limit that is OK from point + // of view of LZMA2 chunk size. + limit = mf->read_pos - mf->read_ahead + + left - mf->match_len_max; + } + + // Save the start position so that we can update + // coder->uncompressed_size. + const uint32_t read_start = mf->read_pos - mf->read_ahead; + + // Call the LZMA encoder until the chunk is finished. + const lzma_ret ret = lzma_lzma_encode(coder->lzma, mf, + coder->buf + LZMA2_HEADER_MAX, + &coder->compressed_size, + LZMA2_CHUNK_MAX, limit); + + coder->uncompressed_size += mf->read_pos - mf->read_ahead + - read_start; + + assert(coder->compressed_size <= LZMA2_CHUNK_MAX); + assert(coder->uncompressed_size <= LZMA2_UNCOMPRESSED_MAX); + + if (ret != LZMA_STREAM_END) + return LZMA_OK; + + // See if the chunk compressed. If it didn't, we encode it + // as uncompressed chunk. This saves a few bytes of space + // and makes decoding faster. + if (coder->compressed_size >= coder->uncompressed_size) { + coder->uncompressed_size += mf->read_ahead; + assert(coder->uncompressed_size + <= LZMA2_UNCOMPRESSED_MAX); + mf->read_ahead = 0; + lzma2_header_uncompressed(coder); + coder->need_state_reset = true; + coder->sequence = SEQ_UNCOMPRESSED_HEADER; + break; + } + + // The chunk did compress at least by one byte, so we store + // the chunk as LZMA. + lzma2_header_lzma(coder); + + coder->sequence = SEQ_LZMA_COPY; + } + + // Fall through + + case SEQ_LZMA_COPY: + // Copy the compressed chunk along its headers to the + // output buffer. + lzma_bufcpy(coder->buf, &coder->buf_pos, + coder->compressed_size, + out, out_pos, out_size); + if (coder->buf_pos != coder->compressed_size) + return LZMA_OK; + + coder->sequence = SEQ_INIT; + break; + + case SEQ_UNCOMPRESSED_HEADER: + // Copy the three-byte header to indicate uncompressed chunk. + lzma_bufcpy(coder->buf, &coder->buf_pos, + LZMA2_HEADER_UNCOMPRESSED, + out, out_pos, out_size); + if (coder->buf_pos != LZMA2_HEADER_UNCOMPRESSED) + return LZMA_OK; + + coder->sequence = SEQ_UNCOMPRESSED_COPY; + + // Fall through + + case SEQ_UNCOMPRESSED_COPY: + // Copy the uncompressed data as is from the dictionary + // to the output buffer. + mf_read(mf, out, out_pos, out_size, &coder->uncompressed_size); + if (coder->uncompressed_size != 0) + return LZMA_OK; + + coder->sequence = SEQ_INIT; + break; + } + + return LZMA_OK; +} + + +static void +lzma2_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder->lzma, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +lzma2_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, + const void *options, lzma_lz_options *lz_options) +{ + if (options == NULL) + return LZMA_PROG_ERROR; + + if (lz->coder == NULL) { + lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (lz->coder == NULL) + return LZMA_MEM_ERROR; + + lz->code = &lzma2_encode; + lz->end = &lzma2_encoder_end; + + lz->coder->lzma = NULL; + } + + lz->coder->sequence = SEQ_INIT; + lz->coder->need_properties = true; + lz->coder->need_state_reset = false; + lz->coder->need_dictionary_reset = true; + + lz->coder->opt_cur = *(const lzma_options_lzma *)(options); + lz->coder->opt_new = lz->coder->opt_cur.persistent + ? options : NULL; + + // Initialize LZMA encoder + return_if_error(lzma_lzma_encoder_create(&lz->coder->lzma, allocator, + &lz->coder->opt_cur, lz_options)); + + // Make sure that we will always have enough history available in + // case we need to use uncompressed chunks. They are used when the + // compressed size of a chunk is not smaller than the uncompressed + // size, so we need to have at least LZMA2_COMPRESSED_MAX bytes + // history available. + if (lz_options->before_size + lz_options->dictionary_size + < LZMA2_CHUNK_MAX) + lz_options->before_size = LZMA2_CHUNK_MAX + - lz_options->dictionary_size; + + return LZMA_OK; +} + + +extern lzma_ret +lzma_lzma2_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_lz_encoder_init( + next, allocator, filters, &lzma2_encoder_init); +} + + +extern uint64_t +lzma_lzma2_encoder_memusage(const void *options) +{ + const uint64_t lzma_memusage = lzma_lzma_encoder_memusage(options); + if (lzma_memusage == UINT64_MAX) + return UINT64_MAX; + + return sizeof(lzma_coder) + lzma_memusage; +} + + +extern lzma_ret +lzma_lzma2_props_encode(const void *options, uint8_t *out) +{ + const lzma_options_lzma *const opt = options; + uint32_t d = MAX(opt->dictionary_size, LZMA_DICTIONARY_SIZE_MIN); + + // Round up to to the next 2^n - 1 or 2^n + 2^(n - 1) - 1 depending + // on which one is the next: + --d; + d |= d >> 2; + d |= d >> 3; + d |= d >> 4; + d |= d >> 8; + d |= d >> 16; + + // Get the highest two bits using the proper encoding: + if (d == UINT32_MAX) + out[0] = 40; + else + out[0] = get_pos_slot(d + 1) - 24; + + return LZMA_OK; +} diff --git a/src/liblzma/common/raw_common.h b/src/liblzma/lzma/lzma2_encoder.h similarity index 60% rename from src/liblzma/common/raw_common.h rename to src/liblzma/lzma/lzma2_encoder.h index 0a27f3dc..3e27f680 100644 --- a/src/liblzma/common/raw_common.h +++ b/src/liblzma/lzma/lzma2_encoder.h @@ -1,9 +1,10 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file raw_common.h -/// \brief Stuff shared between raw encoder and raw decoder +/// \file lzma2_encoder.h +/// \brief LZMA2 encoder // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 1999-2008 Igor Pavlov +// Copyright (C) 2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,14 +18,17 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_RAW_COMMON_H -#define LZMA_RAW_COMMON_H +#ifndef LZMA_LZMA2_ENCODER_H +#define LZMA_LZMA2_ENCODER_H #include "common.h" -extern lzma_ret lzma_raw_coder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_options_filter *options, - lzma_init_function (*get_function)(lzma_vli id), - bool is_encoder); +extern lzma_ret lzma_lzma2_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters); + +extern uint64_t lzma_lzma2_encoder_memusage(const void *options); + +extern lzma_ret lzma_lzma2_props_encode(const void *options, uint8_t *out); #endif diff --git a/src/liblzma/lzma/lzma_common.h b/src/liblzma/lzma/lzma_common.h index f677fcce..6909969b 100644 --- a/src/liblzma/lzma/lzma_common.h +++ b/src/liblzma/lzma/lzma_common.h @@ -22,81 +22,31 @@ #define LZMA_LZMA_COMMON_H #include "common.h" -#include "lzma_literal.h" #include "range_common.h" -/////////////// -// Constants // -/////////////// - -#define REP_DISTANCES 4 - -#define POS_SLOT_BITS 6 -#define DICT_LOG_SIZE_MAX 30 -#define DIST_TABLE_SIZE_MAX (DICT_LOG_SIZE_MAX * 2) -#if (UINT32_C(1) << DICT_LOG_SIZE_MAX) != LZMA_DICTIONARY_SIZE_MAX -# error DICT_LOG_SIZE_MAX is inconsistent with LZMA_DICTIONARY_SIZE_MAX -#endif - -// 2 is for speed optimization -#define LEN_TO_POS_STATES_BITS 2 -#define LEN_TO_POS_STATES (1 << LEN_TO_POS_STATES_BITS) - -#define MATCH_MIN_LEN 2 - -#define ALIGN_BITS 4 -#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) -#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) - -#define START_POS_MODEL_INDEX 4 -#define END_POS_MODEL_INDEX 14 -#define POS_MODELS (END_POS_MODEL_INDEX - START_POS_MODEL_INDEX) - -#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) -#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) +/////////////////// +// Miscellaneous // +/////////////////// +/// Maximum number of position states. A position state is the lowest pos bits +/// number of bits of the current uncompressed offset. In some places there +/// are different sets of probabilities for different pos states. #define POS_STATES_MAX (1 << LZMA_POS_BITS_MAX) -// Length coder & Length price table encoder -#define LEN_LOW_BITS 3 -#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS) -#define LEN_MID_BITS 3 -#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS) -#define LEN_HIGH_BITS 8 -#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS) -#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS) -#define LEN_SPEC_SYMBOLS (LOW_LOW_SYMBOLS + LEN_MID_LEN_SYMBOLS) -#define MATCH_MAX_LEN (MATCH_MIN_LEN + LEN_SYMBOLS - 1) - -// Total number of probs in a Len Encoder -#define LEN_CODER_TOTAL_PROBS (LEN_HIGH_CODER + LEN_HIGH_SYMBOLS) - -// Price table size of Len Encoder -#define LEN_PRICES (LEN_SYMBOLS << LZMA_POS_BITS_MAX) - -// Special lengths used together with distance == UINT32_MAX -#define LEN_SPECIAL_EOPM MATCH_MIN_LEN -#define LEN_SPECIAL_FLUSH (LEN_SPECIAL_EOPM + 1) - - -// Optimal - Number of entries in the optimum array. -#define OPTS (1 << 12) - - -// Miscellaneous -#define INFINITY_PRICE 0x0FFFFFFF - - -//////////// -// Macros // -//////////// - -#define get_len_to_pos_state(len) \ - ((len) < LEN_TO_POS_STATES + MATCH_MIN_LEN \ - ? (len) - MATCH_MIN_LEN \ - : LEN_TO_POS_STATES - 1) +/// Validates literal_context_bits, literal_pos_bits, and pos_bits. +static inline bool +is_lclppb_valid(const lzma_options_lzma *options) +{ + return options->literal_context_bits <= LZMA_LITERAL_CONTEXT_BITS_MAX + && options->literal_pos_bits + <= LZMA_LITERAL_POS_BITS_MAX + && options->literal_context_bits + + options->literal_pos_bits + <= LZMA_LITERAL_BITS_MAX + && options->pos_bits <= LZMA_POS_BITS_MAX; +} /////////// @@ -161,4 +111,126 @@ typedef enum { #define is_literal_state(state) \ ((state) < LIT_STATES) + +///////////// +// Literal // +///////////// + +/// Each literal coder is divided in three sections: +/// - 0x001-0x0FF: Without match byte +/// - 0x101-0x1FF: With match byte; match bit is 0 +/// - 0x201-0x2FF: With match byte; match bit is 1 +/// +/// Match byte is used when the previous LZMA symbol was something else than +/// a literal (that is, it was some kind of match). +#define LITERAL_CODER_SIZE 0x300 + +/// Maximum number of literal coders +#define LITERAL_CODERS_MAX (1 << LZMA_LITERAL_BITS_MAX) + +/// Locate the literal coder for the next literal byte. The choice depends on +/// - the lowest literal_pos_bits bits of the position of the current +/// byte; and +/// - the highest literal_context_bits bits of the previous byte. +#define literal_subcoder(probs, lc, lp_mask, pos, prev_byte) \ + ((probs)[(((pos) & lp_mask) << lc) + ((prev_byte) >> (8 - lc))]) + + +static inline void +literal_init(probability (*probs)[LITERAL_CODER_SIZE], + uint32_t literal_context_bits, uint32_t literal_pos_bits) +{ + assert(literal_context_bits + literal_pos_bits + <= LZMA_LITERAL_BITS_MAX); + + const uint32_t coders + = 1U << (literal_context_bits + literal_pos_bits); + + for (uint32_t i = 0; i < coders; ++i) + for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) + bit_reset(probs[i][j]); + + return; +} + + +////////////////// +// Match length // +////////////////// + +// Minimum length of a match is two bytes. +#define MATCH_LEN_MIN 2 + +// Match length is encoded with 4, 5, or 10 bits. +// +// Length Bits +// 2-9 4 = Choice=0 + 3 bits +// 10-17 5 = Choice=1 + Choice2=0 + 3 bits +// 18-273 10 = Choice=1 + Choice2=1 + 8 bits +#define LEN_LOW_BITS 3 +#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS) +#define LEN_MID_BITS 3 +#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS) +#define LEN_HIGH_BITS 8 +#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS) +#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS) + +// Maximum length of a match is 273 which is a result of the encoding +// described above. +#define MATCH_LEN_MAX (MATCH_LEN_MIN + LEN_SYMBOLS - 1) + + +//////////////////// +// Match distance // +//////////////////// + +// Different set of probabilities is used for match distances that have very +// short match length: Lengths of 2, 3, and 4 bytes have a separate set of +// probabilities for each length. The matches with longer length use a shared +// set of probabilities. +#define LEN_TO_POS_STATES 4 + +// Macro to get the index of the appropriate probability array. +#define get_len_to_pos_state(len) \ + ((len) < LEN_TO_POS_STATES + MATCH_LEN_MIN \ + ? (len) - MATCH_LEN_MIN \ + : LEN_TO_POS_STATES - 1) + +// The highest two bits of a match distance (pos slot) are encoded using six +// bits. See fastpos.h for more explanation. +#define POS_SLOT_BITS 6 +#define POS_SLOTS (1 << POS_SLOT_BITS) + +// Match distances up to 127 are fully encoded using probabilities. Since +// the highest two bits (pos slot) are always encoded using six bits, the +// distances 0-3 don't need any additional bits to encode, since the pos +// slot itself is the same as the actual distance. START_POS_MODEL_INDEX +// indicates the first pos slot where at least one additional bit is needed. +#define START_POS_MODEL_INDEX 4 + +// Match distances greater than 127 are encoded in three pieces: +// - pos slot: the highest two bits +// - direct bits: 2-26 bits below the highest two bits +// - alignment bits: four lowest bits +// +// Direct bits don't use any probabilities. +// +// The pos slot value of 14 is for distances 128-191 (see the table in +// fastpos.h to understand why). +#define END_POS_MODEL_INDEX 14 + +// Seven-bit distances use the full FIXME +#define FULL_DISTANCES_BITS (END_POS_MODEL_INDEX / 2) +#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS) + +// For match distances greater than 127, only the highest two bits and the +// lowest four bits (alignment) is encoded using probabilities. +#define ALIGN_BITS 4 +#define ALIGN_TABLE_SIZE (1 << ALIGN_BITS) +#define ALIGN_MASK (ALIGN_TABLE_SIZE - 1) + +// LZMA remembers the four most recent match distances. Reusing these distances +// tends to take less space than re-encoding the actual distance value. +#define REP_DISTANCES 4 + #endif diff --git a/src/liblzma/lzma/lzma_decoder.c b/src/liblzma/lzma/lzma_decoder.c index 68941021..e9d047d3 100644 --- a/src/liblzma/lzma/lzma_decoder.c +++ b/src/liblzma/lzma/lzma_decoder.c @@ -4,7 +4,7 @@ /// \brief LZMA decoder // // Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -18,74 +18,147 @@ // /////////////////////////////////////////////////////////////////////////////// -// NOTE: If you want to keep the line length in 80 characters, set -// tab width to 4 or less in your editor when editing this file. - +#include "lz_decoder.h" #include "lzma_common.h" #include "lzma_decoder.h" -#include "lz_decoder.h" #include "range_decoder.h" -/// REQUIRED_IN_BUFFER_SIZE is the number of required input bytes -/// for the worst case: longest match with longest distance. -/// LZMA_IN_BUFFER_SIZE must be larger than REQUIRED_IN_BUFFER_SIZE. -/// 23 bits = 2 (match select) + 10 (len) + 6 (distance) + 4 (align) -/// + 1 (rc_normalize) -/// -/// \todo Is this correct for sure? -/// -#define REQUIRED_IN_BUFFER_SIZE \ - ((23 * (BIT_MODEL_TOTAL_BITS - MOVE_BITS + 1) + 26 + 9) / 8) +#ifdef HAVE_SMALL +// Macros for (somewhat) size-optimized code. +#define seq_4(seq) seq -// Length decoders are easiest to have as macros, because they use range -// decoder macros, which use local variables rc_range and rc_code. +#define seq_6(seq) seq -#define length_decode(target, len_decoder, pos_state) \ +#define seq_8(seq) seq + +#define seq_len(seq) \ + seq ## _CHOICE, \ + seq ## _CHOICE2, \ + seq ## _BITTREE + +#define len_decode(target, ld, pos_state, seq) \ do { \ - if_bit_0(len_decoder.choice) { \ - update_bit_0(len_decoder.choice); \ - target = MATCH_MIN_LEN; \ - bittree_decode(target, len_decoder.low[pos_state], LEN_LOW_BITS); \ +case seq ## _CHOICE: \ + rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_update_0(ld.choice); \ + probs = ld.low[pos_state];\ + limit = LEN_LOW_SYMBOLS; \ + target = MATCH_LEN_MIN; \ } else { \ - update_bit_1(len_decoder.choice); \ - if_bit_0(len_decoder.choice2) { \ - update_bit_0(len_decoder.choice2); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ - bittree_decode(target, len_decoder.mid[pos_state], LEN_MID_BITS); \ + rc_update_1(ld.choice); \ +case seq ## _CHOICE2: \ + rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_update_0(ld.choice2); \ + probs = ld.mid[pos_state]; \ + limit = LEN_MID_SYMBOLS; \ + target = MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ } else { \ - update_bit_1(len_decoder.choice2); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ - bittree_decode(target, len_decoder.high, LEN_HIGH_BITS); \ + rc_update_1(ld.choice2); \ + probs = ld.high; \ + limit = LEN_HIGH_SYMBOLS; \ + target = MATCH_LEN_MIN + LEN_LOW_SYMBOLS \ + + LEN_MID_SYMBOLS; \ + } \ + } \ + symbol = 1; \ +case seq ## _BITTREE: \ + do { \ + rc_bit(probs[symbol], , , seq ## _BITTREE); \ + } while (symbol < limit); \ + target += symbol - limit; \ +} while (0) + +#else // HAVE_SMALL + +// Unrolled versions +#define seq_4(seq) \ + seq ## 0, \ + seq ## 1, \ + seq ## 2, \ + seq ## 3 + +#define seq_6(seq) \ + seq ## 0, \ + seq ## 1, \ + seq ## 2, \ + seq ## 3, \ + seq ## 4, \ + seq ## 5 + +#define seq_8(seq) \ + seq ## 0, \ + seq ## 1, \ + seq ## 2, \ + seq ## 3, \ + seq ## 4, \ + seq ## 5, \ + seq ## 6, \ + seq ## 7 + +#define seq_len(seq) \ + seq ## _CHOICE, \ + seq ## _LOW0, \ + seq ## _LOW1, \ + seq ## _LOW2, \ + seq ## _CHOICE2, \ + seq ## _MID0, \ + seq ## _MID1, \ + seq ## _MID2, \ + seq ## _HIGH0, \ + seq ## _HIGH1, \ + seq ## _HIGH2, \ + seq ## _HIGH3, \ + seq ## _HIGH4, \ + seq ## _HIGH5, \ + seq ## _HIGH6, \ + seq ## _HIGH7 + +#define len_decode(target, ld, pos_state, seq) \ +do { \ + symbol = 1; \ +case seq ## _CHOICE: \ + rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_update_0(ld.choice); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ + rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ + target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ + } else { \ + rc_update_1(ld.choice); \ +case seq ## _CHOICE2: \ + rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_update_0(ld.choice2); \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ + seq ## _MID0); \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ + seq ## _MID1); \ + rc_bit_case(ld.mid[pos_state][symbol], , , \ + seq ## _MID2); \ + target = symbol - LEN_MID_SYMBOLS \ + + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ + } else { \ + rc_update_1(ld.choice2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ + rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ + target = symbol - LEN_HIGH_SYMBOLS \ + + MATCH_LEN_MIN \ + + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ } \ } \ } while (0) - -#define length_decode_dummy(target, len_decoder, pos_state) \ -do { \ - if_bit_0(len_decoder.choice) { \ - update_bit_0_dummy(); \ - target = MATCH_MIN_LEN; \ - bittree_decode_dummy(target, \ - len_decoder.low[pos_state], LEN_LOW_BITS); \ - } else { \ - update_bit_1_dummy(); \ - if_bit_0(len_decoder.choice2) { \ - update_bit_0_dummy(); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ - bittree_decode_dummy(target, len_decoder.mid[pos_state], \ - LEN_MID_BITS); \ - } else { \ - update_bit_1_dummy(); \ - target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ - bittree_decode_dummy(target, len_decoder.high, LEN_HIGH_BITS); \ - } \ - } \ -} while (0) +#endif // HAVE_SMALL +/// Length decoder probabilities; see comments in lzma_common.h. typedef struct { probability choice; probability choice2; @@ -96,26 +169,12 @@ typedef struct { struct lzma_coder_s { - /// Data of the next coder, if any. - lzma_next_coder next; + /////////////////// + // Probabilities // + /////////////////// - /// Sliding output window a.k.a. dictionary a.k.a. history buffer. - lzma_lz_decoder lz; - - // Range coder - lzma_range_decoder rc; - - // State - lzma_lzma_state state; - uint32_t rep0; ///< Distance of the latest match - uint32_t rep1; ///< Distance of second latest match - uint32_t rep2; ///< Distance of third latest match - uint32_t rep3; ///< Distance of fourth latest match - uint32_t pos_bits; - uint32_t pos_mask; - uint32_t now_pos; // Lowest 32-bits are enough here. - - lzma_literal_coder literal_coder; + /// Literals; see comments in lzma_common.h. + probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; /// If 1, it's a match. Otherwise it's a single 8-bit literal. probability is_match[STATES][POS_STATES_MAX]; @@ -138,178 +197,107 @@ struct lzma_coder_s { /// the length is decoded from rep_len_decoder. probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_slot_decoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS]; - probability pos_decoders[FULL_DISTANCES - END_POS_MODEL_INDEX]; - probability pos_align_decoder[1 << ALIGN_BITS]; + /// Probability tree for the highest two bits of the match distance. + /// There is a separate probability tree for match lengths of + /// 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. + probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; - /// Length of a match + /// Probility trees for additional bits for match distance when the + /// distance is in the range [4, 127]. + probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + + /// Probability tree for the lowest four bits of a match distance + /// that is equal to or greater than 128. + probability pos_align[ALIGN_TABLE_SIZE]; + + /// Length of a normal match lzma_length_decoder match_len_decoder; - /// Length of a repeated match. + /// Length of a repeated match lzma_length_decoder rep_len_decoder; - /// True when we have produced at least one byte of output since the - /// beginning of the stream or the latest flush marker. - bool has_produced_output; + /////////////////// + // Decoder state // + /////////////////// + + // Range coder + lzma_range_decoder rc; + + // Types of the most recently seen LZMA symbols + lzma_lzma_state state; + + uint32_t rep0; ///< Distance of the latest match + uint32_t rep1; ///< Distance of second latest match + uint32_t rep2; ///< Distance of third latest match + uint32_t rep3; ///< Distance of fourth latest match + + uint32_t pos_mask; // (1U << pos_bits) - 1 + uint32_t literal_context_bits; + uint32_t literal_pos_mask; + + /// Uncompressed size as bytes, or LZMA_VLI_VALUE_UNKNOWN if end of + /// payload marker is expected. + lzma_vli uncompressed_size; + + //////////////////////////////// + // State of incomplete symbol // + //////////////////////////////// + + /// Position where to continue the decoder loop + enum { + SEQ_NORMALIZE, + SEQ_IS_MATCH, + seq_8(SEQ_LITERAL), + seq_8(SEQ_LITERAL_MATCHED), + SEQ_LITERAL_WRITE, + SEQ_IS_REP, + seq_len(SEQ_MATCH_LEN), + seq_6(SEQ_POS_SLOT), + SEQ_POS_MODEL, + SEQ_DIRECT, + seq_4(SEQ_ALIGN), + SEQ_EOPM, + SEQ_IS_REP0, + SEQ_SHORTREP, + SEQ_IS_REP0_LONG, + SEQ_IS_REP1, + SEQ_IS_REP2, + seq_len(SEQ_REP_LEN), + SEQ_COPY, + } sequence; + + /// Base of the current probability tree + probability *probs; + + /// Symbol being decoded. This is also used as an index variable in + /// bittree decoders: probs[symbol] + uint32_t symbol; + + /// Used as a loop termination condition on bittree decoders and + /// direct bits decoder. + uint32_t limit; + + /// Matched literal decoder: 0x100 or 0 to help avoiding branches. + /// Bittree reverse decoders: Offset of the next bit: 1 << offset + uint32_t offset; + + /// If decoding a literal: match byte. + /// If decoding a match: length of the match. + uint32_t len; }; -/// \brief Check if the next iteration of the decoder loop can be run. -/// -/// \note There must always be REQUIRED_IN_BUFFER_SIZE bytes -/// readable space! -/// -static bool lzma_attribute((pure)) -decode_dummy(const lzma_coder *restrict coder, - const uint8_t *restrict in, size_t in_pos_local, - const size_t in_size, lzma_range_decoder rc, - uint32_t state, uint32_t rep0, const uint32_t now_pos) -{ - uint32_t rc_bound; - - do { - const uint32_t pos_state = now_pos & coder->pos_mask; - - if_bit_0(coder->is_match[state][pos_state]) { - // It's a literal i.e. a single 8-bit byte. - - update_bit_0_dummy(); - - const probability *subcoder = literal_get_subcoder( - coder->literal_coder, now_pos, lz_get_byte(coder->lz, 0)); - uint32_t symbol = 1; - - if (is_literal_state(state)) { - // Decode literal without match byte. - do { - if_bit_0(subcoder[symbol]) { - update_bit_0_dummy(); - symbol <<= 1; - } else { - update_bit_1_dummy(); - symbol = (symbol << 1) | 1; - } - } while (symbol < 0x100); - - } else { - // Decode literal with match byte. - uint32_t match_byte = lz_get_byte(coder->lz, rep0); - uint32_t subcoder_offset = 0x100; - - do { - match_byte <<= 1; - const uint32_t match_bit = match_byte & subcoder_offset; - const uint32_t subcoder_index - = subcoder_offset + match_bit + symbol; - - if_bit_0(subcoder[subcoder_index]) { - update_bit_0_dummy(); - symbol <<= 1; - subcoder_offset &= ~match_bit; - } else { - update_bit_1_dummy(); - symbol = (symbol << 1) | 1; - subcoder_offset &= match_bit; - } - } while (symbol < 0x100); - } - - break; - } - - update_bit_1_dummy(); - uint32_t len; - - if_bit_0(coder->is_rep[state]) { - update_bit_0_dummy(); - length_decode_dummy(len, coder->match_len_decoder, pos_state); - - const uint32_t len_to_pos_state = get_len_to_pos_state(len); - uint32_t pos_slot = 0; - bittree_decode_dummy(pos_slot, - coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS); - assert(pos_slot <= 63); - - if (pos_slot >= START_POS_MODEL_INDEX) { - uint32_t direct_bits = (pos_slot >> 1) - 1; - assert(direct_bits >= 1 && direct_bits <= 31); - rep0 = 2 | (pos_slot & 1); - - if (pos_slot < END_POS_MODEL_INDEX) { - assert(direct_bits <= 5); - rep0 <<= direct_bits; - assert(rep0 <= 96); - // -1 is fine, because bittree_reverse_decode() - // starts from table index [1] (not [0]). - assert((int32_t)(rep0 - pos_slot - 1) >= -1); - assert((int32_t)(rep0 - pos_slot - 1) <= 82); - // We add the result to rep0, so rep0 - // must not be part of second argument - // of the macro. - const int32_t offset = rep0 - pos_slot - 1; - bittree_reverse_decode_dummy(coder->pos_decoders + offset, - direct_bits); - } else { - assert(pos_slot >= 14); - assert(direct_bits >= 6); - direct_bits -= ALIGN_BITS; - assert(direct_bits >= 2); - rc_decode_direct_dummy(direct_bits); - - bittree_reverse_decode_dummy(coder->pos_align_decoder, - ALIGN_BITS); - } - } - - } else { - update_bit_1_dummy(); - - if_bit_0(coder->is_rep0[state]) { - update_bit_0_dummy(); - - if_bit_0(coder->is_rep0_long[state][pos_state]) { - update_bit_0_dummy(); - break; - } else { - update_bit_1_dummy(); - } - - } else { - update_bit_1_dummy(); - - if_bit_0(coder->is_rep1[state]) { - update_bit_0_dummy(); - } else { - update_bit_1_dummy(); - - if_bit_0(coder->is_rep2[state]) { - update_bit_0_dummy(); - } else { - update_bit_1_dummy(); - } - } - } - - length_decode_dummy(len, coder->rep_len_decoder, pos_state); - } - } while (0); - - rc_normalize(); - - return in_pos_local <= in_size; -} - - -static bool -decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, - size_t *restrict in_pos, size_t in_size, bool has_safe_buffer) +static lzma_ret +lzma_decode(lzma_coder *restrict coder, lzma_dict *restrict dictptr, + const uint8_t *restrict in, + size_t *restrict in_pos, size_t in_size) { //////////////////// // Initialization // //////////////////// if (!rc_read_init(&coder->rc, in, in_pos, in_size)) - return false; + return LZMA_OK; /////////////// // Variables // @@ -318,8 +306,12 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, // Making local copies of often-used variables improves both // speed and readability. + lzma_dict dict = *dictptr; + + const size_t dict_start = dict.pos; + // Range decoder - rc_to_local(coder->rc); + rc_to_local(coder->rc, *in_pos); // State uint32_t state = coder->state; @@ -328,87 +320,168 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, uint32_t rep2 = coder->rep2; uint32_t rep3 = coder->rep3; - // Misc - uint32_t now_pos = coder->now_pos; - bool has_produced_output = coder->has_produced_output; - - // Variables derived from decoder settings const uint32_t pos_mask = coder->pos_mask; - size_t in_pos_local = *in_pos; // Local copy - size_t in_limit; - if (in_size <= REQUIRED_IN_BUFFER_SIZE) - in_limit = 0; - else - in_limit = in_size - REQUIRED_IN_BUFFER_SIZE; + // These variables are actually needed only if we last time ran + // out of input in the middle of the decoder loop. + probability *probs = coder->probs; + uint32_t symbol = coder->symbol; + uint32_t limit = coder->limit; + uint32_t offset = coder->offset; + uint32_t len = coder->len; + const uint32_t literal_pos_mask = coder->literal_pos_mask; + const uint32_t literal_context_bits = coder->literal_context_bits; - while (coder->lz.pos < coder->lz.limit - && (in_pos_local < in_limit || (has_safe_buffer - && decode_dummy(coder, in, in_pos_local, in_size, - rc, state, rep0, now_pos)))) { + // Temporary variables + uint32_t pos_state = dict.pos & pos_mask; - ///////////////////// - // Actual decoding // - ///////////////////// + lzma_ret ret = LZMA_OK; - const uint32_t pos_state = now_pos & pos_mask; + // If uncompressed size is known, there must be no end of payload + // marker. + const bool no_eopm = coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN; + if (no_eopm && coder->uncompressed_size < dict.limit - dict.pos) + dict.limit = dict.pos + (size_t)(coder->uncompressed_size); - if_bit_0(coder->is_match[state][pos_state]) { - update_bit_0(coder->is_match[state][pos_state]); + // The main decoder loop. The "switch" is used to restart the decoder at + // correct location. Once restarted, the "switch" is no longer used. + switch (coder->sequence) + while (true) { + // Calculate new pos_state. This is skipped on the first loop + // since we already calculated it when setting up the local + // variables. + pos_state = dict.pos & pos_mask; + + case SEQ_NORMALIZE: + case SEQ_IS_MATCH: + if (unlikely(no_eopm && dict.pos == dict.limit)) + break; + + rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { + rc_update_0(coder->is_match[state][pos_state]); // It's a literal i.e. a single 8-bit byte. - probability *subcoder = literal_get_subcoder(coder->literal_coder, - now_pos, lz_get_byte(coder->lz, 0)); - uint32_t symbol = 1; + probs = literal_subcoder(coder->literal, + literal_context_bits, literal_pos_mask, + dict.pos, dict_get(&dict, 0)); + symbol = 1; if (is_literal_state(state)) { // Decode literal without match byte. +#ifdef HAVE_SMALL + case SEQ_LITERAL: do { - if_bit_0(subcoder[symbol]) { - update_bit_0(subcoder[symbol]); - symbol <<= 1; - } else { - update_bit_1(subcoder[symbol]); - symbol = (symbol << 1) | 1; - } - } while (symbol < 0x100); - + rc_bit(probs[symbol], , , SEQ_LITERAL); + } while (symbol < (1 << 8)); +#else + rc_bit_case(probs[symbol], , , SEQ_LITERAL0); + rc_bit_case(probs[symbol], , , SEQ_LITERAL1); + rc_bit_case(probs[symbol], , , SEQ_LITERAL2); + rc_bit_case(probs[symbol], , , SEQ_LITERAL3); + rc_bit_case(probs[symbol], , , SEQ_LITERAL4); + rc_bit_case(probs[symbol], , , SEQ_LITERAL5); + rc_bit_case(probs[symbol], , , SEQ_LITERAL6); + rc_bit_case(probs[symbol], , , SEQ_LITERAL7); +#endif } else { // Decode literal with match byte. // - // The usage of subcoder_offset allows omitting some - // branches, which should give tiny speed improvement on - // some CPUs. subcoder_offset gets set to zero if match_bit - // didn't match. - uint32_t match_byte = lz_get_byte(coder->lz, rep0); - uint32_t subcoder_offset = 0x100; + // We store the byte we compare against + // ("match byte") to "len" to minimize the + // number of variables we need to store + // between decoder calls. + len = dict_get(&dict, rep0) << 1; + // The usage of "offset" allows omitting some + // branches, which should give tiny speed + // improvement on some CPUs. "offset" gets + // set to zero if match_bit didn't match. + offset = 0x100; + +#ifdef HAVE_SMALL + case SEQ_LITERAL_MATCHED: do { - match_byte <<= 1; - const uint32_t match_bit = match_byte & subcoder_offset; + const uint32_t match_bit + = len & offset; const uint32_t subcoder_index - = subcoder_offset + match_bit + symbol; + = offset + match_bit + + symbol; - if_bit_0(subcoder[subcoder_index]) { - update_bit_0(subcoder[subcoder_index]); - symbol <<= 1; - subcoder_offset &= ~match_bit; - } else { - update_bit_1(subcoder[subcoder_index]); - symbol = (symbol << 1) | 1; - subcoder_offset &= match_bit; - } - } while (symbol < 0x100); + rc_bit(probs[subcoder_index], + offset &= ~match_bit, + offset &= match_bit, + SEQ_LITERAL_MATCHED); + + // It seems to be faster to do this + // here instead of putting it to the + // beginning of the loop and then + // putting the "case" in the middle + // of the loop. + len <<= 1; + + } while (symbol < (1 << 8)); +#else + // Unroll the loop. + uint32_t match_bit; + uint32_t subcoder_index; + +# define d(seq) \ + case seq: \ + match_bit = len & offset; \ + subcoder_index = offset + match_bit + symbol; \ + rc_bit(probs[subcoder_index], \ + offset &= ~match_bit, \ + offset &= match_bit, \ + seq) + + d(SEQ_LITERAL_MATCHED0); + len <<= 1; + d(SEQ_LITERAL_MATCHED1); + len <<= 1; + d(SEQ_LITERAL_MATCHED2); + len <<= 1; + d(SEQ_LITERAL_MATCHED3); + len <<= 1; + d(SEQ_LITERAL_MATCHED4); + len <<= 1; + d(SEQ_LITERAL_MATCHED5); + len <<= 1; + d(SEQ_LITERAL_MATCHED6); + len <<= 1; + d(SEQ_LITERAL_MATCHED7); +# undef d +#endif + } + + //update_literal(state); + // Use a lookup table to update to literal state, + // since compared to other state updates, this would + // need two branches. + static const lzma_lzma_state next_state[] = { + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_LIT_LIT, + STATE_MATCH_LIT_LIT, + STATE_REP_LIT_LIT, + STATE_SHORTREP_LIT_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT, + STATE_SHORTREP_LIT, + STATE_MATCH_LIT, + STATE_REP_LIT + }; + state = next_state[state]; + + case SEQ_LITERAL_WRITE: + if (unlikely(dict_put(&dict, symbol))) { + coder->sequence = SEQ_LITERAL_WRITE; + goto out; } - // Put the decoded byte to the dictionary, update the - // decoder state, and start a new decoding loop. - coder->lz.dict[coder->lz.pos++] = (uint8_t)(symbol); - ++now_pos; - update_literal(state); - has_produced_output = true; continue; } @@ -416,115 +489,196 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, // (distance and length) which will be repeated from our // output history. - update_bit_1(coder->is_match[state][pos_state]); - uint32_t len; - - if_bit_0(coder->is_rep[state]) { - update_bit_0(coder->is_rep[state]); + rc_update_1(coder->is_match[state][pos_state]); + case SEQ_IS_REP: + rc_if_0(coder->is_rep[state], SEQ_IS_REP) { // Not a repeated match - // - // We will decode a new distance and store - // the value to distance. - - // Decode the length of the match. - length_decode(len, coder->match_len_decoder, pos_state); - + rc_update_0(coder->is_rep[state]); update_match(state); - const uint32_t len_to_pos_state = get_len_to_pos_state(len); - uint32_t pos_slot = 0; - bittree_decode(pos_slot, - coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS); - assert(pos_slot <= 63); + // The latest three match distances are kept in + // memory in case there are repeated matches. + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; - if (pos_slot >= START_POS_MODEL_INDEX) { - uint32_t direct_bits = (pos_slot >> 1) - 1; - assert(direct_bits >= 1 && direct_bits <= 30); - uint32_t distance = 2 | (pos_slot & 1); + // Decode the length of the match. + len_decode(len, coder->match_len_decoder, + pos_state, SEQ_MATCH_LEN); - if (pos_slot < END_POS_MODEL_INDEX) { - assert(direct_bits <= 5); - distance <<= direct_bits; - assert(distance <= 96); - // -1 is fine, because - // bittree_reverse_decode() - // starts from table index [1] - // (not [0]). - assert((int32_t)(distance - pos_slot - 1) >= -1); - assert((int32_t)(distance - pos_slot - 1) <= 82); - // We add the result to distance, so distance - // must not be part of second argument - // of the macro. - const int32_t offset = distance - pos_slot - 1; - bittree_reverse_decode(distance, coder->pos_decoders + offset, - direct_bits); + // Prepare to decode the highest two bits of the + // match distance. + probs = coder->pos_slot[get_len_to_pos_state(len)]; + symbol = 1; + +#ifdef HAVE_SMALL + case SEQ_POS_SLOT: + do { + rc_bit(probs[symbol], , , SEQ_POS_SLOT); + } while (symbol < POS_SLOTS); +#else + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT0); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT1); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT2); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT3); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT4); + rc_bit_case(probs[symbol], , , SEQ_POS_SLOT5); +#endif + // Get rid of the highest bit that was needed for + // indexing of the probability array. + symbol -= POS_SLOTS; + assert(symbol <= 63); + + if (symbol < START_POS_MODEL_INDEX) { + // Match distances [0, 3] have only two bits. + rep0 = symbol; + } else { + // Decode the lowest [1, 29] bits of + // the match distance. + limit = (symbol >> 1) - 1; + assert(limit >= 1 && limit <= 30); + rep0 = 2 + (symbol & 1); + + if (symbol < END_POS_MODEL_INDEX) { + // Prepare to decode the low bits for + // a distance of [4, 127]. + assert(limit <= 5); + rep0 <<= limit; + assert(rep0 <= 96); + // -1 is fine, because we start + // decoding at probs[1], not probs[0]. + // NOTE: This violates the C standard, + // since we are doing pointer + // arithmetic past the beginning of + // the array. + assert((int32_t)(rep0 - symbol - 1) + >= -1); + assert((int32_t)(rep0 - symbol - 1) + <= 82); + probs = coder->pos_special + rep0 + - symbol - 1; + symbol = 1; + offset = 0; + case SEQ_POS_MODEL: +#ifdef HAVE_SMALL + do { + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + } while (++offset < limit); +#else + switch (limit) { + case 5: + assert(offset == 0); + rc_bit(probs[symbol], , + rep0 += 1, + SEQ_POS_MODEL); + ++offset; + --limit; + case 4: + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + ++offset; + --limit; + case 3: + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + ++offset; + --limit; + case 2: + rc_bit(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + ++offset; + --limit; + case 1: + // We need "symbol" only for + // indexing the probability + // array, thus we can use + // rc_bit_last() here to omit + // the unneeded updating of + // "symbol". + rc_bit_last(probs[symbol], , + rep0 += 1 << offset, + SEQ_POS_MODEL); + } +#endif } else { - assert(pos_slot >= 14); - assert(direct_bits >= 6); - direct_bits -= ALIGN_BITS; - assert(direct_bits >= 2); - rc_decode_direct(distance, direct_bits); - distance <<= ALIGN_BITS; + // The distace is >= 128. Decode the + // lower bits without probabilities + // except the lowest four bits. + assert(symbol >= 14); + assert(limit >= 6); + limit -= ALIGN_BITS; + assert(limit >= 2); + case SEQ_DIRECT: + // Not worth manual unrolling + do { + rc_direct(rep0, SEQ_DIRECT); + } while (--limit > 0); - bittree_reverse_decode(distance, coder->pos_align_decoder, - ALIGN_BITS); + // Decode the lowest four bits using + // probabilities. + rep0 <<= ALIGN_BITS; + symbol = 1; +#ifdef HAVE_SMALL + offset = 0; + case SEQ_ALIGN: + do { + rc_bit(coder->pos_align[ + symbol], , + rep0 += 1 << offset, + SEQ_ALIGN); + } while (++offset < ALIGN_BITS); +#else + case SEQ_ALIGN0: + rc_bit(coder->pos_align[symbol], , + rep0 += 1, SEQ_ALIGN0); + case SEQ_ALIGN1: + rc_bit(coder->pos_align[symbol], , + rep0 += 2, SEQ_ALIGN1); + case SEQ_ALIGN2: + rc_bit(coder->pos_align[symbol], , + rep0 += 4, SEQ_ALIGN2); + case SEQ_ALIGN3: + // Like in SEQ_POS_MODEL, we don't + // need "symbol" for anything else + // than indexing the probability array. + rc_bit_last(coder->pos_align[symbol], , + rep0 += 8, SEQ_ALIGN3); +#endif - if (distance == UINT32_MAX) { - if (len == LEN_SPECIAL_EOPM) { - // End of Payload Marker found. - coder->lz.eopm_detected = true; - break; - - } else if (len == LEN_SPECIAL_FLUSH) { - // Flush marker detected. We must have produced - // at least one byte of output since the previous - // flush marker or the beginning of the stream. - // This is to prevent hanging the decoder with - // malicious input files. - if (!has_produced_output) - return true; - - has_produced_output = false; - - // We know that we have enough input to call - // this macro, because it is tested at the - // end of decode_dummy(). - rc_normalize(); - - rc_reset(rc); - - // If we don't have enough input here, we jump - // out of the loop. Note that while there is a - // useless call to rc_normalize(), it does nothing - // since we have just reset the range decoder. - if (!rc_read_init(&rc, in, &in_pos_local, in_size)) - break; - - continue; - - } else { - return true; + if (rep0 == UINT32_MAX) { + // End of payload marker was + // found. It must not be + // present if uncompressed + // size is known. + if (coder->uncompressed_size + != LZMA_VLI_VALUE_UNKNOWN) { + ret = LZMA_DATA_ERROR; + goto out; } + + case SEQ_EOPM: + // TODO Comment + rc_normalize(SEQ_EOPM); + ret = LZMA_STREAM_END; + goto out; } } + } - // The latest three match distances are kept in - // memory in case there are repeated matches. - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = distance; - - } else { - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = pos_slot; + // Validate the distance we just decoded. + if (unlikely(!dict_is_distance_valid(&dict, rep0))) { + ret = LZMA_DATA_ERROR; + goto out; } } else { - update_bit_1(coder->is_rep[state]); + rc_update_1(coder->is_rep[state]); // Repeated match // @@ -532,242 +686,318 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in, // earlier. The latest four match distances are // available as rep0, rep1, rep2 and rep3. We will // now decode which of them is the new distance. + // + // There cannot be a match if we haven't produced + // any output, so check that first. + if (unlikely(!dict_is_distance_valid(&dict, 0))) { + ret = LZMA_DATA_ERROR; + goto out; + } - if_bit_0(coder->is_rep0[state]) { - update_bit_0(coder->is_rep0[state]); - + case SEQ_IS_REP0: + rc_if_0(coder->is_rep0[state], SEQ_IS_REP0) { + rc_update_0(coder->is_rep0[state]); // The distance is rep0. - if_bit_0(coder->is_rep0_long[state][pos_state]) { - update_bit_0(coder->is_rep0_long[state][pos_state]); + case SEQ_IS_REP0_LONG: + rc_if_0(coder->is_rep0_long[state][pos_state], + SEQ_IS_REP0_LONG) { + rc_update_0(coder->is_rep0_long[ + state][pos_state]); update_short_rep(state); - // Repeat exactly one byte and start a new decoding loop. - // Note that rep0 is known to have a safe value, thus we - // don't need to check if we are wrapping the dictionary - // when it isn't full yet. - if (unlikely(lz_is_empty(coder->lz))) - return true; + case SEQ_SHORTREP: + if (unlikely(dict_put(&dict, dict_get( + &dict, rep0)))) { + coder->sequence = SEQ_SHORTREP; + goto out; + } - coder->lz.dict[coder->lz.pos] - = lz_get_byte(coder->lz, rep0); - ++coder->lz.pos; - ++now_pos; - has_produced_output = true; continue; - - } else { - update_bit_1(coder->is_rep0_long[state][pos_state]); - - // Repeating more than one byte at - // distance of rep0. } - } else { - update_bit_1(coder->is_rep0[state]); + // Repeating more than one byte at + // distance of rep0. + rc_update_1(coder->is_rep0_long[ + state][pos_state]); + } else { + rc_update_1(coder->is_rep0[state]); + + case SEQ_IS_REP1: // The distance is rep1, rep2 or rep3. Once // we find out which one of these three, it // is stored to rep0 and rep1, rep2 and rep3 // are updated accordingly. + rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { + rc_update_0(coder->is_rep1[state]); - uint32_t distance; + const uint32_t distance = rep1; + rep1 = rep0; + rep0 = distance; - if_bit_0(coder->is_rep1[state]) { - update_bit_0(coder->is_rep1[state]); - distance = rep1; } else { - update_bit_1(coder->is_rep1[state]); + rc_update_1(coder->is_rep1[state]); + case SEQ_IS_REP2: + rc_if_0(coder->is_rep2[state], + SEQ_IS_REP2) { + rc_update_0(coder->is_rep2[ + state]); + + const uint32_t distance = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; - if_bit_0(coder->is_rep2[state]) { - update_bit_0(coder->is_rep2[state]); - distance = rep2; } else { - update_bit_1(coder->is_rep2[state]); - distance = rep3; + rc_update_1(coder->is_rep2[ + state]); + + const uint32_t distance = rep3; rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; } - - rep2 = rep1; } - - rep1 = rep0; - rep0 = distance; } update_long_rep(state); // Decode the length of the repeated match. - length_decode(len, coder->rep_len_decoder, pos_state); + len_decode(len, coder->rep_len_decoder, + pos_state, SEQ_REP_LEN); } - ///////////////////////////////// // Repeat from history buffer. // ///////////////////////////////// // The length is always between these limits. There is no way // to trigger the algorithm to set len outside this range. - assert(len >= MATCH_MIN_LEN); - assert(len <= MATCH_MAX_LEN); - - now_pos += len; - has_produced_output = true; + assert(len >= MATCH_LEN_MIN); + assert(len <= MATCH_LEN_MAX); + case SEQ_COPY: // Repeat len bytes from distance of rep0. - if (!lzma_lz_out_repeat(&coder->lz, rep0, len)) - return true; + if (unlikely(dict_repeat(&dict, rep0, &len))) { + coder->sequence = SEQ_COPY; + goto out; + } } - rc_normalize(); + rc_normalize(SEQ_NORMALIZE); + coder->sequence = SEQ_IS_MATCH; +out: + // Save state - ///////////////////////////////// - // Update the *data structure. // - ///////////////////////////////// + // NOTE: Must not copy dict.limit. + dictptr->pos = dict.pos; + dictptr->full = dict.full; - // Range decoder - rc_from_local(coder->rc); + rc_from_local(coder->rc, *in_pos); - // State coder->state = state; coder->rep0 = rep0; coder->rep1 = rep1; coder->rep2 = rep2; coder->rep3 = rep3; - // Misc - coder->now_pos = now_pos; - coder->has_produced_output = has_produced_output; - *in_pos = in_pos_local; + coder->probs = probs; + coder->symbol = symbol; + coder->limit = limit; + coder->offset = offset; + coder->len = len; - return false; + // Update the remaining amount of uncompressed data if uncompressed + // size was known. + if (coder->uncompressed_size != LZMA_VLI_VALUE_UNKNOWN) { + coder->uncompressed_size -= dict.pos - dict_start; + + // Since there cannot be end of payload marker if the + // uncompressed size was known, we check here if we + // finished decoding. + if (coder->uncompressed_size == 0 && ret == LZMA_OK + && coder->sequence != SEQ_NORMALIZE) + ret = coder->sequence == SEQ_IS_MATCH + ? LZMA_STREAM_END : LZMA_DATA_ERROR; + } + + // We can do an additional check in the range decoder to catch some + // corrupted files. + if (ret == LZMA_STREAM_END) { + if (!rc_is_finished(coder->rc)) + ret = LZMA_DATA_ERROR; + + // Reset the range decoder so that it is ready to reinitialize + // for a new LZMA2 chunk. + rc_reset(coder->rc); + } + + return ret; } + static void -lzma_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +lzma_decoder_uncompressed(lzma_coder *coder, lzma_vli uncompressed_size) { - lzma_next_coder_end(&coder->next, allocator); - lzma_lz_decoder_end(&coder->lz, allocator); - lzma_free(coder, allocator); + coder->uncompressed_size = uncompressed_size; +} + +/* +extern void +lzma_lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size) +{ + // This is hack. + (*(lzma_coder **)(coder))->uncompressed_size = uncompressed_size; +} +*/ + +static void +lzma_decoder_reset(lzma_coder *coder, const void *opt) +{ + const lzma_options_lzma *options = opt; + + // NOTE: We assume that lc/lp/pb are valid since they were + // successfully decoded with lzma_lzma_decode_properties(). + // FIXME? + + // Calculate pos_mask. We don't need pos_bits as is for anything. + coder->pos_mask = (1U << options->pos_bits) - 1; + + // Initialize the literal decoder. + literal_init(coder->literal, options->literal_context_bits, + options->literal_pos_bits); + + coder->literal_context_bits = options->literal_context_bits; + coder->literal_pos_mask = (1 << options->literal_pos_bits) - 1; + + // State + coder->state = STATE_LIT_LIT; + coder->rep0 = 0; + coder->rep1 = 0; + coder->rep2 = 0; + coder->rep3 = 0; + coder->pos_mask = (1 << options->pos_bits) - 1; + + // Range decoder + rc_reset(coder->rc); + + // Bit and bittree decoders + for (uint32_t i = 0; i < STATES; ++i) { + for (uint32_t j = 0; j <= coder->pos_mask; ++j) { + bit_reset(coder->is_match[i][j]); + bit_reset(coder->is_rep0_long[i][j]); + } + + bit_reset(coder->is_rep[i]); + bit_reset(coder->is_rep0[i]); + bit_reset(coder->is_rep1[i]); + bit_reset(coder->is_rep2[i]); + } + + for (uint32_t i = 0; i < LEN_TO_POS_STATES; ++i) + bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + + for (uint32_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + bit_reset(coder->pos_special[i]); + + bittree_reset(coder->pos_align, ALIGN_BITS); + + // Len decoders (also bit/bittree) + const uint32_t num_pos_states = 1 << options->pos_bits; + bit_reset(coder->match_len_decoder.choice); + bit_reset(coder->match_len_decoder.choice2); + bit_reset(coder->rep_len_decoder.choice); + bit_reset(coder->rep_len_decoder.choice2); + + for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { + bittree_reset(coder->match_len_decoder.low[pos_state], + LEN_LOW_BITS); + bittree_reset(coder->match_len_decoder.mid[pos_state], + LEN_MID_BITS); + + bittree_reset(coder->rep_len_decoder.low[pos_state], + LEN_LOW_BITS); + bittree_reset(coder->rep_len_decoder.mid[pos_state], + LEN_MID_BITS); + } + + bittree_reset(coder->match_len_decoder.high, LEN_HIGH_BITS); + bittree_reset(coder->rep_len_decoder.high, LEN_HIGH_BITS); + + coder->sequence = SEQ_IS_MATCH; + coder->probs = NULL; + coder->symbol = 0; + coder->limit = 0; + coder->offset = 0; + coder->len = 0; + return; } +extern lzma_ret +lzma_lzma_decoder_create(lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *opt, size_t *dict_size) +{ + if (lz->coder == NULL) { + lz->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (lz->coder == NULL) + return LZMA_MEM_ERROR; + + lz->code = &lzma_decode; + lz->reset = &lzma_decoder_reset; + lz->set_uncompressed = &lzma_decoder_uncompressed; + } + + // All dictionary sizes are OK here. LZ decoder will take care of + // the special cases. + const lzma_options_lzma *options = opt; + *dict_size = options->dictionary_size; + + return LZMA_OK; +} + + +/// Allocate and initialize LZMA decoder. This is used only via LZ +/// initialization (lzma_lzma_decoder_init() passes function pointer to +/// the LZ initialization). +static lzma_ret +lzma_decoder_init(lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *options, size_t *dict_size) +{ + if (!is_lclppb_valid(options)) + return LZMA_PROG_ERROR; + + return_if_error(lzma_lzma_decoder_create( + lz, allocator, options, dict_size)); + + lzma_decoder_reset(lz->coder, options); + lzma_decoder_uncompressed(lz->coder, LZMA_VLI_VALUE_UNKNOWN); + + return LZMA_OK; +} + + extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters) { - // LZMA can only be the last filter in the chain. + // LZMA can only be the last filter in the chain. This is enforced + // by the raw_decoder initialization. assert(filters[1].init == NULL); - // Validate pos_bits. Other options are validated by the - // respective initialization functions. - const lzma_options_lzma *options = filters[0].options; - if (options->pos_bits > LZMA_POS_BITS_MAX) - return LZMA_HEADER_ERROR; - - // Allocate memory for the decoder if needed. - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) - return LZMA_MEM_ERROR; - - next->code = &lzma_lz_decode; - next->end = &lzma_decoder_end; - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->lz = LZMA_LZ_DECODER_INIT; - } - - // Store the pos_bits and calculate pos_mask. - next->coder->pos_bits = options->pos_bits; - next->coder->pos_mask = (1U << next->coder->pos_bits) - 1; - - // Initialize the literal decoder. - return_if_error(lzma_literal_init(&next->coder->literal_coder, - options->literal_context_bits, - options->literal_pos_bits)); - - // Allocate and initialize the LZ decoder. - return_if_error(lzma_lz_decoder_reset(&next->coder->lz, allocator, - &decode_real, options->dictionary_size, - MATCH_MAX_LEN)); - - // State - next->coder->state = 0; - next->coder->rep0 = 0; - next->coder->rep1 = 0; - next->coder->rep2 = 0; - next->coder->rep3 = 0; - next->coder->pos_bits = options->pos_bits; - next->coder->pos_mask = (1 << next->coder->pos_bits) - 1; - next->coder->now_pos = 0; - - // Range decoder - rc_reset(next->coder->rc); - - // Bit and bittree decoders - for (uint32_t i = 0; i < STATES; ++i) { - for (uint32_t j = 0; j <= next->coder->pos_mask; ++j) { - bit_reset(next->coder->is_match[i][j]); - bit_reset(next->coder->is_rep0_long[i][j]); - } - - bit_reset(next->coder->is_rep[i]); - bit_reset(next->coder->is_rep0[i]); - bit_reset(next->coder->is_rep1[i]); - bit_reset(next->coder->is_rep2[i]); - } - - for (uint32_t i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(next->coder->pos_slot_decoder[i], POS_SLOT_BITS); - - for (uint32_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(next->coder->pos_decoders[i]); - - bittree_reset(next->coder->pos_align_decoder, ALIGN_BITS); - - // Len decoders (also bit/bittree) - const uint32_t num_pos_states = 1 << next->coder->pos_bits; - bit_reset(next->coder->match_len_decoder.choice); - bit_reset(next->coder->match_len_decoder.choice2); - bit_reset(next->coder->rep_len_decoder.choice); - bit_reset(next->coder->rep_len_decoder.choice2); - - for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { - bittree_reset(next->coder->match_len_decoder.low[pos_state], - LEN_LOW_BITS); - bittree_reset(next->coder->match_len_decoder.mid[pos_state], - LEN_MID_BITS); - - bittree_reset(next->coder->rep_len_decoder.low[pos_state], - LEN_LOW_BITS); - bittree_reset(next->coder->rep_len_decoder.mid[pos_state], - LEN_MID_BITS); - } - - bittree_reset(next->coder->match_len_decoder.high, LEN_HIGH_BITS); - bittree_reset(next->coder->rep_len_decoder.high, LEN_HIGH_BITS); - - next->coder->has_produced_output = false; - - return LZMA_OK; -} - - -extern void -lzma_lzma_decoder_uncompressed_size( - lzma_next_coder *next, lzma_vli uncompressed_size) -{ - next->coder->lz.uncompressed_size = uncompressed_size; - return; + return lzma_lz_decoder_init(next, allocator, filters, + &lzma_decoder_init); } extern bool -lzma_lzma_decode_properties(lzma_options_lzma *options, uint8_t byte) +lzma_lzma_lclppb_decode(lzma_options_lzma *options, uint8_t byte) { if (byte > (4 * 5 + 4) * 9 + 8) return true; @@ -781,3 +1011,49 @@ lzma_lzma_decode_properties(lzma_options_lzma *options, uint8_t byte) return options->literal_context_bits + options->literal_pos_bits > LZMA_LITERAL_BITS_MAX; } + + +extern uint64_t +lzma_lzma_decoder_memusage(const void *options) +{ + const lzma_options_lzma *const opt = options; + const uint64_t lz_memusage + = lzma_lz_decoder_memusage(opt->dictionary_size); + if (lz_memusage == UINT64_MAX) + return UINT64_MAX; + + return sizeof(lzma_coder) + lz_memusage; +} + + +extern lzma_ret +lzma_lzma_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size != 5) + return LZMA_HEADER_ERROR; + + lzma_options_lzma *opt + = lzma_alloc(sizeof(lzma_options_lzma), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + if (lzma_lzma_lclppb_decode(opt, props[0])) + goto error; + + // All dictionary sizes are accepted, including zero. LZ decoder + // will automatically use a dictionary at least a few KiB even if + // a smaller dictionary is requested. + opt->dictionary_size = integer_read_32(props + 1); + + opt->preset_dictionary = NULL; + opt->preset_dictionary_size = 0; + + *options = opt; + + return LZMA_OK; + +error: + lzma_free(opt, allocator); + return LZMA_HEADER_ERROR; +} diff --git a/src/liblzma/lzma/lzma_decoder.h b/src/liblzma/lzma/lzma_decoder.h index 9d57c7e5..3792f452 100644 --- a/src/liblzma/lzma/lzma_decoder.h +++ b/src/liblzma/lzma/lzma_decoder.h @@ -28,16 +28,27 @@ extern lzma_ret lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); -/// Set known uncompressed size. This is a hack needed to support -/// LZMA_Alone files that don't have EOPM. -extern void lzma_lzma_decoder_uncompressed_size( - lzma_next_coder *next, lzma_vli uncompressed_size); +extern uint64_t lzma_lzma_decoder_memusage(const void *options); + +extern lzma_ret lzma_lzma_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + /// \brief Decodes the LZMA Properties byte (lc/lp/pb) /// /// \return true if error occorred, false on success /// -extern bool lzma_lzma_decode_properties( +extern bool lzma_lzma_lclppb_decode( lzma_options_lzma *options, uint8_t byte); + +#ifdef LZMA_LZ_DECODER_H +/// Allocate and setup function pointers only. This is used by LZMA1 and +/// LZMA2 decoders. +extern lzma_ret lzma_lzma_decoder_create( + lzma_lz_decoder *lz, lzma_allocator *allocator, + const void *opt, size_t *dict_size); +#endif + #endif diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c index afb1d5ed..a84801e7 100644 --- a/src/liblzma/lzma/lzma_encoder.c +++ b/src/liblzma/lzma/lzma_encoder.c @@ -30,40 +30,33 @@ static inline void literal_matched(lzma_range_encoder *rc, probability *subcoder, uint32_t match_byte, uint32_t symbol) { - uint32_t context = 1; - uint32_t bit_count = 8; + uint32_t offset = 0x100; + symbol += UINT32_C(1) << 8; do { - uint32_t bit = (symbol >> --bit_count) & 1; - const uint32_t match_bit = (match_byte >> bit_count) & 1; - rc_bit(rc, &subcoder[(0x100 << match_bit) + context], bit); - context = (context << 1) | bit; + match_byte <<= 1; + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; + rc_bit(rc, &subcoder[subcoder_index], bit); - if (match_bit != bit) { - // The bit from the literal being encoded and the bit - // from the previous match differ. Finish encoding - // as a normal literal. - while (bit_count != 0) { - bit = (symbol >> --bit_count) & 1; - rc_bit(rc, &subcoder[context], bit); - context = (context << 1) | bit; - } + symbol <<= 1; + offset &= ~(match_byte ^ symbol); - break; - } - - } while (bit_count != 0); + } while (symbol < (UINT32_C(1) << 16)); } static inline void -literal(lzma_coder *coder) +literal(lzma_coder *coder, lzma_mf *mf, uint32_t position) { // Locate the literal byte to be encoded and the subcoder. - const uint8_t cur_byte = coder->lz.buffer[ - coder->lz.read_pos - coder->additional_offset]; - probability *subcoder = literal_get_subcoder(coder->literal_coder, - coder->now_pos, coder->previous_byte); + const uint8_t cur_byte = mf->buffer[ + mf->read_pos - mf->read_ahead]; + probability *subcoder = literal_subcoder(coder->literal, + coder->literal_context_bits, coder->literal_pos_mask, + position, mf->buffer[mf->read_pos - mf->read_ahead - 1]); if (is_literal_state(coder->state)) { // Previous LZMA-symbol was a literal. Encode a normal @@ -73,14 +66,13 @@ literal(lzma_coder *coder) // Previous LZMA-symbol was a match. Use the last byte of // the match as a "match byte". That is, compare the bits // of the current literal and the match byte. - const uint8_t match_byte = coder->lz.buffer[ - coder->lz.read_pos - coder->reps[0] - 1 - - coder->additional_offset]; + const uint8_t match_byte = mf->buffer[ + mf->read_pos - coder->reps[0] - 1 + - mf->read_ahead]; literal_matched(&coder->rc, subcoder, match_byte, cur_byte); } update_literal(coder->state); - coder->previous_byte = cur_byte; } @@ -88,12 +80,41 @@ literal(lzma_coder *coder) // Match length // ////////////////// +static void +length_update_prices(lzma_length_encoder *lc, const uint32_t pos_state) +{ + const uint32_t table_size = lc->table_size; + lc->counters[pos_state] = table_size; + + const uint32_t a0 = rc_bit_0_price(lc->choice); + const uint32_t a1 = rc_bit_1_price(lc->choice); + const uint32_t b0 = a1 + rc_bit_0_price(lc->choice2); + const uint32_t b1 = a1 + rc_bit_1_price(lc->choice2); + uint32_t *const prices = lc->prices[pos_state]; + + uint32_t i; + for (i = 0; i < table_size && i < LEN_LOW_SYMBOLS; ++i) + prices[i] = a0 + rc_bittree_price(lc->low[pos_state], + LEN_LOW_BITS, i); + + for (; i < table_size && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i) + prices[i] = b0 + rc_bittree_price(lc->mid[pos_state], + LEN_MID_BITS, i - LEN_LOW_SYMBOLS); + + for (; i < table_size; ++i) + prices[i] = b1 + rc_bittree_price(lc->high, LEN_HIGH_BITS, + i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS); + + return; +} + + static inline void length(lzma_range_encoder *rc, lzma_length_encoder *lc, - const uint32_t pos_state, uint32_t len) + const uint32_t pos_state, uint32_t len, const bool fast_mode) { - assert(len <= MATCH_MAX_LEN); - len -= MATCH_MIN_LEN; + assert(len <= MATCH_LEN_MAX); + len -= MATCH_LEN_MIN; if (len < LEN_LOW_SYMBOLS) { rc_bit(rc, &lc->choice, 0); @@ -111,6 +132,12 @@ length(lzma_range_encoder *rc, lzma_length_encoder *lc, rc_bittree(rc, lc->high, LEN_HIGH_BITS, len); } } + + // Only getoptimum uses the prices so don't update the table when + // in fast mode. + if (!fast_mode) + if (--lc->counters[pos_state] == 0) + length_update_prices(lc, pos_state); } @@ -124,12 +151,12 @@ match(lzma_coder *coder, const uint32_t pos_state, { update_match(coder->state); - length(&coder->rc, &coder->match_len_encoder, pos_state, len); - coder->prev_len_encoder = &coder->match_len_encoder; + length(&coder->rc, &coder->match_len_encoder, pos_state, len, + coder->fast_mode); const uint32_t pos_slot = get_pos_slot(distance); const uint32_t len_to_pos_state = get_len_to_pos_state(len); - rc_bittree(&coder->rc, coder->pos_slot_encoder[len_to_pos_state], + rc_bittree(&coder->rc, coder->pos_slot[len_to_pos_state], POS_SLOT_BITS, pos_slot); if (pos_slot >= START_POS_MODEL_INDEX) { @@ -139,13 +166,13 @@ match(lzma_coder *coder, const uint32_t pos_state, if (pos_slot < END_POS_MODEL_INDEX) { rc_bittree_reverse(&coder->rc, - &coder->pos_encoders[base - pos_slot - 1], + &coder->pos_special[base - pos_slot - 1], footer_bits, pos_reduced); } else { rc_direct(&coder->rc, pos_reduced >> ALIGN_BITS, footer_bits - ALIGN_BITS); rc_bittree_reverse( - &coder->rc, coder->pos_align_encoder, + &coder->rc, coder->pos_align, ALIGN_BITS, pos_reduced & ALIGN_MASK); ++coder->align_price_count; } @@ -196,8 +223,8 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, if (len == 1) { update_short_rep(coder->state); } else { - length(&coder->rc, &coder->rep_len_encoder, pos_state, len); - coder->prev_len_encoder = &coder->rep_len_encoder; + length(&coder->rc, &coder->rep_len_encoder, pos_state, len, + coder->fast_mode); update_long_rep(coder->state); } } @@ -208,118 +235,124 @@ rep_match(lzma_coder *coder, const uint32_t pos_state, ////////// static void -encode_symbol(lzma_coder *coder, uint32_t pos, uint32_t len) +encode_symbol(lzma_coder *coder, lzma_mf *mf, + uint32_t back, uint32_t len, uint32_t position) { - const uint32_t pos_state = coder->now_pos & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; - if (len == 1 && pos == UINT32_MAX) { + if (back == UINT32_MAX) { // Literal i.e. eight-bit byte + assert(len == 1); rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 0); - literal(coder); + literal(coder, mf, position); } else { // Some type of match rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); - if (pos < REP_DISTANCES) { + if (back < REP_DISTANCES) { // It's a repeated match i.e. the same distance // has been used earlier. rc_bit(&coder->rc, &coder->is_rep[coder->state], 1); - rep_match(coder, pos_state, pos, len); + rep_match(coder, pos_state, back, len); } else { // Normal match rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, pos - REP_DISTANCES, len); + match(coder, pos_state, back - REP_DISTANCES, len); } - - coder->previous_byte = coder->lz.buffer[ - coder->lz.read_pos + len - 1 - - coder->additional_offset]; } - assert(coder->additional_offset >= len); - coder->additional_offset -= len; - coder->now_pos += len; + assert(mf->read_ahead >= len); + mf->read_ahead -= len; +} + + +static bool +encode_init(lzma_coder *coder, lzma_mf *mf) +{ + if (mf->read_pos == mf->read_limit) { + if (mf->action == LZMA_RUN) + return false; // We cannot do anything. + + // We are finishing (we cannot get here when flushing). + assert(mf->write_pos == mf->read_pos); + assert(mf->action == LZMA_FINISH); + } else { + // Do the actual initialization. The first LZMA symbol must + // always be a literal. + mf_skip(mf, 1); + mf->read_ahead = 0; + rc_bit(&coder->rc, &coder->is_match[0][0], 0); + rc_bittree(&coder->rc, coder->literal[0], 8, mf->buffer[0]); + } + + // Initialization is done (except if empty file). + coder->is_initialized = true; + + return true; } static void -encode_eopm(lzma_coder *coder) +encode_eopm(lzma_coder *coder, uint32_t position) { - const uint32_t pos_state = coder->now_pos & coder->pos_mask; + const uint32_t pos_state = position & coder->pos_mask; rc_bit(&coder->rc, &coder->is_match[coder->state][pos_state], 1); rc_bit(&coder->rc, &coder->is_rep[coder->state], 0); - match(coder, pos_state, UINT32_MAX, MATCH_MIN_LEN); + match(coder, pos_state, UINT32_MAX, MATCH_LEN_MIN); } -/** - * \brief LZMA encoder - * - * \return true if end of stream was reached, false otherwise. - */ -extern bool -lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size) +/// Number of bytes that a single encoding loop in lzma_lzma_encode() can +/// consume from the dictionary. This limit comes from lzma_lzma_optimum() +/// and may need to be updated if that function is significantly modified. +#define LOOP_INPUT_MAX (OPTS + 1) + + +extern lzma_ret +lzma_lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, uint32_t limit) { // Initialize the stream if no data has been encoded yet. - if (!coder->is_initialized) { - if (coder->lz.read_pos == coder->lz.read_limit) { - if (coder->lz.sequence == SEQ_RUN) - return false; // We cannot do anything. + if (!coder->is_initialized && !encode_init(coder, mf)) + return LZMA_OK; - // We are finishing (we cannot get here when flushing). - assert(coder->lz.write_pos == coder->lz.read_pos); - assert(coder->lz.sequence == SEQ_FINISH); - } else { - // Do the actual initialization. - uint32_t len; - uint32_t num_distance_pairs; - lzma_read_match_distances(coder, - &len, &num_distance_pairs); + // Get the lowest bits of the uncompressed offset from the LZ layer. + uint32_t position = mf_position(mf); - encode_symbol(coder, UINT32_MAX, 1); - - assert(coder->additional_offset == 0); + while (true) { + // Encode pending bits, if any. Calling this before encoding + // the next symbol is needed only with plain LZMA, since + // LZMA2 always provides big enough buffer to flush + // everything out from the range encoder. For the same reason, + // rc_encode() never returns true when this function is used + // as part of LZMA2 encoder. + if (rc_encode(&coder->rc, out, out_pos, out_size)) { + assert(limit == UINT32_MAX); + return LZMA_OK; } - // Initialization is done (except if empty file). - coder->is_initialized = true; - } - - // Encoding loop - while (true) { - // Encode pending bits, if any. - if (rc_encode(&coder->rc, out, out_pos, out_size)) - return false; + // With LZMA2 we need to take care that compressed size of + // a chunk doesn't get too big. + // TODO + if (limit != UINT32_MAX + && (mf->read_pos - mf->read_ahead >= limit + || *out_pos + rc_pending(&coder->rc) + >= (UINT32_C(1) << 16) + - LOOP_INPUT_MAX)) + break; // Check that there is some input to process. - if (coder->lz.read_pos >= coder->lz.read_limit) { - // If flushing or finishing, we must keep encoding - // until additional_offset becomes zero to make - // all the input available at output. - if (coder->lz.sequence == SEQ_RUN) - return false; + if (mf->read_pos >= mf->read_limit) { + if (mf->action == LZMA_RUN) + return LZMA_OK; - if (coder->additional_offset == 0) + if (mf->read_ahead == 0) break; } - assert(coder->lz.read_pos <= coder->lz.write_pos); - -#ifndef NDEBUG - if (coder->lz.sequence != SEQ_RUN) { - assert(coder->lz.read_limit == coder->lz.write_pos); - } else { - assert(coder->lz.read_limit + coder->lz.keep_size_after - == coder->lz.write_pos); - } -#endif - - uint32_t pos; - uint32_t len; - // Get optimal match (repeat position and length). // Value ranges for pos: // - [0, REP_DISTANCES): repeated match @@ -327,33 +360,324 @@ lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, // match at (pos - REP_DISTANCES) // - UINT32_MAX: not a match but a literal // Value ranges for len: - // - [MATCH_MIN_LEN, MATCH_MAX_LEN] - if (coder->best_compression) - lzma_get_optimum(coder, &pos, &len); + // - [MATCH_LEN_MIN, MATCH_LEN_MAX] + uint32_t len; + uint32_t back; + + if (coder->fast_mode) + lzma_lzma_optimum_fast(coder, mf, &back, &len); else - lzma_get_optimum_fast(coder, &pos, &len); + lzma_lzma_optimum_normal( + coder, mf, &back, &len, position); - encode_symbol(coder, pos, len); + encode_symbol(coder, mf, back, len, position); + + position += len; } - assert(!coder->longest_match_was_found); - - if (coder->is_flushed) { - coder->is_flushed = false; - return true; - } - - // We don't support encoding old LZMA streams without EOPM, and LZMA2 - // doesn't use EOPM at LZMA level. - if (coder->write_eopm) - encode_eopm(coder); - - rc_flush(&coder->rc); - - if (rc_encode(&coder->rc, out, out_pos, out_size)) { + if (!coder->is_flushed) { coder->is_flushed = true; - return false; + + // We don't support encoding plain LZMA streams without EOPM, + // and LZMA2 doesn't use EOPM at LZMA level. + if (limit == UINT32_MAX) + encode_eopm(coder, position); + + // Flush the remaining bytes from the range encoder. + rc_flush(&coder->rc); + + // Copy the remaining bytes to the output buffer. If there + // isn't enough output space, we will copy out the remaining + // bytes on the next call to this function by using + // the rc_encode() call in the encoding loop above. + if (rc_encode(&coder->rc, out, out_pos, out_size)) { + assert(limit == UINT32_MAX); + return LZMA_OK; + } } - return true; + // Make it ready for the next LZMA2 chunk. + coder->is_flushed = false; + + return LZMA_STREAM_END; +} + + +static lzma_ret +lzma_encode(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + // Plain LZMA has no support for sync-flushing. + if (unlikely(mf->action == LZMA_SYNC_FLUSH)) + return LZMA_HEADER_ERROR; + + return lzma_lzma_encode(coder, mf, out, out_pos, out_size, UINT32_MAX); +} + + +//////////////////// +// Initialization // +//////////////////// + +static bool +set_lz_options(lzma_lz_options *lz_options, const lzma_options_lzma *options) +{ + if (!is_lclppb_valid(options) + || options->fast_bytes < LZMA_FAST_BYTES_MIN + || options->fast_bytes > LZMA_FAST_BYTES_MAX) + return true; + + // FIXME validation + + lz_options->before_size = OPTS; + lz_options->dictionary_size = options->dictionary_size; + lz_options->after_size = LOOP_INPUT_MAX; + lz_options->match_len_max = MATCH_LEN_MAX; + lz_options->find_len_max = options->fast_bytes; + lz_options->match_finder = options->match_finder; + lz_options->match_finder_cycles = options->match_finder_cycles; + lz_options->preset_dictionary = options->preset_dictionary; + lz_options->preset_dictionary_size = options->preset_dictionary_size; + + return false; +} + + +static void +length_encoder_reset(lzma_length_encoder *lencoder, + const uint32_t num_pos_states, const bool fast_mode) +{ + bit_reset(lencoder->choice); + bit_reset(lencoder->choice2); + + for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { + bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); + bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); + } + + bittree_reset(lencoder->high, LEN_HIGH_BITS); + + if (!fast_mode) + for (size_t pos_state = 0; pos_state < num_pos_states; + ++pos_state) + length_update_prices(lencoder, pos_state); + + return; +} + + +extern void +lzma_lzma_encoder_reset(lzma_coder *coder, const lzma_options_lzma *options) +{ + assert(!coder->is_flushed); + + coder->pos_mask = (1U << options->pos_bits) - 1; + coder->literal_context_bits = options->literal_context_bits; + coder->literal_pos_mask = (1 << options->literal_pos_bits) - 1; + + + // Range coder + rc_reset(&coder->rc); + + // State + coder->state = 0; + for (size_t i = 0; i < REP_DISTANCES; ++i) + coder->reps[i] = 0; + + literal_init(coder->literal, options->literal_context_bits, + options->literal_pos_bits); + + // Bit encoders + for (size_t i = 0; i < STATES; ++i) { + for (size_t j = 0; j <= coder->pos_mask; ++j) { + bit_reset(coder->is_match[i][j]); + bit_reset(coder->is_rep0_long[i][j]); + } + + bit_reset(coder->is_rep[i]); + bit_reset(coder->is_rep0[i]); + bit_reset(coder->is_rep1[i]); + bit_reset(coder->is_rep2[i]); + } + + for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) + bit_reset(coder->pos_special[i]); + + // Bit tree encoders + for (size_t i = 0; i < LEN_TO_POS_STATES; ++i) + bittree_reset(coder->pos_slot[i], POS_SLOT_BITS); + + bittree_reset(coder->pos_align, ALIGN_BITS); + + // Length encoders + length_encoder_reset(&coder->match_len_encoder, + 1U << options->pos_bits, coder->fast_mode); + + length_encoder_reset(&coder->rep_len_encoder, + 1U << options->pos_bits, coder->fast_mode); + + // FIXME: Too big or too small won't work when resetting in the middle of LZMA2. + coder->match_price_count = UINT32_MAX / 2; + coder->align_price_count = UINT32_MAX / 2; + + coder->opts_end_index = 0; + coder->opts_current_index = 0; +} + + +extern lzma_ret +lzma_lzma_encoder_create(lzma_coder **coder_ptr, lzma_allocator *allocator, + const lzma_options_lzma *options, lzma_lz_options *lz_options) +{ + if (*coder_ptr == NULL) { + *coder_ptr = lzma_alloc(sizeof(lzma_coder), allocator); + if (*coder_ptr == NULL) + return LZMA_MEM_ERROR; + } + + lzma_coder *coder = *coder_ptr; + + // Validate options that aren't validated elsewhere. + if (!is_lclppb_valid(options) + || options->fast_bytes < LZMA_FAST_BYTES_MIN + || options->fast_bytes > LZMA_FAST_BYTES_MAX) + return LZMA_HEADER_ERROR; + + // Set compression mode. + switch (options->mode) { + case LZMA_MODE_FAST: + coder->fast_mode = true; + break; + + case LZMA_MODE_NORMAL: { + coder->fast_mode = false; + + // Set dist_table_size. + // Round the dictionary size up to next 2^n. + uint32_t log_size = 0; + while ((UINT32_C(1) << log_size) + < options->dictionary_size) + ++log_size; + + coder->dist_table_size = log_size * 2; + + // Length encoders' price table size + coder->match_len_encoder.table_size + = options->fast_bytes + 1 - MATCH_LEN_MIN; + coder->rep_len_encoder.table_size + = options->fast_bytes + 1 - MATCH_LEN_MIN; + break; + } + + default: + return LZMA_HEADER_ERROR; + } + + coder->is_initialized = false; + coder->is_flushed = false; + + lzma_lzma_encoder_reset(coder, options); + + // LZ encoder options FIXME validation + if (set_lz_options(lz_options, options)) + return LZMA_HEADER_ERROR; + + return LZMA_OK; +} + + +static lzma_ret +lzma_encoder_init(lzma_lz_encoder *lz, lzma_allocator *allocator, + const void *options, lzma_lz_options *lz_options) +{ + lz->code = &lzma_encode; + return lzma_lzma_encoder_create( + &lz->coder, allocator, options, lz_options); +} + + +extern lzma_ret +lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + // Initialization call chain: + // + // lzma_lzma_encoder_init() + // `-- lzma_lz_encoder_init() + // `-- lzma_encoder_init() + // `-- lzma_encoder_init2() + // + // The above complexity is to let LZ encoder store the pointer to + // the LZMA encoder structure. Encoding call tree: + // + // lz_encode() + // |-- fill_window() + // | `-- Next coder in the chain, if any + // `-- lzma_encode() + // |-- lzma_dict_find() + // `-- lzma_dict_skip() + // + // FIXME ^ + // + return lzma_lz_encoder_init( + next, allocator, filters, &lzma_encoder_init); +} + + +extern uint64_t +lzma_lzma_encoder_memusage(const void *options) +{ + lzma_lz_options lz_options; + if (set_lz_options(&lz_options, options)) + return UINT64_MAX; + + const uint64_t lz_memusage = lzma_lz_encoder_memusage(&lz_options); + if (lz_memusage == UINT64_MAX) + return UINT64_MAX; + + return (uint64_t)(sizeof(lzma_coder)) + lz_memusage; +} + + +extern bool +lzma_lzma_lclppb_encode(const lzma_options_lzma *options, uint8_t *byte) +{ + if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX + || options->literal_pos_bits + > LZMA_LITERAL_POS_BITS_MAX + || options->pos_bits > LZMA_POS_BITS_MAX + || options->literal_context_bits + + options->literal_pos_bits + > LZMA_LITERAL_BITS_MAX) + return true; + + *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9 + + options->literal_context_bits; + assert(*byte <= (4 * 5 + 4) * 9 + 8); + + return false; +} + + +#ifdef HAVE_ENCODER_LZMA +extern lzma_ret +lzma_lzma_props_encode(const void *options, uint8_t *out) +{ + const lzma_options_lzma *const opt = options; + + if (lzma_lzma_lclppb_encode(opt, out)) + return LZMA_PROG_ERROR; + + integer_write_32(out + 1, opt->dictionary_size); + + return LZMA_OK; +} +#endif + + +extern LZMA_API lzma_bool +lzma_mode_is_available(lzma_mode mode) +{ + return mode == LZMA_MODE_FAST || mode == LZMA_MODE_NORMAL; } diff --git a/src/liblzma/lzma/lzma_encoder.h b/src/liblzma/lzma/lzma_encoder.h index 1c57f80a..e270cc27 100644 --- a/src/liblzma/lzma/lzma_encoder.h +++ b/src/liblzma/lzma/lzma_encoder.h @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.h -/// \brief LZMA method handler API +/// \brief LZMA encoder API // // Copyright (C) 1999-2006 Igor Pavlov // Copyright (C) 2007 Lasse Collin @@ -23,13 +23,47 @@ #include "common.h" + extern lzma_ret lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, const lzma_filter_info *filters); -extern bool lzma_lzma_encode_properties( + +extern uint64_t lzma_lzma_encoder_memusage(const void *options); + +extern lzma_ret lzma_lzma_props_encode(const void *options, uint8_t *out); + + +/// Encodes lc/lp/pb into one byte. Returns false on success and true on error. +extern bool lzma_lzma_lclppb_encode( const lzma_options_lzma *options, uint8_t *byte); + +#ifdef HAVE_SMALL + /// Initializes the lzma_fastpos[] array. extern void lzma_fastpos_init(void); #endif + + +#ifdef LZMA_LZ_ENCODER_H + +/// Initializes raw LZMA encoder; this is used by LZMA2. +extern lzma_ret lzma_lzma_encoder_create( + lzma_coder **coder_ptr, lzma_allocator *allocator, + const lzma_options_lzma *options, lzma_lz_options *lz_options); + + +/// Resets an already initialized LZMA encoder; this is used by LZMA2. +extern void lzma_lzma_encoder_reset( + lzma_coder *coder, const lzma_options_lzma *options); + + +extern lzma_ret lzma_lzma_encode(lzma_coder *restrict coder, + lzma_mf *restrict mf, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + uint32_t read_limit); + +#endif + +#endif diff --git a/src/liblzma/lzma/lzma_encoder_features.c b/src/liblzma/lzma/lzma_encoder_features.c index 56e59c6a..9fecee48 100644 --- a/src/liblzma/lzma/lzma_encoder_features.c +++ b/src/liblzma/lzma/lzma_encoder_features.c @@ -22,7 +22,7 @@ static lzma_mode modes[] = { LZMA_MODE_FAST, - LZMA_MODE_BEST, + LZMA_MODE_NORMAL, LZMA_MODE_INVALID }; diff --git a/src/liblzma/lzma/lzma_encoder_getoptimum.c b/src/liblzma/lzma/lzma_encoder_getoptimum.c deleted file mode 100644 index b175e4cb..00000000 --- a/src/liblzma/lzma/lzma_encoder_getoptimum.c +++ /dev/null @@ -1,925 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_encoder_getoptimum.c -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -// NOTE: If you want to keep the line length in 80 characters, set -// tab width to 4 or less in your editor when editing this file. - - -// "Would you love the monster code? -// Could you understand beauty of the beast?" -// --Adapted from Lordi's "Would you love a monster man". - - -#include "lzma_encoder_private.h" -#include "fastpos.h" - - -#define length_get_price(length_encoder, symbol, pos_state) \ - (length_encoder).prices[pos_state][symbol] - - -#define get_rep_len_1_price(state, pos_state) \ - bit_get_price_0(coder->is_rep0[state]) \ - + bit_get_price_0(coder->is_rep0_long[state][pos_state]) - - -// Adds to price_target. -#define get_pure_rep_price(price_target, rep_index, state, pos_state) \ -do { \ - if ((rep_index) == 0) { \ - price_target += bit_get_price_0(coder->is_rep0[state]); \ - price_target += bit_get_price_1( \ - coder->is_rep0_long[state][pos_state]); \ - } else { \ - price_target += bit_get_price_1(coder->is_rep0[state]); \ - if ((rep_index) == 1) { \ - price_target += bit_get_price_0(coder->is_rep1[state]); \ - } else { \ - price_target += bit_get_price_1(coder->is_rep1[state]); \ - price_target += bit_get_price( \ - coder->is_rep2[state], (rep_index) - 2); \ - } \ - } \ -} while (0) - - -// Adds to price_target. -#define get_rep_price(price_target, rep_index, len, state, pos_state) \ -do { \ - get_pure_rep_price(price_target, rep_index, state, pos_state); \ - price_target += length_get_price(coder->rep_len_encoder, \ - (len) - MATCH_MIN_LEN, pos_state); \ -} while (0) - - -// Adds to price_target. -#define get_pos_len_price(price_target, pos, len, pos_state) \ -do { \ - const uint32_t len_to_pos_state_tmp = get_len_to_pos_state(len); \ - if ((pos) < FULL_DISTANCES) { \ - price_target += distances_prices[len_to_pos_state_tmp][pos]; \ - } else { \ - price_target \ - += pos_slot_prices[len_to_pos_state_tmp][get_pos_slot_2(pos)] \ - + align_prices[(pos) & ALIGN_MASK]; \ - } \ - price_target += length_get_price( \ - coder->match_len_encoder, (len) - MATCH_MIN_LEN, pos_state); \ -} while (0) - - -// Three macros to manipulate lzma_optimal structures: -#define make_as_char(opt) \ -do { \ - (opt).back_prev = UINT32_MAX; \ - (opt).prev_1_is_char = false; \ -} while (0) - - -#define make_as_short_rep(opt) \ -do { \ - (opt).back_prev = 0; \ - (opt).prev_1_is_char = false; \ -} while (0) - - -#define is_short_rep(opt) \ - ((opt).back_prev == 0) - - -static void -fill_length_prices(lzma_length_encoder *lc, uint32_t pos_state) -{ - const uint32_t num_symbols = lc->table_size; - const uint32_t a0 = bit_get_price_0(lc->choice); - const uint32_t a1 = bit_get_price_1(lc->choice); - const uint32_t b0 = a1 + bit_get_price_0(lc->choice2); - const uint32_t b1 = a1 + bit_get_price_1(lc->choice2); - - uint32_t *prices = lc->prices[pos_state]; - uint32_t i = 0; - - for (i = 0; i < num_symbols && i < LEN_LOW_SYMBOLS; ++i) - prices[i] = a0 + bittree_get_price(lc->low[pos_state], - LEN_LOW_BITS, i); - - for (; i < num_symbols && i < LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; ++i) - prices[i] = b0 + bittree_get_price(lc->mid[pos_state], - LEN_MID_BITS, i - LEN_LOW_SYMBOLS); - - for (; i < num_symbols; ++i) - prices[i] = b1 + bittree_get_price(lc->high, LEN_HIGH_BITS, - i - LEN_LOW_SYMBOLS - LEN_MID_SYMBOLS); - - lc->counters[pos_state] = num_symbols; - - return; -} - - -static void -fill_distances_prices(lzma_coder *coder) -{ - uint32_t temp_prices[FULL_DISTANCES]; - - for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { - const uint32_t pos_slot = get_pos_slot(i); - const uint32_t footer_bits = ((pos_slot >> 1) - 1); - const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; - temp_prices[i] = bittree_reverse_get_price( - coder->pos_encoders + base - pos_slot - 1, - footer_bits, i - base); - } - - const uint32_t dist_table_size = coder->dist_table_size; - - for (uint32_t len_to_pos_state = 0; - len_to_pos_state < LEN_TO_POS_STATES; - ++len_to_pos_state) { - - const probability *encoder = coder->pos_slot_encoder[len_to_pos_state]; - uint32_t *pos_slot_prices = coder->pos_slot_prices[len_to_pos_state]; - - for (uint32_t pos_slot = 0; - pos_slot < dist_table_size; - ++pos_slot) { - pos_slot_prices[pos_slot] = bittree_get_price(encoder, - POS_SLOT_BITS, pos_slot); - } - - for (uint32_t pos_slot = END_POS_MODEL_INDEX; - pos_slot < dist_table_size; - ++pos_slot) - pos_slot_prices[pos_slot] += (((pos_slot >> 1) - 1) - - ALIGN_BITS) << BIT_PRICE_SHIFT_BITS; - - - uint32_t *distances_prices - = coder->distances_prices[len_to_pos_state]; - - uint32_t i; - for (i = 0; i < START_POS_MODEL_INDEX; ++i) - distances_prices[i] = pos_slot_prices[i]; - - for (; i < FULL_DISTANCES; ++i) - distances_prices[i] = pos_slot_prices[get_pos_slot(i)] - + temp_prices[i]; - } - - coder->match_price_count = 0; - - return; -} - - -static void -fill_align_prices(lzma_coder *coder) -{ - for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i) - coder->align_prices[i] = bittree_reverse_get_price( - coder->pos_align_encoder, ALIGN_BITS, i); - - coder->align_price_count = 0; - return; -} - - -// The first argument is a pointer returned by literal_get_subcoder(). -static uint32_t -literal_get_price(const probability *encoders, const bool match_mode, - const uint8_t match_byte, const uint8_t symbol) -{ - uint32_t price = 0; - uint32_t context = 1; - int i = 8; - - if (match_mode) { - do { - --i; - const uint32_t match_bit = (match_byte >> i) & 1; - const uint32_t bit = (symbol >> i) & 1; - const uint32_t subcoder_index - = 0x100 + (match_bit << 8) + context; - - price += bit_get_price(encoders[subcoder_index], bit); - context = (context << 1) | bit; - - if (match_bit != bit) - break; - - } while (i != 0); - } - - while (i != 0) { - --i; - const uint32_t bit = (symbol >> i) & 1; - price += bit_get_price(encoders[context], bit); - context = (context << 1) | bit; - } - - return price; -} - - -static void -backward(lzma_coder *restrict coder, uint32_t *restrict len_res, - uint32_t *restrict back_res, uint32_t cur) -{ - coder->optimum_end_index = cur; - - uint32_t pos_mem = coder->optimum[cur].pos_prev; - uint32_t back_mem = coder->optimum[cur].back_prev; - - do { - if (coder->optimum[cur].prev_1_is_char) { - make_as_char(coder->optimum[pos_mem]); - coder->optimum[pos_mem].pos_prev = pos_mem - 1; - - if (coder->optimum[cur].prev_2) { - coder->optimum[pos_mem - 1].prev_1_is_char = false; - coder->optimum[pos_mem - 1].pos_prev - = coder->optimum[cur].pos_prev_2; - coder->optimum[pos_mem - 1].back_prev - = coder->optimum[cur].back_prev_2; - } - } - - uint32_t pos_prev = pos_mem; - uint32_t back_cur = back_mem; - - back_mem = coder->optimum[pos_prev].back_prev; - pos_mem = coder->optimum[pos_prev].pos_prev; - - coder->optimum[pos_prev].back_prev = back_cur; - coder->optimum[pos_prev].pos_prev = cur; - cur = pos_prev; - - } while (cur != 0); - - coder->optimum_current_index = coder->optimum[0].pos_prev; - *len_res = coder->optimum[0].pos_prev; - *back_res = coder->optimum[0].back_prev; - - return; -} - - -extern void -lzma_get_optimum(lzma_coder *restrict coder, - uint32_t *restrict back_res, uint32_t *restrict len_res) -{ - uint32_t position = coder->now_pos; - uint32_t pos_state = position & coder->pos_mask; - - // Update the price tables. In the C++ LZMA SDK 4.42 this was done in both - // initialization function and in the main loop. In liblzma they were - // moved into this single place. - if (coder->additional_offset == 0) { - if (coder->match_price_count >= (1 << 7)) - fill_distances_prices(coder); - - if (coder->align_price_count >= ALIGN_TABLE_SIZE) - fill_align_prices(coder); - } - - if (coder->prev_len_encoder != NULL) { - if (--coder->prev_len_encoder->counters[pos_state] == 0) - fill_length_prices(coder->prev_len_encoder, pos_state); - - coder->prev_len_encoder = NULL; - } - - - if (coder->optimum_end_index != coder->optimum_current_index) { - *len_res = coder->optimum[coder->optimum_current_index].pos_prev - - coder->optimum_current_index; - *back_res = coder->optimum[coder->optimum_current_index].back_prev; - coder->optimum_current_index = coder->optimum[ - coder->optimum_current_index].pos_prev; - return; - } - - coder->optimum_current_index = 0; - coder->optimum_end_index = 0; - - - const uint32_t fast_bytes = coder->fast_bytes; - uint32_t *match_distances = coder->match_distances; - - uint32_t len_main; - uint32_t num_distance_pairs; - - if (!coder->longest_match_was_found) { - lzma_read_match_distances(coder, &len_main, &num_distance_pairs); - } else { - len_main = coder->longest_match_length; - num_distance_pairs = coder->num_distance_pairs; - coder->longest_match_was_found = false; - } - - - const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1; - uint32_t num_available_bytes - = coder->lz.write_pos - coder->lz.read_pos + 1; - if (num_available_bytes < 2) { - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - - if (num_available_bytes > MATCH_MAX_LEN) - num_available_bytes = MATCH_MAX_LEN; - - - uint32_t reps[REP_DISTANCES]; - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - reps[i] = coder->reps[i]; - const uint32_t back_offset = reps[i] + 1; - - if (buf[0] != *(buf - back_offset) - || buf[1] != *(buf + 1 - back_offset)) { - rep_lens[i] = 0; - continue; - } - - uint32_t len_test; - for (len_test = 2; len_test < num_available_bytes - && buf[len_test] == *(buf + len_test - back_offset); - ++len_test) ; - - rep_lens[i] = len_test; - if (len_test > rep_lens[rep_max_index]) - rep_max_index = i; - } - - if (rep_lens[rep_max_index] >= fast_bytes) { - *back_res = rep_max_index; - *len_res = rep_lens[rep_max_index]; - move_pos(*len_res - 1); - return; - } - - - if (len_main >= fast_bytes) { - *back_res = match_distances[num_distance_pairs] + REP_DISTANCES; - *len_res = len_main; - move_pos(len_main - 1); - return; - } - - uint8_t current_byte = *buf; - uint8_t match_byte = *(buf - reps[0] - 1); - - if (len_main < 2 && current_byte != match_byte - && rep_lens[rep_max_index] < 2) { - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - - coder->optimum[0].state = coder->state; - - coder->optimum[1].price = bit_get_price_0( - coder->is_match[coder->state][pos_state]) - + literal_get_price( - literal_get_subcoder(coder->literal_coder, - position, coder->previous_byte), - !is_literal_state(coder->state), match_byte, current_byte); - - make_as_char(coder->optimum[1]); - - uint32_t match_price - = bit_get_price_1(coder->is_match[coder->state][pos_state]); - uint32_t rep_match_price - = match_price + bit_get_price_1(coder->is_rep[coder->state]); - - - if (match_byte == current_byte) { - const uint32_t short_rep_price = rep_match_price - + get_rep_len_1_price(coder->state, pos_state); - - if (short_rep_price < coder->optimum[1].price) { - coder->optimum[1].price = short_rep_price; - make_as_short_rep(coder->optimum[1]); - } - } - - uint32_t len_end = (len_main >= rep_lens[rep_max_index]) - ? len_main - : rep_lens[rep_max_index]; - - if (len_end < 2) { - *back_res = coder->optimum[1].back_prev; - *len_res = 1; - return; - } - - coder->optimum[1].pos_prev = 0; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) - coder->optimum[0].backs[i] = reps[i]; - - uint32_t len = len_end; - do { - coder->optimum[len].price = INFINITY_PRICE; - } while (--len >= 2); - - - uint32_t (*distances_prices)[FULL_DISTANCES] = coder->distances_prices; - uint32_t (*pos_slot_prices)[DIST_TABLE_SIZE_MAX] = coder->pos_slot_prices; - uint32_t *align_prices = coder->align_prices; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - uint32_t rep_len = rep_lens[i]; - if (rep_len < 2) - continue; - - uint32_t price = rep_match_price; - get_pure_rep_price(price, i, coder->state, pos_state); - - do { - const uint32_t cur_and_len_price = price - + length_get_price( - coder->rep_len_encoder, - rep_len - 2, pos_state); - - if (cur_and_len_price < coder->optimum[rep_len].price) { - coder->optimum[rep_len].price = cur_and_len_price; - coder->optimum[rep_len].pos_prev = 0; - coder->optimum[rep_len].back_prev = i; - coder->optimum[rep_len].prev_1_is_char = false; - } - } while (--rep_len >= 2); - } - - - uint32_t normal_match_price = match_price - + bit_get_price_0(coder->is_rep[coder->state]); - - len = (rep_lens[0] >= 2) ? rep_lens[0] + 1 : 2; - - if (len <= len_main) { - uint32_t offs = 0; - - while (len > match_distances[offs + 1]) - offs += 2; - - for(; ; ++len) { - const uint32_t distance = match_distances[offs + 2]; - uint32_t cur_and_len_price = normal_match_price; - get_pos_len_price(cur_and_len_price, distance, len, pos_state); - - if (cur_and_len_price < coder->optimum[len].price) { - coder->optimum[len].price = cur_and_len_price; - coder->optimum[len].pos_prev = 0; - coder->optimum[len].back_prev = distance + REP_DISTANCES; - coder->optimum[len].prev_1_is_char = false; - } - - if (len == match_distances[offs + 1]) { - offs += 2; - if (offs == num_distance_pairs) - break; - } - } - } - - - ////////////////// - // Big loop ;-) // - ////////////////// - - uint32_t cur = 0; - - // The rest of this function is a huge while-loop. To avoid extreme - // indentation, the indentation level is not increased here. - while (true) { - - ++cur; - - assert(cur < OPTS); - - if (cur == len_end) { - backward(coder, len_res, back_res, cur); - return; - } - - uint32_t new_len; - - lzma_read_match_distances(coder, &new_len, &num_distance_pairs); - - if (new_len >= fast_bytes) { - coder->num_distance_pairs = num_distance_pairs; - coder->longest_match_length = new_len; - coder->longest_match_was_found = true; - backward(coder, len_res, back_res, cur); - return; - } - - - ++position; - - uint32_t pos_prev = coder->optimum[cur].pos_prev; - uint32_t state; - - if (coder->optimum[cur].prev_1_is_char) { - --pos_prev; - - if (coder->optimum[cur].prev_2) { - state = coder->optimum[coder->optimum[cur].pos_prev_2].state; - - if (coder->optimum[cur].back_prev_2 < REP_DISTANCES) - update_long_rep(state); - else - update_match(state); - - } else { - state = coder->optimum[pos_prev].state; - } - - update_literal(state); - - } else { - state = coder->optimum[pos_prev].state; - } - - if (pos_prev == cur - 1) { - if (is_short_rep(coder->optimum[cur])) - update_short_rep(state); - else - update_literal(state); - } else { - uint32_t pos; - if (coder->optimum[cur].prev_1_is_char && coder->optimum[cur].prev_2) { - pos_prev = coder->optimum[cur].pos_prev_2; - pos = coder->optimum[cur].back_prev_2; - update_long_rep(state); - } else { - pos = coder->optimum[cur].back_prev; - if (pos < REP_DISTANCES) - update_long_rep(state); - else - update_match(state); - } - - if (pos < REP_DISTANCES) { - reps[0] = coder->optimum[pos_prev].backs[pos]; - - uint32_t i; - for (i = 1; i <= pos; ++i) - reps[i] = coder->optimum[pos_prev].backs[i - 1]; - - for (; i < REP_DISTANCES; ++i) - reps[i] = coder->optimum[pos_prev].backs[i]; - - } else { - reps[0] = pos - REP_DISTANCES; - - for (uint32_t i = 1; i < REP_DISTANCES; ++i) - reps[i] = coder->optimum[pos_prev].backs[i - 1]; - } - } - - coder->optimum[cur].state = state; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) - coder->optimum[cur].backs[i] = reps[i]; - - const uint32_t cur_price = coder->optimum[cur].price; - - buf = coder->lz.buffer + coder->lz.read_pos - 1; - current_byte = *buf; - match_byte = *(buf - reps[0] - 1); - - pos_state = position & coder->pos_mask; - - const uint32_t cur_and_1_price = cur_price - + bit_get_price_0(coder->is_match[state][pos_state]) - + literal_get_price( - literal_get_subcoder(coder->literal_coder, - position, buf[-1]), - !is_literal_state(state), match_byte, current_byte); - - bool next_is_char = false; - - if (cur_and_1_price < coder->optimum[cur + 1].price) { - coder->optimum[cur + 1].price = cur_and_1_price; - coder->optimum[cur + 1].pos_prev = cur; - make_as_char(coder->optimum[cur + 1]); - next_is_char = true; - } - - match_price = cur_price - + bit_get_price_1(coder->is_match[state][pos_state]); - rep_match_price = match_price - + bit_get_price_1(coder->is_rep[state]); - - if (match_byte == current_byte - && !(coder->optimum[cur + 1].pos_prev < cur - && coder->optimum[cur + 1].back_prev == 0)) { - - const uint32_t short_rep_price = rep_match_price - + get_rep_len_1_price(state, pos_state); - - if (short_rep_price <= coder->optimum[cur + 1].price) { - coder->optimum[cur + 1].price = short_rep_price; - coder->optimum[cur + 1].pos_prev = cur; - make_as_short_rep(coder->optimum[cur + 1]); - next_is_char = true; - } - } - - uint32_t num_available_bytes_full - = coder->lz.write_pos - coder->lz.read_pos + 1; - num_available_bytes_full = MIN(OPTS - 1 - cur, num_available_bytes_full); - num_available_bytes = num_available_bytes_full; - - if (num_available_bytes < 2) - continue; - - if (num_available_bytes > fast_bytes) - num_available_bytes = fast_bytes; - - if (!next_is_char && match_byte != current_byte) { // speed optimization - // try literal + rep0 - const uint32_t back_offset = reps[0] + 1; - const uint32_t limit = MIN(num_available_bytes_full, fast_bytes + 1); - - uint32_t temp; - for (temp = 1; temp < limit - && buf[temp] == *(buf + temp - back_offset); - ++temp) ; - - const uint32_t len_test_2 = temp - 1; - - if (len_test_2 >= 2) { - uint32_t state_2 = state; - update_literal(state_2); - - const uint32_t pos_state_next = (position + 1) & coder->pos_mask; - const uint32_t next_rep_match_price = cur_and_1_price - + bit_get_price_1(coder->is_match[state_2][pos_state_next]) - + bit_get_price_1(coder->is_rep[state_2]); - - // for (; len_test_2 >= 2; --len_test_2) { - const uint32_t offset = cur + 1 + len_test_2; - - while (len_end < offset) - coder->optimum[++len_end].price = INFINITY_PRICE; - - uint32_t cur_and_len_price = next_rep_match_price; - get_rep_price(cur_and_len_price, - 0, len_test_2, state_2, pos_state_next); - - if (cur_and_len_price < coder->optimum[offset].price) { - coder->optimum[offset].price = cur_and_len_price; - coder->optimum[offset].pos_prev = cur + 1; - coder->optimum[offset].back_prev = 0; - coder->optimum[offset].prev_1_is_char = true; - coder->optimum[offset].prev_2 = false; - } -// } - } - } - - - uint32_t start_len = 2; // speed optimization - - for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { - const uint32_t back_offset = reps[rep_index] + 1; - - if (buf[0] != *(buf - back_offset) || buf[1] != *(buf + 1 - back_offset)) - continue; - - uint32_t len_test; - for (len_test = 2; len_test < num_available_bytes - && buf[len_test] == *(buf + len_test - back_offset); - ++len_test) ; - - while (len_end < cur + len_test) - coder->optimum[++len_end].price = INFINITY_PRICE; - - const uint32_t len_test_temp = len_test; - uint32_t price = rep_match_price; - get_pure_rep_price(price, rep_index, state, pos_state); - - do { - const uint32_t cur_and_len_price = price - + length_get_price(coder->rep_len_encoder, - len_test - 2, pos_state); - - if (cur_and_len_price < coder->optimum[cur + len_test].price) { - coder->optimum[cur + len_test].price = cur_and_len_price; - coder->optimum[cur + len_test].pos_prev = cur; - coder->optimum[cur + len_test].back_prev = rep_index; - coder->optimum[cur + len_test].prev_1_is_char = false; - } - } while (--len_test >= 2); - - len_test = len_test_temp; - - if (rep_index == 0) - start_len = len_test + 1; - - - uint32_t len_test_2 = len_test + 1; - const uint32_t limit = MIN(num_available_bytes_full, - len_test_2 + fast_bytes); - for (; len_test_2 < limit - && buf[len_test_2] == *(buf + len_test_2 - back_offset); - ++len_test_2) ; - - len_test_2 -= len_test + 1; - - if (len_test_2 >= 2) { - uint32_t state_2 = state; - update_long_rep(state_2); - - uint32_t pos_state_next = (position + len_test) & coder->pos_mask; - - const uint32_t cur_and_len_char_price = price - + length_get_price(coder->rep_len_encoder, - len_test - 2, pos_state) - + bit_get_price_0(coder->is_match[state_2][pos_state_next]) - + literal_get_price( - literal_get_subcoder(coder->literal_coder, - position + len_test, buf[len_test - 1]), - true, *(buf + len_test - back_offset), buf[len_test]); - - update_literal(state_2); - - pos_state_next = (position + len_test + 1) & coder->pos_mask; - - const uint32_t next_rep_match_price = cur_and_len_char_price - + bit_get_price_1(coder->is_match[state_2][pos_state_next]) - + bit_get_price_1(coder->is_rep[state_2]); - -// for(; len_test_2 >= 2; len_test_2--) { - const uint32_t offset = cur + len_test + 1 + len_test_2; - - while (len_end < offset) - coder->optimum[++len_end].price = INFINITY_PRICE; - - uint32_t cur_and_len_price = next_rep_match_price; - get_rep_price(cur_and_len_price, - 0, len_test_2, state_2, pos_state_next); - - if (cur_and_len_price < coder->optimum[offset].price) { - coder->optimum[offset].price = cur_and_len_price; - coder->optimum[offset].pos_prev = cur + len_test + 1; - coder->optimum[offset].back_prev = 0; - coder->optimum[offset].prev_1_is_char = true; - coder->optimum[offset].prev_2 = true; - coder->optimum[offset].pos_prev_2 = cur; - coder->optimum[offset].back_prev_2 = rep_index; - } -// } - } - } - - -// for (uint32_t len_test = 2; len_test <= new_len; ++len_test) - if (new_len > num_available_bytes) { - new_len = num_available_bytes; - - for (num_distance_pairs = 0; - new_len > match_distances[num_distance_pairs + 1]; - num_distance_pairs += 2) ; - - match_distances[num_distance_pairs + 1] = new_len; - num_distance_pairs += 2; - } - - - if (new_len >= start_len) { - normal_match_price = match_price - + bit_get_price_0(coder->is_rep[state]); - - while (len_end < cur + new_len) - coder->optimum[++len_end].price = INFINITY_PRICE; - - uint32_t offs = 0; - while (start_len > match_distances[offs + 1]) - offs += 2; - - uint32_t cur_back = match_distances[offs + 2]; - uint32_t pos_slot = get_pos_slot_2(cur_back); - - for (uint32_t len_test = start_len; ; ++len_test) { - uint32_t cur_and_len_price = normal_match_price; - const uint32_t len_to_pos_state = get_len_to_pos_state(len_test); - - if (cur_back < FULL_DISTANCES) - cur_and_len_price += distances_prices[ - len_to_pos_state][cur_back]; - else - cur_and_len_price += pos_slot_prices[ - len_to_pos_state][pos_slot] - + align_prices[cur_back & ALIGN_MASK]; - - cur_and_len_price += length_get_price(coder->match_len_encoder, - len_test - MATCH_MIN_LEN, pos_state); - - if (cur_and_len_price < coder->optimum[cur + len_test].price) { - coder->optimum[cur + len_test].price = cur_and_len_price; - coder->optimum[cur + len_test].pos_prev = cur; - coder->optimum[cur + len_test].back_prev - = cur_back + REP_DISTANCES; - coder->optimum[cur + len_test].prev_1_is_char = false; - } - - if (len_test == match_distances[offs + 1]) { - // Try Match + Literal + Rep0 - const uint32_t back_offset = cur_back + 1; - uint32_t len_test_2 = len_test + 1; - const uint32_t limit = MIN(num_available_bytes_full, - len_test_2 + fast_bytes); - - for (; len_test_2 < limit && - buf[len_test_2] == *(buf + len_test_2 - back_offset); - ++len_test_2) ; - - len_test_2 -= len_test + 1; - - if (len_test_2 >= 2) { - uint32_t state_2 = state; - update_match(state_2); - uint32_t pos_state_next - = (position + len_test) & coder->pos_mask; - - const uint32_t cur_and_len_char_price = cur_and_len_price - + bit_get_price_0( - coder->is_match[state_2][pos_state_next]) - + literal_get_price( - literal_get_subcoder( - coder->literal_coder, - position + len_test, - buf[len_test - 1]), - true, - *(buf + len_test - back_offset), - buf[len_test]); - - update_literal(state_2); - pos_state_next = (pos_state_next + 1) & coder->pos_mask; - - const uint32_t next_rep_match_price - = cur_and_len_char_price - + bit_get_price_1( - coder->is_match[state_2][pos_state_next]) - + bit_get_price_1(coder->is_rep[state_2]); - - // for(; len_test_2 >= 2; --len_test_2) { - const uint32_t offset = cur + len_test + 1 + len_test_2; - - while (len_end < offset) - coder->optimum[++len_end].price = INFINITY_PRICE; - - cur_and_len_price = next_rep_match_price; - get_rep_price(cur_and_len_price, - 0, len_test_2, state_2, pos_state_next); - - if (cur_and_len_price < coder->optimum[offset].price) { - coder->optimum[offset].price = cur_and_len_price; - coder->optimum[offset].pos_prev = cur + len_test + 1; - coder->optimum[offset].back_prev = 0; - coder->optimum[offset].prev_1_is_char = true; - coder->optimum[offset].prev_2 = true; - coder->optimum[offset].pos_prev_2 = cur; - coder->optimum[offset].back_prev_2 - = cur_back + REP_DISTANCES; - } -// } - } - - offs += 2; - if (offs == num_distance_pairs) - break; - - cur_back = match_distances[offs + 2]; - if (cur_back >= FULL_DISTANCES) - pos_slot = get_pos_slot_2(cur_back); - } - } - } - - } // Closes: while (true) -} diff --git a/src/liblzma/lzma/lzma_encoder_getoptimumfast.c b/src/liblzma/lzma/lzma_encoder_getoptimumfast.c deleted file mode 100644 index fa06be21..00000000 --- a/src/liblzma/lzma/lzma_encoder_getoptimumfast.c +++ /dev/null @@ -1,201 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_encoder_getoptimumfast.c -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -// NOTE: If you want to keep the line length in 80 characters, set -// tab width to 4 or less in your editor when editing this file. - - -#include "lzma_encoder_private.h" - - -#define change_pair(small_dist, big_dist) \ - (((big_dist) >> 7) > (small_dist)) - - -extern void -lzma_get_optimum_fast(lzma_coder *restrict coder, - uint32_t *restrict back_res, uint32_t *restrict len_res) -{ - // Local copies - const uint32_t fast_bytes = coder->fast_bytes; - - uint32_t len_main; - uint32_t num_distance_pairs; - if (!coder->longest_match_was_found) { - lzma_read_match_distances(coder, &len_main, &num_distance_pairs); - } else { - len_main = coder->longest_match_length; - num_distance_pairs = coder->num_distance_pairs; - coder->longest_match_was_found = false; - } - - const uint8_t *buf = coder->lz.buffer + coder->lz.read_pos - 1; - uint32_t num_available_bytes - = coder->lz.write_pos - coder->lz.read_pos + 1; - - if (num_available_bytes < 2) { - // There's not enough input left to encode a match. - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - - if (num_available_bytes > MATCH_MAX_LEN) - num_available_bytes = MATCH_MAX_LEN; - - - // Look for repetitive matches; scan the previous four match distances - uint32_t rep_lens[REP_DISTANCES]; - uint32_t rep_max_index = 0; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - const uint32_t back_offset = coder->reps[i] + 1; - - // If the first two bytes (2 == MATCH_MIN_LEN) do not match, - // this rep_distance[i] is not useful. This is indicated - // using zero as the length of the repetitive match. - if (buf[0] != *(buf - back_offset) - || buf[1] != *(buf + 1 - back_offset)) { - rep_lens[i] = 0; - continue; - } - - // The first two bytes matched. - // Calculate the length of the match. - uint32_t len; - for (len = 2; len < num_available_bytes - && buf[len] == *(buf + len - back_offset); - ++len) ; - - // If we have found a repetitive match that is at least - // as long as fast_bytes, return it immediatelly. - if (len >= fast_bytes) { - *back_res = i; - *len_res = len; - move_pos(len - 1); - return; - } - - rep_lens[i] = len; - - // After this loop, rep_lens[rep_max_index] is the biggest - // value of all values in rep_lens[]. - if (len > rep_lens[rep_max_index]) - rep_max_index = i; - } - - - if (len_main >= fast_bytes) { - *back_res = coder->match_distances[num_distance_pairs] - + REP_DISTANCES; - *len_res = len_main; - move_pos(len_main - 1); - return; - } - - uint32_t back_main = 0; - if (len_main >= 2) { - back_main = coder->match_distances[num_distance_pairs]; - - while (num_distance_pairs > 2 && len_main == - coder->match_distances[num_distance_pairs - 3] + 1) { - if (!change_pair(coder->match_distances[ - num_distance_pairs - 2], back_main)) - break; - - num_distance_pairs -= 2; - len_main = coder->match_distances[num_distance_pairs - 1]; - back_main = coder->match_distances[num_distance_pairs]; - } - - if (len_main == 2 && back_main >= 0x80) - len_main = 1; - } - - if (rep_lens[rep_max_index] >= 2) { - if (rep_lens[rep_max_index] + 1 >= len_main - || (rep_lens[rep_max_index] + 2 >= len_main - && (back_main > (1 << 9))) - || (rep_lens[rep_max_index] + 3 >= len_main - && (back_main > (1 << 15)))) { - *back_res = rep_max_index; - *len_res = rep_lens[rep_max_index]; - move_pos(*len_res - 1); - return; - } - } - - if (len_main >= 2 && num_available_bytes > 2) { - lzma_read_match_distances(coder, &coder->longest_match_length, - &coder->num_distance_pairs); - - if (coder->longest_match_length >= 2) { - const uint32_t new_distance = coder->match_distances[ - coder->num_distance_pairs]; - - if ((coder->longest_match_length >= len_main - && new_distance < back_main) - || (coder->longest_match_length == len_main + 1 - && !change_pair(back_main, new_distance)) - || (coder->longest_match_length > len_main + 1) - || (coder->longest_match_length + 1 >= len_main - && len_main >= 3 - && change_pair(new_distance, back_main))) { - coder->longest_match_was_found = true; - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - } - - ++buf; - --num_available_bytes; - - for (uint32_t i = 0; i < REP_DISTANCES; ++i) { - const uint32_t back_offset = coder->reps[i] + 1; - - if (buf[1] != *(buf + 1 - back_offset) - || buf[2] != *(buf + 2 - back_offset)) { - rep_lens[i] = 0; - continue; - } - - uint32_t len; - for (len = 2; len < num_available_bytes - && buf[len] == *(buf + len - back_offset); - ++len) ; - - if (len + 1 >= len_main) { - coder->longest_match_was_found = true; - *back_res = UINT32_MAX; - *len_res = 1; - return; - } - } - - *back_res = back_main + REP_DISTANCES; - *len_res = len_main; - move_pos(len_main - 2); - return; - } - - *back_res = UINT32_MAX; - *len_res = 1; - return; -} diff --git a/src/liblzma/lzma/lzma_encoder_init.c b/src/liblzma/lzma/lzma_encoder_init.c deleted file mode 100644 index 21335f95..00000000 --- a/src/liblzma/lzma/lzma_encoder_init.c +++ /dev/null @@ -1,228 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_encoder_init.c -/// \brief Creating, resetting and destroying the LZMA encoder -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "lzma_encoder_private.h" - - -/// \brief Initializes the length encoder -static void -length_encoder_reset(lzma_length_encoder *lencoder, - const uint32_t num_pos_states, const uint32_t table_size) -{ - // NLength::CPriceTableEncoder::SetTableSize() - lencoder->table_size = table_size; - - // NLength::CEncoder::Init() - bit_reset(lencoder->choice); - bit_reset(lencoder->choice2); - - for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { - bittree_reset(lencoder->low[pos_state], LEN_LOW_BITS); - bittree_reset(lencoder->mid[pos_state], LEN_MID_BITS); - } - - bittree_reset(lencoder->high, LEN_HIGH_BITS); - - // NLength::CPriceTableEncoder::UpdateTables() - for (size_t pos_state = 0; pos_state < num_pos_states; ++pos_state) - lencoder->counters[pos_state] = 1; - - return; -} - - -static void -lzma_lzma_encoder_end(lzma_coder *coder, lzma_allocator *allocator) -{ - lzma_lz_encoder_end(&coder->lz, allocator); - lzma_free(coder, allocator); - return; -} - - -extern lzma_ret -lzma_lzma_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, - const lzma_filter_info *filters) -{ - if (next->coder == NULL) { - next->coder = lzma_alloc(sizeof(lzma_coder), allocator); - if (next->coder == NULL) - return LZMA_MEM_ERROR; - - next->coder->next = LZMA_NEXT_CODER_INIT; - next->coder->lz = LZMA_LZ_ENCODER_INIT; - } - - // Validate options that aren't validated elsewhere. - const lzma_options_lzma *options = filters[0].options; - if (options->pos_bits > LZMA_POS_BITS_MAX - || options->fast_bytes < LZMA_FAST_BYTES_MIN - || options->fast_bytes > LZMA_FAST_BYTES_MAX) { - lzma_lzma_encoder_end(next->coder, allocator); - return LZMA_HEADER_ERROR; - } - - // Set compression mode. - switch (options->mode) { - case LZMA_MODE_FAST: - next->coder->best_compression = false; - break; - - case LZMA_MODE_BEST: - next->coder->best_compression = true; - break; - - default: - lzma_lzma_encoder_end(next->coder, allocator); - return LZMA_HEADER_ERROR; - } - - // Initialize literal coder. - { - const lzma_ret ret = lzma_literal_init( - &next->coder->literal_coder, - options->literal_context_bits, - options->literal_pos_bits); - if (ret != LZMA_OK) - return ret; - } - - // Initialize LZ encoder. - { - const lzma_ret ret = lzma_lz_encoder_reset( - &next->coder->lz, allocator, &lzma_lzma_encode, - options->dictionary_size, OPTS, - options->fast_bytes, MATCH_MAX_LEN + 1 + OPTS, - options->match_finder, - options->match_finder_cycles, - options->preset_dictionary, - options->preset_dictionary_size); - if (ret != LZMA_OK) { - lzma_lzma_encoder_end(next->coder, allocator); - return ret; - } - } - - // Set dist_table_size. - { - // Round the dictionary size up to next 2^n. - uint32_t log_size; - for (log_size = 0; (UINT32_C(1) << log_size) - < options->dictionary_size; ++log_size) ; - - next->coder->dist_table_size = log_size * 2; - } - - // Misc FIXME desc - next->coder->align_price_count = UINT32_MAX; - next->coder->match_price_count = UINT32_MAX; - next->coder->dictionary_size = options->dictionary_size; - next->coder->pos_mask = (1U << options->pos_bits) - 1; - next->coder->fast_bytes = options->fast_bytes; - - // Range coder - rc_reset(&next->coder->rc); - - // State - next->coder->state = 0; - next->coder->previous_byte = 0; - for (size_t i = 0; i < REP_DISTANCES; ++i) - next->coder->reps[i] = 0; - - // Bit encoders - for (size_t i = 0; i < STATES; ++i) { - for (size_t j = 0; j <= next->coder->pos_mask; ++j) { - bit_reset(next->coder->is_match[i][j]); - bit_reset(next->coder->is_rep0_long[i][j]); - } - - bit_reset(next->coder->is_rep[i]); - bit_reset(next->coder->is_rep0[i]); - bit_reset(next->coder->is_rep1[i]); - bit_reset(next->coder->is_rep2[i]); - } - - for (size_t i = 0; i < FULL_DISTANCES - END_POS_MODEL_INDEX; ++i) - bit_reset(next->coder->pos_encoders[i]); - - // Bit tree encoders - for (size_t i = 0; i < LEN_TO_POS_STATES; ++i) - bittree_reset(next->coder->pos_slot_encoder[i], POS_SLOT_BITS); - - bittree_reset(next->coder->pos_align_encoder, ALIGN_BITS); - - // Length encoders - length_encoder_reset(&next->coder->match_len_encoder, - 1U << options->pos_bits, - options->fast_bytes + 1 - MATCH_MIN_LEN); - - length_encoder_reset(&next->coder->rep_len_encoder, - 1U << options->pos_bits, - next->coder->fast_bytes + 1 - MATCH_MIN_LEN); - - next->coder->prev_len_encoder = NULL; - - // Misc - next->coder->longest_match_was_found = false; - next->coder->optimum_end_index = 0; - next->coder->optimum_current_index = 0; - next->coder->additional_offset = 0; - - next->coder->now_pos = 0; - next->coder->is_initialized = false; - next->coder->is_flushed = false, - next->coder->write_eopm = true; - - // Initialize the next decoder in the chain, if any. - { - const lzma_ret ret = lzma_next_filter_init(&next->coder->next, - allocator, filters + 1); - if (ret != LZMA_OK) { - lzma_lzma_encoder_end(next->coder, allocator); - return ret; - } - } - - // Initialization successful. Set the function pointers. - next->code = &lzma_lz_encode; - next->end = &lzma_lzma_encoder_end; - - return LZMA_OK; -} - - -extern bool -lzma_lzma_encode_properties(const lzma_options_lzma *options, uint8_t *byte) -{ - if (options->literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX - || options->literal_pos_bits - > LZMA_LITERAL_POS_BITS_MAX - || options->pos_bits > LZMA_POS_BITS_MAX - || options->literal_context_bits - + options->literal_pos_bits - > LZMA_LITERAL_BITS_MAX) - return true; - - *byte = (options->pos_bits * 5 + options->literal_pos_bits) * 9 - + options->literal_context_bits; - assert(*byte <= (4 * 5 + 4) * 9 + 8); - - return false; -} diff --git a/src/liblzma/lzma/lzma_encoder_optimum_fast.c b/src/liblzma/lzma/lzma_encoder_optimum_fast.c new file mode 100644 index 00000000..9da7e79e --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -0,0 +1,193 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_optimum_fast.c +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_encoder_private.h" + + +#define change_pair(small_dist, big_dist) \ + (((big_dist) >> 7) > (small_dist)) + + +static inline void +literal(const lzma_coder *restrict coder, const uint8_t *restrict buf, + uint32_t *restrict back_res, uint32_t *restrict len_res) +{ + // Try short rep0 instead of always coding it as a literal. + *back_res = *buf == *(buf - coder->reps[0] - 1) ? 0 : UINT32_MAX; + *len_res = 1; + return; +} + + +extern void +lzma_lzma_optimum_fast(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res) +{ + const uint32_t fast_bytes = mf->find_len_max; + + uint32_t len_main; + uint32_t matches_count; + if (mf->read_ahead == 0) { + len_main = mf_find(mf, &matches_count, coder->matches); + } else { + assert(mf->read_ahead == 1); + len_main = coder->longest_match_length; + matches_count = coder->matches_count; + } + + const uint8_t *buf = mf_ptr(mf) - 1; + const uint32_t buf_avail = MIN(mf_avail(mf) + 1, MATCH_LEN_MAX); + + if (buf_avail < 2) { + // There's not enough input left to encode a match. + literal(coder, buf, back_res, len_res); + return; + } + + // Look for repeated matches; scan the previous four match distances + uint32_t rep_len = 0; + uint32_t rep_index = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + // Pointer to the beginning of the match candidate + const uint8_t *const buf_back = buf - coder->reps[i] - 1; + + // If the first two bytes (2 == MATCH_LEN_MIN) do not match, + // this rep is not useful. + if (not_equal_16(buf, buf_back)) + continue; + + // The first two bytes matched. + // Calculate the length of the match. + uint32_t len; + for (len = 2; len < buf_avail + && buf[len] == buf_back[len]; ++len) ; + + // If we have found a repeated match that is at least + // fast_bytes long, return it immediatelly. + if (len >= fast_bytes) { + *back_res = i; + *len_res = len; + mf_skip(mf, len - 1); + return; + } + + if (len > rep_len) { + rep_index = i; + rep_len = len; + } + } + + // We didn't find a long enough repeated match. Encode it as a normal + // match if the match length is at least fast_bytes. + if (len_main >= fast_bytes) { + *back_res = coder->matches[matches_count - 1].dist + + REP_DISTANCES; + *len_res = len_main; + mf_skip(mf, len_main - 1); + return; + } + + uint32_t back_main = 0; + if (len_main >= 2) { + back_main = coder->matches[matches_count - 1].dist; + + while (matches_count > 1 && len_main == + coder->matches[matches_count - 2].len + 1) { + if (!change_pair(coder->matches[ + matches_count - 2].dist, + back_main)) + break; + + --matches_count; + len_main = coder->matches[matches_count - 1].len; + back_main = coder->matches[matches_count - 1].dist; + } + + if (len_main == 2 && back_main >= 0x80) + len_main = 1; + } + + if (rep_len >= 2) { + if (rep_len + 1 >= len_main + || (rep_len + 2 >= len_main + && back_main > (UINT32_C(1) << 9)) + || (rep_len + 3 >= len_main + && back_main > (UINT32_C(1) << 15))) { + *back_res = rep_index; + *len_res = rep_len; + mf_skip(mf, rep_len - 1); + return; + } + } + + if (len_main < 2 || buf_avail <= 2) { + literal(coder, buf, back_res, len_res); + return; + } + + // Get the matches for the next byte. If we find a better match, + // the current byte is encoded as a literal. + coder->longest_match_length = mf_find(mf, + &coder->matches_count, coder->matches); + + if (coder->longest_match_length >= 2) { + const uint32_t new_dist = coder->matches[ + coder->matches_count - 1].dist; + + if ((coder->longest_match_length >= len_main + && new_dist < back_main) + || (coder->longest_match_length == len_main + 1 + && !change_pair(back_main, new_dist)) + || (coder->longest_match_length > len_main + 1) + || (coder->longest_match_length + 1 >= len_main + && len_main >= 3 + && change_pair(new_dist, back_main))) { + literal(coder, buf, back_res, len_res); + return; + } + } + + // In contrast to LZMA SDK, dictionary could not have been moved + // between mf_find() calls, thus it is safe to just increment + // the old buf pointer instead of recalculating it with mf_ptr(). + ++buf; + + const uint32_t limit = len_main - 1; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + const uint8_t *const buf_back = buf - coder->reps[i] - 1; + + if (not_equal_16(buf, buf_back)) + continue; + + uint32_t len; + for (len = 2; len < limit + && buf[len] == buf_back[len]; ++len) ; + + if (len >= limit) { + literal(coder, buf - 1, back_res, len_res); + return; + } + } + + *back_res = back_main + REP_DISTANCES; + *len_res = len_main; + mf_skip(mf, len_main - 2); + return; +} diff --git a/src/liblzma/lzma/lzma_encoder_optimum_normal.c b/src/liblzma/lzma/lzma_encoder_optimum_normal.c new file mode 100644 index 00000000..f0dd92c9 --- /dev/null +++ b/src/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -0,0 +1,875 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file lzma_encoder_optimum_normal.c +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "lzma_encoder_private.h" +#include "fastpos.h" + + +//////////// +// Prices // +//////////// + +static uint32_t +get_literal_price(const lzma_coder *const coder, const uint32_t pos, + const uint32_t prev_byte, const bool match_mode, + uint32_t match_byte, uint32_t symbol) +{ + const probability *const subcoder = literal_subcoder(coder->literal, + coder->literal_context_bits, coder->literal_pos_mask, + pos, prev_byte); + + uint32_t price = 0; + + if (!match_mode) { + price = rc_bittree_price(subcoder, 8, symbol); + } else { + uint32_t offset = 0x100; + symbol += UINT32_C(1) << 8; + + do { + match_byte <<= 1; + + const uint32_t match_bit = match_byte & offset; + const uint32_t subcoder_index + = offset + match_bit + (symbol >> 8); + const uint32_t bit = (symbol >> 7) & 1; + price += rc_bit_price(subcoder[subcoder_index], bit); + + symbol <<= 1; + offset &= ~(match_byte ^ symbol); + + } while (symbol < (UINT32_C(1) << 16)); + } + + return price; +} + + +static inline uint32_t +get_len_price(const lzma_length_encoder *const lencoder, + const uint32_t len, const uint32_t pos_state) +{ + // NOTE: Unlike the other price tables, length prices are updated + // in lzma_encoder.c + return lencoder->prices[pos_state][len - MATCH_LEN_MIN]; +} + + +static inline uint32_t +get_short_rep_price(const lzma_coder *const coder, + const lzma_lzma_state state, const uint32_t pos_state) +{ + return rc_bit_0_price(coder->is_rep0[state]) + + rc_bit_0_price(coder->is_rep0_long[state][pos_state]); +} + + +static inline uint32_t +get_pure_rep_price(const lzma_coder *const coder, const uint32_t rep_index, + const lzma_lzma_state state, uint32_t pos_state) +{ + uint32_t price; + + if (rep_index == 0) { + price = rc_bit_0_price(coder->is_rep0[state]); + price += rc_bit_1_price(coder->is_rep0_long[state][pos_state]); + } else { + price = rc_bit_1_price(coder->is_rep0[state]); + + if (rep_index == 1) { + price += rc_bit_0_price(coder->is_rep1[state]); + } else { + price += rc_bit_1_price(coder->is_rep1[state]); + price += rc_bit_price(coder->is_rep2[state], + rep_index - 2); + } + } + + return price; +} + + +static inline uint32_t +get_rep_price(const lzma_coder *const coder, const uint32_t rep_index, + const uint32_t len, const lzma_lzma_state state, + const uint32_t pos_state) +{ + return get_len_price(&coder->rep_len_encoder, len, pos_state) + + get_pure_rep_price(coder, rep_index, state, pos_state); +} + + +static inline uint32_t +get_pos_len_price(const lzma_coder *const coder, const uint32_t pos, + const uint32_t len, const uint32_t pos_state) +{ + const uint32_t len_to_pos_state = get_len_to_pos_state(len); + uint32_t price; + + if (pos < FULL_DISTANCES) { + price = coder->distances_prices[len_to_pos_state][pos]; + } else { + const uint32_t pos_slot = get_pos_slot_2(pos); + price = coder->pos_slot_prices[len_to_pos_state][pos_slot] + + coder->align_prices[pos & ALIGN_MASK]; + } + + price += get_len_price(&coder->match_len_encoder, len, pos_state); + + return price; +} + + +static void +fill_distances_prices(lzma_coder *coder) +{ + for (uint32_t len_to_pos_state = 0; + len_to_pos_state < LEN_TO_POS_STATES; + ++len_to_pos_state) { + + uint32_t *const pos_slot_prices + = coder->pos_slot_prices[len_to_pos_state]; + + // Price to encode the pos_slot. + for (uint32_t pos_slot = 0; + pos_slot < coder->dist_table_size; ++pos_slot) + pos_slot_prices[pos_slot] = rc_bittree_price( + coder->pos_slot[len_to_pos_state], + POS_SLOT_BITS, pos_slot); + + // For matches with distance >= FULL_DISTANCES, add the price + // of the direct bits part of the match distance. (Align bits + // are handled by fill_align_prices()). + for (uint32_t pos_slot = END_POS_MODEL_INDEX; + pos_slot < coder->dist_table_size; ++pos_slot) + pos_slot_prices[pos_slot] += rc_direct_price( + ((pos_slot >> 1) - 1) - ALIGN_BITS); + + // Distances in the range [0, 3] are fully encoded with + // pos_slot, so they are used for coder->distances_prices + // as is. + for (uint32_t i = 0; i < START_POS_MODEL_INDEX; ++i) + coder->distances_prices[len_to_pos_state][i] + = pos_slot_prices[i]; + } + + // Distances in the range [4, 127] depend on pos_slot and pos_special. + // We do this in a loop separate from the above loop to avoid + // redundant calls to get_pos_slot(). + for (uint32_t i = START_POS_MODEL_INDEX; i < FULL_DISTANCES; ++i) { + const uint32_t pos_slot = get_pos_slot(i); + const uint32_t footer_bits = ((pos_slot >> 1) - 1); + const uint32_t base = (2 | (pos_slot & 1)) << footer_bits; + const uint32_t price = rc_bittree_reverse_price( + coder->pos_special + base - pos_slot - 1, + footer_bits, i - base); + + for (uint32_t len_to_pos_state = 0; + len_to_pos_state < LEN_TO_POS_STATES; + ++len_to_pos_state) + coder->distances_prices[len_to_pos_state][i] + = price + coder->pos_slot_prices[ + len_to_pos_state][pos_slot]; + } + + coder->match_price_count = 0; + return; +} + + +static void +fill_align_prices(lzma_coder *coder) +{ + for (uint32_t i = 0; i < ALIGN_TABLE_SIZE; ++i) + coder->align_prices[i] = rc_bittree_reverse_price( + coder->pos_align, ALIGN_BITS, i); + + coder->align_price_count = 0; + return; +} + + +///////////// +// Optimal // +///////////// + +static inline void +make_literal(lzma_optimal *optimal) +{ + optimal->back_prev = UINT32_MAX; + optimal->prev_1_is_literal = false; +} + + +static inline void +make_short_rep(lzma_optimal *optimal) +{ + optimal->back_prev = 0; + optimal->prev_1_is_literal = false; +} + + +#define is_short_rep(optimal) \ + ((optimal).back_prev == 0) + + +static void +backward(lzma_coder *restrict coder, uint32_t *restrict len_res, + uint32_t *restrict back_res, uint32_t cur) +{ + coder->opts_end_index = cur; + + uint32_t pos_mem = coder->opts[cur].pos_prev; + uint32_t back_mem = coder->opts[cur].back_prev; + + do { + if (coder->opts[cur].prev_1_is_literal) { + make_literal(&coder->opts[pos_mem]); + coder->opts[pos_mem].pos_prev = pos_mem - 1; + + if (coder->opts[cur].prev_2) { + coder->opts[pos_mem - 1].prev_1_is_literal + = false; + coder->opts[pos_mem - 1].pos_prev + = coder->opts[cur].pos_prev_2; + coder->opts[pos_mem - 1].back_prev + = coder->opts[cur].back_prev_2; + } + } + + const uint32_t pos_prev = pos_mem; + const uint32_t back_cur = back_mem; + + back_mem = coder->opts[pos_prev].back_prev; + pos_mem = coder->opts[pos_prev].pos_prev; + + coder->opts[pos_prev].back_prev = back_cur; + coder->opts[pos_prev].pos_prev = cur; + cur = pos_prev; + + } while (cur != 0); + + coder->opts_current_index = coder->opts[0].pos_prev; + *len_res = coder->opts[0].pos_prev; + *back_res = coder->opts[0].back_prev; + + return; +} + + +////////// +// Main // +////////// + +static inline uint32_t +helper1(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, + uint32_t position) +{ + const uint32_t fast_bytes = mf->find_len_max; + + uint32_t len_main; + uint32_t matches_count; + + if (mf->read_ahead == 0) { + len_main = mf_find(mf, &matches_count, coder->matches); + } else { + assert(mf->read_ahead == 1); + len_main = coder->longest_match_length; + matches_count = coder->matches_count; + } + + const uint32_t buf_avail = MIN(mf_avail(mf) + 1, MATCH_LEN_MAX); + if (buf_avail < 2) { + *back_res = UINT32_MAX; + *len_res = 1; + return UINT32_MAX; + } + + const uint8_t *const buf = mf_ptr(mf) - 1; + + uint32_t rep_lens[REP_DISTANCES]; + uint32_t rep_max_index = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + const uint8_t *const buf_back = buf - coder->reps[i] - 1; + + if (not_equal_16(buf, buf_back)) { + rep_lens[i] = 0; + continue; + } + + uint32_t len_test; + for (len_test = 2; len_test < buf_avail + && buf[len_test] == buf_back[len_test]; + ++len_test) ; + + rep_lens[i] = len_test; + if (len_test > rep_lens[rep_max_index]) + rep_max_index = i; + } + + if (rep_lens[rep_max_index] >= fast_bytes) { + *back_res = rep_max_index; + *len_res = rep_lens[rep_max_index]; + mf_skip(mf, *len_res - 1); + return UINT32_MAX; + } + + + if (len_main >= fast_bytes) { + *back_res = coder->matches[matches_count - 1].dist + + REP_DISTANCES; + *len_res = len_main; + mf_skip(mf, len_main - 1); + return UINT32_MAX; + } + + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - coder->reps[0] - 1); + + if (len_main < 2 && current_byte != match_byte + && rep_lens[rep_max_index] < 2) { + *back_res = UINT32_MAX; + *len_res = 1; + return UINT32_MAX; + } + + coder->opts[0].state = coder->state; + + const uint32_t pos_state = position & coder->pos_mask; + + coder->opts[1].price = rc_bit_0_price( + coder->is_match[coder->state][pos_state]) + + get_literal_price(coder, position, buf[-1], + !is_literal_state(coder->state), + match_byte, current_byte); + + make_literal(&coder->opts[1]); + + const uint32_t match_price = rc_bit_1_price( + coder->is_match[coder->state][pos_state]); + const uint32_t rep_match_price = match_price + + rc_bit_1_price(coder->is_rep[coder->state]); + + if (match_byte == current_byte) { + const uint32_t short_rep_price = rep_match_price + + get_short_rep_price( + coder, coder->state, pos_state); + + if (short_rep_price < coder->opts[1].price) { + coder->opts[1].price = short_rep_price; + make_short_rep(&coder->opts[1]); + } + } + + const uint32_t len_end = MAX(len_main, rep_lens[rep_max_index]); + + if (len_end < 2) { + *back_res = coder->opts[1].back_prev; + *len_res = 1; + return UINT32_MAX; + } + + coder->opts[1].pos_prev = 0; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) + coder->opts[0].backs[i] = coder->reps[i]; + + uint32_t len = len_end; + do { + coder->opts[len].price = RC_INFINITY_PRICE; + } while (--len >= 2); + + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) { + uint32_t rep_len = rep_lens[i]; + if (rep_len < 2) + continue; + + const uint32_t price = rep_match_price + get_pure_rep_price( + coder, i, coder->state, pos_state); + + do { + const uint32_t cur_and_len_price = price + + get_len_price( + &coder->rep_len_encoder, + rep_len, pos_state); + + if (cur_and_len_price < coder->opts[rep_len].price) { + coder->opts[rep_len].price = cur_and_len_price; + coder->opts[rep_len].pos_prev = 0; + coder->opts[rep_len].back_prev = i; + coder->opts[rep_len].prev_1_is_literal = false; + } + } while (--rep_len >= 2); + } + + + const uint32_t normal_match_price = match_price + + rc_bit_0_price(coder->is_rep[coder->state]); + + len = rep_lens[0] >= 2 ? rep_lens[0] + 1 : 2; + if (len <= len_main) { + uint32_t i = 0; + while (len > coder->matches[i].len) + ++i; + + for(; ; ++len) { + const uint32_t dist = coder->matches[i].dist; + const uint32_t cur_and_len_price = normal_match_price + + get_pos_len_price(coder, + dist, len, pos_state); + + if (cur_and_len_price < coder->opts[len].price) { + coder->opts[len].price = cur_and_len_price; + coder->opts[len].pos_prev = 0; + coder->opts[len].back_prev + = dist + REP_DISTANCES; + coder->opts[len].prev_1_is_literal = false; + } + + if (len == coder->matches[i].len) + if (++i == matches_count) + break; + } + } + + return len_end; +} + + +static inline uint32_t +helper2(lzma_coder *coder, uint32_t *reps, const uint8_t *buf, + uint32_t len_end, uint32_t position, const uint32_t cur, + const uint32_t fast_bytes, const uint32_t buf_avail_full) +{ + uint32_t matches_count = coder->matches_count; + uint32_t new_len = coder->longest_match_length; + uint32_t pos_prev = coder->opts[cur].pos_prev; + uint32_t state; + + if (coder->opts[cur].prev_1_is_literal) { + --pos_prev; + + if (coder->opts[cur].prev_2) { + state = coder->opts[coder->opts[cur].pos_prev_2].state; + + if (coder->opts[cur].back_prev_2 < REP_DISTANCES) + update_long_rep(state); + else + update_match(state); + + } else { + state = coder->opts[pos_prev].state; + } + + update_literal(state); + + } else { + state = coder->opts[pos_prev].state; + } + + if (pos_prev == cur - 1) { + if (is_short_rep(coder->opts[cur])) + update_short_rep(state); + else + update_literal(state); + } else { + uint32_t pos; + if (coder->opts[cur].prev_1_is_literal + && coder->opts[cur].prev_2) { + pos_prev = coder->opts[cur].pos_prev_2; + pos = coder->opts[cur].back_prev_2; + update_long_rep(state); + } else { + pos = coder->opts[cur].back_prev; + if (pos < REP_DISTANCES) + update_long_rep(state); + else + update_match(state); + } + + if (pos < REP_DISTANCES) { + reps[0] = coder->opts[pos_prev].backs[pos]; + + uint32_t i; + for (i = 1; i <= pos; ++i) + reps[i] = coder->opts[pos_prev].backs[i - 1]; + + for (; i < REP_DISTANCES; ++i) + reps[i] = coder->opts[pos_prev].backs[i]; + + } else { + reps[0] = pos - REP_DISTANCES; + + for (uint32_t i = 1; i < REP_DISTANCES; ++i) + reps[i] = coder->opts[pos_prev].backs[i - 1]; + } + } + + coder->opts[cur].state = state; + + for (uint32_t i = 0; i < REP_DISTANCES; ++i) + coder->opts[cur].backs[i] = reps[i]; + + const uint32_t cur_price = coder->opts[cur].price; + + const uint8_t current_byte = *buf; + const uint8_t match_byte = *(buf - reps[0] - 1); + + const uint32_t pos_state = position & coder->pos_mask; + + const uint32_t cur_and_1_price = cur_price + + rc_bit_0_price(coder->is_match[state][pos_state]) + + get_literal_price(coder, position, buf[-1], + !is_literal_state(state), match_byte, current_byte); + + bool next_is_literal = false; + + if (cur_and_1_price < coder->opts[cur + 1].price) { + coder->opts[cur + 1].price = cur_and_1_price; + coder->opts[cur + 1].pos_prev = cur; + make_literal(&coder->opts[cur + 1]); + next_is_literal = true; + } + + const uint32_t match_price = cur_price + + rc_bit_1_price(coder->is_match[state][pos_state]); + const uint32_t rep_match_price = match_price + + rc_bit_1_price(coder->is_rep[state]); + + if (match_byte == current_byte + && !(coder->opts[cur + 1].pos_prev < cur + && coder->opts[cur + 1].back_prev == 0)) { + + const uint32_t short_rep_price = rep_match_price + + get_short_rep_price(coder, state, pos_state); + + if (short_rep_price <= coder->opts[cur + 1].price) { + coder->opts[cur + 1].price = short_rep_price; + coder->opts[cur + 1].pos_prev = cur; + make_short_rep(&coder->opts[cur + 1]); + next_is_literal = true; + } + } + + if (buf_avail_full < 2) + return len_end; + + const uint32_t buf_avail = MIN(buf_avail_full, fast_bytes); + + if (!next_is_literal && match_byte != current_byte) { // speed optimization + // try literal + rep0 + const uint8_t *const buf_back = buf - reps[0] - 1; + const uint32_t limit = MIN(buf_avail_full, fast_bytes + 1); + + uint32_t len_test = 1; + while (len_test < limit && buf[len_test] == buf_back[len_test]) + ++len_test; + + --len_test; + + if (len_test >= 2) { + uint32_t state_2 = state; + update_literal(state_2); + + const uint32_t pos_state_next = (position + 1) & coder->pos_mask; + const uint32_t next_rep_match_price = cur_and_1_price + + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + + rc_bit_1_price(coder->is_rep[state_2]); + + //for (; len_test >= 2; --len_test) { + const uint32_t offset = cur + 1 + len_test; + + while (len_end < offset) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + const uint32_t cur_and_len_price = next_rep_match_price + + get_rep_price(coder, 0, len_test, + state_2, pos_state_next); + + if (cur_and_len_price < coder->opts[offset].price) { + coder->opts[offset].price = cur_and_len_price; + coder->opts[offset].pos_prev = cur + 1; + coder->opts[offset].back_prev = 0; + coder->opts[offset].prev_1_is_literal = true; + coder->opts[offset].prev_2 = false; + } + //} + } + } + + + uint32_t start_len = 2; // speed optimization + + for (uint32_t rep_index = 0; rep_index < REP_DISTANCES; ++rep_index) { + const uint8_t *const buf_back = buf - reps[rep_index] - 1; + if (not_equal_16(buf, buf_back)) + continue; + + uint32_t len_test; + for (len_test = 2; len_test < buf_avail + && buf[len_test] == buf_back[len_test]; + ++len_test) ; + + while (len_end < cur + len_test) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + const uint32_t len_test_temp = len_test; + const uint32_t price = rep_match_price + get_pure_rep_price( + coder, rep_index, state, pos_state); + + do { + const uint32_t cur_and_len_price = price + + get_len_price(&coder->rep_len_encoder, + len_test, pos_state); + + if (cur_and_len_price < coder->opts[cur + len_test].price) { + coder->opts[cur + len_test].price = cur_and_len_price; + coder->opts[cur + len_test].pos_prev = cur; + coder->opts[cur + len_test].back_prev = rep_index; + coder->opts[cur + len_test].prev_1_is_literal = false; + } + } while (--len_test >= 2); + + len_test = len_test_temp; + + if (rep_index == 0) + start_len = len_test + 1; + + + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = MIN(buf_avail_full, + len_test_2 + fast_bytes); + for (; len_test_2 < limit + && buf[len_test_2] == buf_back[len_test_2]; + ++len_test_2) ; + + len_test_2 -= len_test + 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_long_rep(state_2); + + uint32_t pos_state_next = (position + len_test) & coder->pos_mask; + + const uint32_t cur_and_len_literal_price = price + + get_len_price(&coder->rep_len_encoder, + len_test, pos_state) + + rc_bit_0_price(coder->is_match[state_2][pos_state_next]) + + get_literal_price(coder, position + len_test, + buf[len_test - 1], true, + buf_back[len_test], buf[len_test]); + + update_literal(state_2); + + pos_state_next = (position + len_test + 1) & coder->pos_mask; + + const uint32_t next_rep_match_price = cur_and_len_literal_price + + rc_bit_1_price(coder->is_match[state_2][pos_state_next]) + + rc_bit_1_price(coder->is_rep[state_2]); + + //for(; len_test_2 >= 2; len_test_2--) { + const uint32_t offset = cur + len_test + 1 + len_test_2; + + while (len_end < offset) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + const uint32_t cur_and_len_price = next_rep_match_price + + get_rep_price(coder, 0, len_test_2, + state_2, pos_state_next); + + if (cur_and_len_price < coder->opts[offset].price) { + coder->opts[offset].price = cur_and_len_price; + coder->opts[offset].pos_prev = cur + len_test + 1; + coder->opts[offset].back_prev = 0; + coder->opts[offset].prev_1_is_literal = true; + coder->opts[offset].prev_2 = true; + coder->opts[offset].pos_prev_2 = cur; + coder->opts[offset].back_prev_2 = rep_index; + } + //} + } + } + + + //for (uint32_t len_test = 2; len_test <= new_len; ++len_test) + if (new_len > buf_avail) { + new_len = buf_avail; + + matches_count = 0; + while (new_len > coder->matches[matches_count].len) + ++matches_count; + + coder->matches[matches_count++].len = new_len; + } + + + if (new_len >= start_len) { + const uint32_t normal_match_price = match_price + + rc_bit_0_price(coder->is_rep[state]); + + while (len_end < cur + new_len) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + uint32_t i = 0; + while (start_len > coder->matches[i].len) + ++i; + + for (uint32_t len_test = start_len; ; ++len_test) { + const uint32_t cur_back = coder->matches[i].dist; + uint32_t cur_and_len_price = normal_match_price + + get_pos_len_price(coder, + cur_back, len_test, pos_state); + + if (cur_and_len_price < coder->opts[cur + len_test].price) { + coder->opts[cur + len_test].price = cur_and_len_price; + coder->opts[cur + len_test].pos_prev = cur; + coder->opts[cur + len_test].back_prev + = cur_back + REP_DISTANCES; + coder->opts[cur + len_test].prev_1_is_literal = false; + } + + if (len_test == coder->matches[i].len) { + // Try Match + Literal + Rep0 + const uint8_t *const buf_back = buf - cur_back - 1; + uint32_t len_test_2 = len_test + 1; + const uint32_t limit = MIN(buf_avail_full, + len_test_2 + fast_bytes); + + for (; len_test_2 < limit && + buf[len_test_2] == buf_back[len_test_2]; + ++len_test_2) ; + + len_test_2 -= len_test + 1; + + if (len_test_2 >= 2) { + uint32_t state_2 = state; + update_match(state_2); + uint32_t pos_state_next + = (position + len_test) & coder->pos_mask; + + const uint32_t cur_and_len_literal_price = cur_and_len_price + + rc_bit_0_price( + coder->is_match[state_2][pos_state_next]) + + get_literal_price(coder, + position + len_test, + buf[len_test - 1], + true, + buf_back[len_test], + buf[len_test]); + + update_literal(state_2); + pos_state_next = (pos_state_next + 1) & coder->pos_mask; + + const uint32_t next_rep_match_price + = cur_and_len_literal_price + + rc_bit_1_price( + coder->is_match[state_2][pos_state_next]) + + rc_bit_1_price(coder->is_rep[state_2]); + + // for(; len_test_2 >= 2; --len_test_2) { + const uint32_t offset = cur + len_test + 1 + len_test_2; + + while (len_end < offset) + coder->opts[++len_end].price = RC_INFINITY_PRICE; + + cur_and_len_price = next_rep_match_price + + get_rep_price(coder, 0, len_test_2, + state_2, pos_state_next); + + if (cur_and_len_price < coder->opts[offset].price) { + coder->opts[offset].price = cur_and_len_price; + coder->opts[offset].pos_prev = cur + len_test + 1; + coder->opts[offset].back_prev = 0; + coder->opts[offset].prev_1_is_literal = true; + coder->opts[offset].prev_2 = true; + coder->opts[offset].pos_prev_2 = cur; + coder->opts[offset].back_prev_2 + = cur_back + REP_DISTANCES; + } + //} + } + + if (++i == matches_count) + break; + } + } + } + + return len_end; +} + + +extern void +lzma_lzma_optimum_normal(lzma_coder *restrict coder, lzma_mf *restrict mf, + uint32_t *restrict back_res, uint32_t *restrict len_res, + uint32_t position) +{ + // If we have symbols pending, return the next pending symbol. + if (coder->opts_end_index != coder->opts_current_index) { + assert(mf->read_ahead > 0); + *len_res = coder->opts[coder->opts_current_index].pos_prev + - coder->opts_current_index; + *back_res = coder->opts[coder->opts_current_index].back_prev; + coder->opts_current_index = coder->opts[ + coder->opts_current_index].pos_prev; + return; + } + + // Update the price tables. In LZMA SDK <= 4.60 (and possibly later) + // this was done in both initialization function and in the main loop. + // In liblzma they were moved into this single place. + if (mf->read_ahead == 0) { + if (coder->match_price_count >= (1 << 7)) + fill_distances_prices(coder); + + if (coder->align_price_count >= ALIGN_TABLE_SIZE) + fill_align_prices(coder); + } + + // TODO: This needs quite a bit of cleaning still. But splitting + // the oroginal function to two pieces makes it at least a little + // more readable, since those two parts don't share many variables. + + uint32_t len_end = helper1(coder, mf, back_res, len_res, position); + if (len_end == UINT32_MAX) + return; + + uint32_t reps[REP_DISTANCES]; + memcpy(reps, coder->reps, sizeof(reps)); + + uint32_t cur; + for (cur = 1; cur < len_end; ++cur) { + assert(cur < OPTS); + + coder->longest_match_length = mf_find( + mf, &coder->matches_count, coder->matches); + + if (coder->longest_match_length >= mf->find_len_max) + break; + + len_end = helper2(coder, reps, mf_ptr(mf) - 1, len_end, + position + cur, cur, mf->find_len_max, + MIN(mf_avail(mf) + 1, OPTS - 1 - cur)); + } + + backward(coder, len_res, back_res, cur); + return; +} diff --git a/src/liblzma/lzma/lzma_encoder_presets.c b/src/liblzma/lzma/lzma_encoder_presets.c index 966c7c86..08f339e9 100644 --- a/src/liblzma/lzma/lzma_encoder_presets.c +++ b/src/liblzma/lzma/lzma_encoder_presets.c @@ -20,15 +20,47 @@ #include "common.h" +#define pow2(e) (UINT32_C(1) << (e)) + + LZMA_API const lzma_options_lzma lzma_preset_lzma[9] = { -// dictionary_size lc lp pb mode fb mf mfc -{ UINT32_C(1) << 16, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0 }, -{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0 }, -{ UINT32_C(1) << 19, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 20, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 64, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 21, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 22, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 23, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 128, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 24, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 }, -{ UINT32_C(1) << 25, 3, 0, 2, NULL, 0, LZMA_MODE_BEST, 273, LZMA_MF_BT4, 0 }, +// dict lc lp pb mode fb mf mfc +{ pow2(16), NULL, 0, 3, 0, 2, false, LZMA_MODE_FAST, 64, LZMA_MF_HC3, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(20), NULL, 0, 3, 0, 0, false, LZMA_MODE_FAST, 64, LZMA_MF_HC4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(19), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 64, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(20), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 64, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(21), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(22), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(23), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 128, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(24), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 273, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, +{ pow2(25), NULL, 0, 3, 0, 0, false, LZMA_MODE_NORMAL, 273, LZMA_MF_BT4, 0, 0, 0, 0, 0, NULL, NULL }, }; + + +/* +extern LZMA_API lzma_bool +lzma_preset_lzma(lzma_options_lzma *options, uint32_t level) +{ + *options = (lzma_options_lzma){ + + }; + + options->literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT + options->literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT; + options->pos_bits = LZMA_POS_BITS_DEFAULT; + options->preset_dictionary = NULL; + options->preset_dictionary_size = 0; + options->persistent = false; + + options->mode = level <= 2 ? LZMA_MODE_FAST : LZMA_MODE_NORMAL; + options->fast_bytes = level <= + + options->match_finder = level == 1 ? LZMA_MF_HC3 + : (level == 2 ? LZMA_MF_HC4 : LZMA_MF_BT4); + options->match_finder_cycles = 0; + + + + options->dictionary_size = +} +*/ diff --git a/src/liblzma/lzma/lzma_encoder_private.h b/src/liblzma/lzma/lzma_encoder_private.h index a16051f8..7533bc79 100644 --- a/src/liblzma/lzma/lzma_encoder_private.h +++ b/src/liblzma/lzma/lzma_encoder_private.h @@ -21,20 +21,27 @@ #ifndef LZMA_LZMA_ENCODER_PRIVATE_H #define LZMA_LZMA_ENCODER_PRIVATE_H -#include "lzma_encoder.h" -#include "lzma_common.h" #include "lz_encoder.h" #include "range_encoder.h" +#include "lzma_common.h" +#include "lzma_encoder.h" -#define move_pos(num) \ -do { \ - assert((int32_t)(num) >= 0); \ - if ((num) != 0) { \ - coder->additional_offset += num; \ - coder->lz.skip(&coder->lz, num); \ - } \ -} while (0) +// Macro to compare if the first two bytes in two buffers differ. This is +// needed in lzma_lzma_optimum_*() to test if the match is at least +// MATCH_LEN_MIN bytes. Unaligned access gives tiny gain so there's no +// reason to not use it when it is supported. +#ifdef HAVE_FAST_UNALIGNED_ACCESS +# define not_equal_16(a, b) \ + (*(const uint16_t *)(a) != *(const uint16_t *)(b)) +#else +# define not_equal_16(a, b) \ + ((a)[0] != (b)[0] || (a)[1] != (b)[1]) +#endif + + +// Optimal - Number of entries in the optimum array. +#define OPTS (1 << 12) typedef struct { @@ -54,7 +61,7 @@ typedef struct { typedef struct { lzma_lzma_state state; - bool prev_1_is_char; + bool prev_1_is_literal; bool prev_2; uint32_t pos_prev_2; @@ -70,132 +77,79 @@ typedef struct { struct lzma_coder_s { - // Next coder in the chain - lzma_next_coder next; - - // In window and match finder - lzma_lz_encoder lz; - - // Range encoder + /// Range encoder lzma_range_encoder rc; - // State + /// State lzma_lzma_state state; - uint8_t previous_byte; + + /// The four most recent match distances uint32_t reps[REP_DISTANCES]; - // Misc - uint32_t match_distances[MATCH_MAX_LEN * 2 + 2 + 1]; - uint32_t num_distance_pairs; - uint32_t additional_offset; - uint32_t now_pos; // Lowest 32 bits are enough here. - bool best_compression; ///< True when LZMA_MODE_BEST is used + /// Array of match candidates + lzma_match matches[MATCH_LEN_MAX + 1]; + + /// Number of match candidates in matches[] + uint32_t matches_count; + + /// Varibale to hold the length of the longest match between calls + /// to lzma_lzma_optimum_*(). + uint32_t longest_match_length; + + /// True if using getoptimumfast + bool fast_mode; + + /// True if the encoder has been initialized by encoding the first + /// byte as a literal. bool is_initialized; + + /// True if the range encoder has been flushed, but not all bytes + /// have been written to the output buffer yet. bool is_flushed; - bool write_eopm; - // Literal encoder - lzma_literal_coder literal_coder; + uint32_t pos_mask; ///< (1 << pos_bits) - 1 + uint32_t literal_context_bits; + uint32_t literal_pos_mask; - // Bit encoders + // These are the same as in lzma_decoder.c. See comments there. + probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; probability is_match[STATES][POS_STATES_MAX]; probability is_rep[STATES]; probability is_rep0[STATES]; probability is_rep1[STATES]; probability is_rep2[STATES]; probability is_rep0_long[STATES][POS_STATES_MAX]; - probability pos_encoders[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_slot[LEN_TO_POS_STATES][POS_SLOTS]; + probability pos_special[FULL_DISTANCES - END_POS_MODEL_INDEX]; + probability pos_align[ALIGN_TABLE_SIZE]; - // Bit Tree Encoders - probability pos_slot_encoder[LEN_TO_POS_STATES][1 << POS_SLOT_BITS]; - probability pos_align_encoder[1 << ALIGN_BITS]; - - // Length encoders + // These are the same as in lzma_decoder.c except that the encoders + // include also price tables. lzma_length_encoder match_len_encoder; lzma_length_encoder rep_len_encoder; - lzma_length_encoder *prev_len_encoder; - // Optimal - lzma_optimal optimum[OPTS]; - uint32_t optimum_end_index; - uint32_t optimum_current_index; - uint32_t longest_match_length; - bool longest_match_was_found; - - // Prices - uint32_t pos_slot_prices[LEN_TO_POS_STATES][DIST_TABLE_SIZE_MAX]; + // Price tables + uint32_t pos_slot_prices[LEN_TO_POS_STATES][POS_SLOTS]; uint32_t distances_prices[LEN_TO_POS_STATES][FULL_DISTANCES]; - uint32_t align_prices[ALIGN_TABLE_SIZE]; - uint32_t align_price_count; uint32_t dist_table_size; uint32_t match_price_count; - // LZMA specific settings - uint32_t dictionary_size; ///< Size in bytes - uint32_t fast_bytes; - uint32_t pos_state_bits; - uint32_t pos_mask; ///< (1 << pos_state_bits) - 1 + uint32_t align_prices[ALIGN_TABLE_SIZE]; + uint32_t align_price_count; + + // Optimal + uint32_t opts_end_index; + uint32_t opts_current_index; + lzma_optimal opts[OPTS]; }; -extern void lzma_length_encoder_update_table(lzma_length_encoder *lencoder, - const uint32_t pos_state); - -extern bool lzma_lzma_encode(lzma_coder *coder, uint8_t *restrict out, - size_t *restrict out_pos, size_t out_size); - -extern void lzma_get_optimum(lzma_coder *restrict coder, +extern void lzma_lzma_optimum_fast( + lzma_coder *restrict coder, lzma_mf *restrict mf, uint32_t *restrict back_res, uint32_t *restrict len_res); -extern void lzma_get_optimum_fast(lzma_coder *restrict coder, - uint32_t *restrict back_res, uint32_t *restrict len_res); - - -// NOTE: Don't add 'restrict'. -static inline void -lzma_read_match_distances(lzma_coder *coder, - uint32_t *len_res, uint32_t *num_distance_pairs) -{ - *len_res = 0; - - coder->lz.get_matches(&coder->lz, coder->match_distances); - - *num_distance_pairs = coder->match_distances[0]; - - if (*num_distance_pairs > 0) { - *len_res = coder->match_distances[*num_distance_pairs - 1]; - assert(*len_res <= MATCH_MAX_LEN); - - if (*len_res == coder->fast_bytes) { - uint32_t offset = *len_res - 1; - const uint32_t distance = coder->match_distances[ - *num_distance_pairs] + 1; - uint32_t limit = MATCH_MAX_LEN - *len_res; - - assert(offset + limit < coder->lz.keep_size_after); - assert(coder->lz.read_pos <= coder->lz.write_pos); - - // If we are close to end of the stream, we may need - // to limit the length of the match. - if (coder->lz.write_pos - coder->lz.read_pos - < offset + limit) - limit = coder->lz.write_pos - - (coder->lz.read_pos + offset); - - offset += coder->lz.read_pos; - uint32_t i = 0; - while (i < limit && coder->lz.buffer[offset + i] - == coder->lz.buffer[ - offset + i - distance]) - ++i; - - *len_res += i; - } - } - - ++coder->additional_offset; - - return; -} +extern void lzma_lzma_optimum_normal(lzma_coder *restrict coder, + lzma_mf *restrict mf, uint32_t *restrict back_res, + uint32_t *restrict len_res, uint32_t position); #endif diff --git a/src/liblzma/lzma/lzma_literal.c b/src/liblzma/lzma/lzma_literal.c deleted file mode 100644 index 3611a1f7..00000000 --- a/src/liblzma/lzma/lzma_literal.c +++ /dev/null @@ -1,51 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_literal.c -/// \brief Literal Coder -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#include "lzma_literal.h" - - -extern lzma_ret -lzma_literal_init(lzma_literal_coder *coder, - uint32_t literal_context_bits, uint32_t literal_pos_bits) -{ - // Verify that arguments are sane. - if (literal_context_bits > LZMA_LITERAL_CONTEXT_BITS_MAX - || literal_pos_bits > LZMA_LITERAL_POS_BITS_MAX) - return LZMA_HEADER_ERROR; - - // Calculate the number of states the literal coder must store. - const uint32_t states = literal_states( - literal_pos_bits, literal_context_bits); - - // Store the new settings. - coder->literal_context_bits = literal_context_bits; - coder->literal_pos_bits = literal_pos_bits; - - // Calculate also the literal_pos_mask. It's not changed - // anywhere else than here. - coder->literal_pos_mask = (1 << literal_pos_bits) - 1; - - // Reset the literal coder. - for (uint32_t i = 0; i < states; ++i) - for (uint32_t j = 0; j < LIT_SIZE; ++j) - bit_reset(coder->coders[i][j]); - - return LZMA_OK; -} diff --git a/src/liblzma/lzma/lzma_literal.h b/src/liblzma/lzma/lzma_literal.h deleted file mode 100644 index 208abd99..00000000 --- a/src/liblzma/lzma/lzma_literal.h +++ /dev/null @@ -1,71 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file lzma_literal.h -/// \brief Literal Coder -/// -/// This is used as is by both LZMA encoder and decoder. -// -// Copyright (C) 1999-2006 Igor Pavlov -// Copyright (C) 2007 Lasse Collin -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef LZMA_LITERAL_H -#define LZMA_LITERAL_H - -#include "common.h" - -// We need typedef of `probability'. -#include "range_common.h" - - -/// Each literal coder is divided in three sections: -/// - 0x001-0x0FF: Without match byte -/// - 0x101-0x1FF: With match byte; match bit is 0 -/// - 0x201-0x2FF: With match byte; match bit is 1 -#define LIT_SIZE 0x300 - -/// Calculate how many states are needed. Each state has -/// LIT_SIZE `probability' variables. -#define literal_states(literal_context_bits, literal_pos_bits) \ - (1U << ((literal_context_bits) + (literal_pos_bits))) - -/// Locate the literal coder for the next literal byte. The choice depends on -/// - the lowest literal_pos_bits bits of the position of the current -/// byte; and -/// - the highest literal_context_bits bits of the previous byte. -#define literal_get_subcoder(literal_coder, pos, prev_byte) \ - (literal_coder).coders[(((pos) & (literal_coder).literal_pos_mask) \ - << (literal_coder).literal_context_bits) \ - + ((prev_byte) >> (8 - (literal_coder).literal_context_bits))] - - -typedef struct { - uint32_t literal_context_bits; - uint32_t literal_pos_bits; - - /// literal_pos_mask is always (1 << literal_pos_bits) - 1. - uint32_t literal_pos_mask; - - /// There are (1 << (literal_pos_bits + literal_context_bits)) - /// literal coders. - probability coders[1 << LZMA_LITERAL_BITS_MAX][LIT_SIZE]; - -} lzma_literal_coder; - - -extern lzma_ret lzma_literal_init( - lzma_literal_coder *coder, - uint32_t literal_context_bits, uint32_t literal_pos_bits); - -#endif diff --git a/src/liblzma/rangecoder/Makefile.am b/src/liblzma/rangecoder/Makefile.am index 6e80f8d7..f6824292 100644 --- a/src/liblzma/rangecoder/Makefile.am +++ b/src/liblzma/rangecoder/Makefile.am @@ -12,7 +12,7 @@ ## Lesser General Public License for more details. ## -EXTRA_DIST = price_table_gen.c +EXTRA_DIST = price_tablegen.c noinst_LTLIBRARIES = librangecoder.la @@ -21,8 +21,10 @@ librangecoder_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/common -if COND_MAIN_ENCODER -librangecoder_la_SOURCES += range_encoder.h +if COND_ENCODER_LZMA +librangecoder_la_SOURCES += \ + range_encoder.h \ + price.h if COND_SMALL librangecoder_la_SOURCES += price_table_init.c else @@ -30,6 +32,6 @@ librangecoder_la_SOURCES += price_table.c endif endif -if COND_MAIN_DECODER +if COND_DECODER_LZMA librangecoder_la_SOURCES += range_decoder.h endif diff --git a/src/liblzma/rangecoder/price.h b/src/liblzma/rangecoder/price.h new file mode 100644 index 00000000..001f753d --- /dev/null +++ b/src/liblzma/rangecoder/price.h @@ -0,0 +1,111 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file price.h +/// \brief Probability price calculation +// +// Copyright (C) 1999-2008 Igor Pavlov +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_PRICE_H +#define LZMA_PRICE_H + + +#define RC_MOVE_REDUCING_BITS 4 +#define RC_BIT_PRICE_SHIFT_BITS 4 +#define RC_PRICE_TABLE_SIZE (RC_BIT_MODEL_TOTAL >> RC_MOVE_REDUCING_BITS) + +#define RC_INFINITY_PRICE (UINT32_C(1) << 30) + + +#if !defined(LZMA_RANGE_ENCODER_H) || defined(HAVE_SMALL) +/// Probability prices used by *_get_price() macros. This is initialized +/// by lzma_rc_init() and is not modified later. +extern uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE]; + +/// Initializes lzma_rc_prices[]. This needs to be called only once. +extern void lzma_rc_init(void); + +#else +// Not building a size optimized version, so we use a precomputed +// constant table. +extern const uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE]; + +#endif + + +static inline uint32_t +rc_bit_price(const probability prob, const uint32_t bit) +{ + return lzma_rc_prices[(prob ^ ((UINT32_C(0) - bit) + & (RC_BIT_MODEL_TOTAL - 1))) >> RC_MOVE_REDUCING_BITS]; +} + + +static inline uint32_t +rc_bit_0_price(const probability prob) +{ + return lzma_rc_prices[prob >> RC_MOVE_REDUCING_BITS]; +} + + +static inline uint32_t +rc_bit_1_price(const probability prob) +{ + return lzma_rc_prices[(prob ^ (RC_BIT_MODEL_TOTAL - 1)) + >> RC_MOVE_REDUCING_BITS]; +} + + +static inline uint32_t +rc_bittree_price(const probability *const probs, + const uint32_t bit_levels, uint32_t symbol) +{ + uint32_t price = 0; + symbol += UINT32_C(1) << bit_levels; + + do { + const uint32_t bit = symbol & 1; + symbol >>= 1; + price += rc_bit_price(probs[symbol], bit); + } while (symbol != 1); + + return price; +} + + +static inline uint32_t +rc_bittree_reverse_price(const probability *const probs, + uint32_t bit_levels, uint32_t symbol) +{ + uint32_t price = 0; + uint32_t model_index = 1; + + do { + const uint32_t bit = symbol & 1; + symbol >>= 1; + price += rc_bit_price(probs[model_index], bit); + model_index = (model_index << 1) + bit; + } while (--bit_levels != 0); + + return price; +} + + +static inline uint32_t +rc_direct_price(const uint32_t bits) +{ + return bits << RC_BIT_PRICE_SHIFT_BITS; +} + +#endif diff --git a/src/liblzma/rangecoder/price_table.c b/src/liblzma/rangecoder/price_table.c index d0b50fa6..539206b1 100644 --- a/src/liblzma/rangecoder/price_table.c +++ b/src/liblzma/rangecoder/price_table.c @@ -1,70 +1,22 @@ -/* This file has been automatically generated by price_table_gen.c. */ +/* This file has been automatically generated by price_tablegen.c. */ #include "range_encoder.h" -const uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS] = { - 0, 576, 512, 480, 448, 432, 416, 400, - 384, 376, 368, 360, 352, 344, 336, 328, - 320, 316, 312, 308, 304, 300, 296, 292, - 288, 284, 280, 276, 272, 268, 264, 260, - 256, 254, 252, 250, 248, 246, 244, 242, - 240, 238, 236, 234, 232, 230, 228, 226, - 224, 222, 220, 218, 216, 214, 212, 210, - 208, 206, 204, 202, 200, 198, 196, 194, - 192, 191, 190, 189, 188, 187, 186, 185, - 184, 183, 182, 181, 180, 179, 178, 177, - 176, 175, 174, 173, 172, 171, 170, 169, - 168, 167, 166, 165, 164, 163, 162, 161, - 160, 159, 158, 157, 156, 155, 154, 153, - 152, 151, 150, 149, 148, 147, 146, 145, - 144, 143, 142, 141, 140, 139, 138, 137, - 136, 135, 134, 133, 132, 131, 130, 129, - 128, 127, 127, 126, 126, 125, 125, 124, - 124, 123, 123, 122, 122, 121, 121, 120, - 120, 119, 119, 118, 118, 117, 117, 116, - 116, 115, 115, 114, 114, 113, 113, 112, - 112, 111, 111, 110, 110, 109, 109, 108, - 108, 107, 107, 106, 106, 105, 105, 104, - 104, 103, 103, 102, 102, 101, 101, 100, - 100, 99, 99, 98, 98, 97, 97, 96, - 96, 95, 95, 94, 94, 93, 93, 92, - 92, 91, 91, 90, 90, 89, 89, 88, - 88, 87, 87, 86, 86, 85, 85, 84, - 84, 83, 83, 82, 82, 81, 81, 80, - 80, 79, 79, 78, 78, 77, 77, 76, - 76, 75, 75, 74, 74, 73, 73, 72, - 72, 71, 71, 70, 70, 69, 69, 68, - 68, 67, 67, 66, 66, 65, 65, 64, - 64, 63, 63, 63, 63, 62, 62, 62, - 62, 61, 61, 61, 61, 60, 60, 60, - 60, 59, 59, 59, 59, 58, 58, 58, - 58, 57, 57, 57, 57, 56, 56, 56, - 56, 55, 55, 55, 55, 54, 54, 54, - 54, 53, 53, 53, 53, 52, 52, 52, - 52, 51, 51, 51, 51, 50, 50, 50, - 50, 49, 49, 49, 49, 48, 48, 48, - 48, 47, 47, 47, 47, 46, 46, 46, - 46, 45, 45, 45, 45, 44, 44, 44, - 44, 43, 43, 43, 43, 42, 42, 42, - 42, 41, 41, 41, 41, 40, 40, 40, - 40, 39, 39, 39, 39, 38, 38, 38, - 38, 37, 37, 37, 37, 36, 36, 36, - 36, 35, 35, 35, 35, 34, 34, 34, - 34, 33, 33, 33, 33, 32, 32, 32, - 32, 31, 31, 31, 31, 30, 30, 30, - 30, 29, 29, 29, 29, 28, 28, 28, - 28, 27, 27, 27, 27, 26, 26, 26, - 26, 25, 25, 25, 25, 24, 24, 24, - 24, 23, 23, 23, 23, 22, 22, 22, - 22, 21, 21, 21, 21, 20, 20, 20, - 20, 19, 19, 19, 19, 18, 18, 18, - 18, 17, 17, 17, 17, 16, 16, 16, - 16, 15, 15, 15, 15, 14, 14, 14, - 14, 13, 13, 13, 13, 12, 12, 12, - 12, 11, 11, 11, 11, 10, 10, 10, - 10, 9, 9, 9, 9, 8, 8, 8, - 8, 7, 7, 7, 7, 6, 6, 6, - 6, 5, 5, 5, 5, 4, 4, 4, - 4, 3, 3, 3, 3, 2, 2, 2, - 2, 1, 1, 1, 1, 0, 0, 0 +const uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE] = { + 128, 103, 91, 84, 78, 73, 69, 66, + 63, 61, 58, 56, 54, 52, 51, 49, + 48, 46, 45, 44, 43, 42, 41, 40, + 39, 38, 37, 36, 35, 34, 34, 33, + 32, 31, 31, 30, 29, 29, 28, 28, + 27, 26, 26, 25, 25, 24, 24, 23, + 23, 22, 22, 22, 21, 21, 20, 20, + 19, 19, 19, 18, 18, 17, 17, 17, + 16, 16, 16, 15, 15, 15, 14, 14, + 14, 13, 13, 13, 12, 12, 12, 11, + 11, 11, 11, 10, 10, 10, 10, 9, + 9, 9, 9, 8, 8, 8, 8, 7, + 7, 7, 7, 6, 6, 6, 6, 5, + 5, 5, 5, 5, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 2, 2, 2, + 2, 2, 2, 1, 1, 1, 1, 1 }; diff --git a/src/liblzma/rangecoder/price_table_init.c b/src/liblzma/rangecoder/price_table_init.c index 4714dfd6..9c7d799b 100644 --- a/src/liblzma/rangecoder/price_table_init.c +++ b/src/liblzma/rangecoder/price_table_init.c @@ -23,25 +23,32 @@ #endif -#define NUM_BITS (BIT_MODEL_TOTAL_BITS - MOVE_REDUCING_BITS) - - -uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; +uint32_t lzma_rc_prices[RC_PRICE_TABLE_SIZE]; extern void lzma_rc_init(void) { - // Initialize lzma_rc_prob_prices[]. - for (int i = NUM_BITS - 1; i >= 0; --i) { - const uint32_t start = 1 << (NUM_BITS - i - 1); - const uint32_t end = 1 << (NUM_BITS - i); + for (uint32_t i = (UINT32_C(1) << RC_MOVE_REDUCING_BITS) / 2; + i < RC_BIT_MODEL_TOTAL; + i += (UINT32_C(1) << RC_MOVE_REDUCING_BITS)) { + const uint32_t cycles_bits = RC_BIT_PRICE_SHIFT_BITS; + uint32_t w = i; + uint32_t bit_count = 0; - for (uint32_t j = start; j < end; ++j) { - lzma_rc_prob_prices[j] = (i << BIT_PRICE_SHIFT_BITS) - + (((end - j) << BIT_PRICE_SHIFT_BITS) - >> (NUM_BITS - i - 1)); + for (uint32_t j = 0; j < cycles_bits; ++j) { + w *= w; + bit_count <<= 1; + + while (w >= (UINT32_C(1) << 16)) { + w >>= 1; + ++bit_count; + } } + + lzma_rc_prices[i >> RC_MOVE_REDUCING_BITS] + = (RC_BIT_MODEL_TOTAL_BITS << cycles_bits) + - 15 - bit_count; } return; diff --git a/src/liblzma/rangecoder/price_table_gen.c b/src/liblzma/rangecoder/price_tablegen.c similarity index 75% rename from src/liblzma/rangecoder/price_table_gen.c rename to src/liblzma/rangecoder/price_tablegen.c index 946d8215..68513635 100644 --- a/src/liblzma/rangecoder/price_table_gen.c +++ b/src/liblzma/rangecoder/price_tablegen.c @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file price_table_gen.c +/// \file price_tablegen.c /// \brief Probability price table generator /// -/// Compiling: gcc -std=c99 -o price_table_gen price_table_gen.c +/// Compiling: gcc -std=c99 -o price_tablegen price_tablegen.c // // Copyright (C) 2007 Lasse Collin // @@ -19,10 +19,11 @@ // /////////////////////////////////////////////////////////////////////////////// -#include +#include #include #include #include "range_common.h" +#include "price.h" #include "price_table_init.c" @@ -32,18 +33,18 @@ main(void) lzma_rc_init(); printf("/* This file has been automatically generated by " - "price_table_gen.c. */\n\n" + "price_tablegen.c. */\n\n" "#include \"range_encoder.h\"\n\n" - "const uint32_t lzma_rc_prob_prices[" - "BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS] = {"); + "const uint32_t lzma_rc_prices[" + "RC_PRICE_TABLE_SIZE] = {"); - const size_t array_size = sizeof(lzma_rc_prob_prices) - / sizeof(lzma_rc_prob_prices[0]); + const size_t array_size = sizeof(lzma_rc_prices) + / sizeof(lzma_rc_prices[0]); for (size_t i = 0; i < array_size; ++i) { if (i % 8 == 0) printf("\n\t"); - printf("%4" PRIu32, lzma_rc_prob_prices[i]); + printf("%4" PRIu32, lzma_rc_prices[i]); if (i != array_size - 1) printf(","); diff --git a/src/liblzma/rangecoder/range_common.h b/src/liblzma/rangecoder/range_common.h index 7613621a..6e5b0994 100644 --- a/src/liblzma/rangecoder/range_common.h +++ b/src/liblzma/rangecoder/range_common.h @@ -30,15 +30,12 @@ // Constants // /////////////// -#define SHIFT_BITS 8 -#define TOP_BITS 24 -#define TOP_VALUE (UINT32_C(1) << TOP_BITS) -#define BIT_MODEL_TOTAL_BITS 11 -#define BIT_MODEL_TOTAL (UINT32_C(1) << BIT_MODEL_TOTAL_BITS) -#define MOVE_BITS 5 - -#define MOVE_REDUCING_BITS 2 -#define BIT_PRICE_SHIFT_BITS 6 +#define RC_SHIFT_BITS 8 +#define RC_TOP_BITS 24 +#define RC_TOP_VALUE (UINT32_C(1) << RC_TOP_BITS) +#define RC_BIT_MODEL_TOTAL_BITS 11 +#define RC_BIT_MODEL_TOTAL (UINT32_C(1) << RC_BIT_MODEL_TOTAL_BITS) +#define RC_MOVE_BITS 5 //////////// @@ -47,7 +44,7 @@ // Resets the probability so that both 0 and 1 have probability of 50 % #define bit_reset(prob) \ - prob = BIT_MODEL_TOTAL >> 1 + prob = RC_BIT_MODEL_TOTAL >> 1 // This does the same for a complete bit tree. // (A tree represented as an array.) diff --git a/src/liblzma/rangecoder/range_decoder.h b/src/liblzma/rangecoder/range_decoder.h index 62162448..ca2d392e 100644 --- a/src/liblzma/rangecoder/range_decoder.h +++ b/src/liblzma/rangecoder/range_decoder.h @@ -31,6 +31,7 @@ typedef struct { } lzma_range_decoder; +/// Reads the first five bytes to initialize the range decoder. static inline bool rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, size_t *restrict in_pos, size_t in_size) @@ -48,14 +49,22 @@ rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, } -/// Makes local copies of range decoder variables. -#define rc_to_local(range_decoder) \ +/// Makes local copies of range decoder and *in_pos variables. Doing this +/// improves speed significantly. The range decoder macros expect also +/// variables `in' and `in_size' to be defined. +#define rc_to_local(range_decoder, in_pos) \ lzma_range_decoder rc = range_decoder; \ + size_t rc_in_pos = (in_pos); \ uint32_t rc_bound + /// Stores the local copes back to the range decoder structure. -#define rc_from_local(range_decoder) \ - range_decoder = rc +#define rc_from_local(range_decoder, in_pos) \ +do { \ + range_decoder = rc; \ + in_pos = rc_in_pos; \ +} while (0) + /// Resets the range decoder structure. #define rc_reset(range_decoder) \ @@ -66,158 +75,112 @@ do { \ } while (0) -// All of the macros in this file expect the following variables being defined: -// - lzma_range_decoder range_decoder; -// - uint32_t rc_bound; // Temporary variable -// - uint8_t *in; -// - size_t in_pos_local; // Local alias for *in_pos +/// When decoding has been properly finished, rc.code is always zero unless +/// the input stream is corrupt. So checking this can catch some corrupt +/// files especially if they don't have any other integrity check. +#define rc_is_finished(range_decoder) \ + ((range_decoder).code == 0) -////////////////// -// Buffer "I/O" // -////////////////// - -// Read the next byte of compressed data from buffer_in, if needed. -#define rc_normalize() \ +/// Read the next input byte if needed. If more input is needed but there is +/// no more input available, "goto out" is used to jump out of the main +/// decoder loop. +#define rc_normalize(seq) \ do { \ - if (rc.range < TOP_VALUE) { \ - rc.range <<= SHIFT_BITS; \ - rc.code = (rc.code << SHIFT_BITS) | in[in_pos_local++]; \ + if (rc.range < RC_TOP_VALUE) { \ + if (unlikely(rc_in_pos == in_size)) { \ + coder->sequence = seq; \ + goto out; \ + } \ + rc.range <<= RC_SHIFT_BITS; \ + rc.code = (rc.code << RC_SHIFT_BITS) | in[rc_in_pos++]; \ } \ } while (0) -////////////////// -// Bit decoding // -////////////////// - -// Range decoder's DecodeBit() is splitted into three macros: -// if_bit_0(prob) { -// update_bit_0(prob) -// ... -// } else { -// update_bit_1(prob) -// ... -// } - -#define if_bit_0(prob) \ - rc_normalize(); \ - rc_bound = (rc.range >> BIT_MODEL_TOTAL_BITS) * (prob); \ +/// Start decoding a bit. This must be used together with rc_update_0() +/// and rc_update_1(): +/// +/// rc_if_0(prob, seq) { +/// rc_update_0(prob); +/// // Do something +/// } else { +/// rc_update_1(prob); +/// // Do something else +/// } +/// +#define rc_if_0(prob, seq) \ + rc_normalize(seq); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) -#define update_bit_0(prob) \ +/// Update the range decoder state and the used probability variable to +/// match a decoded bit of 0. +#define rc_update_0(prob) \ do { \ rc.range = rc_bound; \ - prob += (BIT_MODEL_TOTAL - (prob)) >> MOVE_BITS; \ + prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \ } while (0) -#define update_bit_1(prob) \ +/// Update the range decoder state and the used probability variable to +/// match a decoded bit of 1. +#define rc_update_1(prob) \ do { \ rc.range -= rc_bound; \ rc.code -= rc_bound; \ - prob -= (prob) >> MOVE_BITS; \ + prob -= (prob) >> RC_MOVE_BITS; \ } while (0) -#define rc_decode_direct(dest, count) \ +/// Decodes one bit and runs action0 or action1 depending on the decoded bit. +/// This macro is used as the last step in bittree reverse decoders since +/// those don't use "symbol" for anything else than indexing the probability +/// arrays. +#define rc_bit_last(prob, action0, action1, seq) \ do { \ - rc_normalize(); \ + rc_if_0(prob, seq) { \ + rc_update_0(prob); \ + action0; \ + } else { \ + rc_update_1(prob); \ + action1; \ + } \ +} while (0) + + +/// Decodes one bit, updates "symbol", and runs action0 or action1 depending +/// on the decoded bit. +#define rc_bit(prob, action0, action1, seq) \ + rc_bit_last(prob, \ + symbol <<= 1; action0, \ + symbol = (symbol << 1) + 1; action1, \ + seq); + + +/// Like rc_bit() but add "case seq:" as a prefix. This makes the unrolled +/// loops more readable because the code isn't littered with "case" +/// statements. On the other hand this also makes it less readable, since +/// spotting the places where the decoder loop may be restarted is less +/// obvious. +#define rc_bit_case(prob, action0, action1, seq) \ + case seq: rc_bit(prob, action0, action1, seq) + + +/// Decode a bit without using a probability. +#define rc_direct(dest, seq) \ +do { \ + rc_normalize(seq); \ rc.range >>= 1; \ rc.code -= rc.range; \ rc_bound = UINT32_C(0) - (rc.code >> 31); \ rc.code += rc.range & rc_bound; \ dest = (dest << 1) + (rc_bound + 1); \ -} while (--count > 0) - - -// Dummy versions don't update prob or dest. -#define update_bit_0_dummy() \ - rc.range = rc_bound - - -#define update_bit_1_dummy() \ -do { \ - rc.range -= rc_bound; \ - rc.code -= rc_bound; \ } while (0) -#define rc_decode_direct_dummy(count) \ -do { \ - rc_normalize(); \ - rc.range >>= 1; \ - rc.code -= rc.range; \ - rc.code += rc.range & (UINT32_C(0) - (rc.code >> 31)); \ -} while (--count > 0) - - -/////////////////////// -// Bit tree decoding // -/////////////////////// - -#define bittree_decode(target, probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0((probs)[model_index]); \ - model_index <<= 1; \ - } else { \ - update_bit_1((probs)[model_index]); \ - model_index = (model_index << 1) | 1; \ - } \ - } \ - target += model_index - (1 << bit_levels); \ -} while (0) - - -#define bittree_reverse_decode(target, probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0((probs)[model_index]); \ - model_index <<= 1; \ - } else { \ - update_bit_1((probs)[model_index]); \ - model_index = (model_index << 1) | 1; \ - target += 1 << bit_index; \ - } \ - } \ -} while (0) - - -// Dummy versions don't update prob. -#define bittree_decode_dummy(target, probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = (bit_levels); bit_index != 0; --bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0_dummy(); \ - model_index <<= 1; \ - } else { \ - update_bit_1_dummy(); \ - model_index = (model_index << 1) | 1; \ - } \ - } \ - target += model_index - (1 << bit_levels); \ -} while (0) - - -#define bittree_reverse_decode_dummy(probs, bit_levels) \ -do { \ - uint32_t model_index = 1; \ - for (uint32_t bit_index = 0; bit_index < bit_levels; ++bit_index) { \ - if_bit_0((probs)[model_index]) { \ - update_bit_0_dummy(); \ - model_index <<= 1; \ - } else { \ - update_bit_1_dummy(); \ - model_index = (model_index << 1) | 1; \ - } \ - } \ -} while (0) +// NOTE: No macros are provided for bittree decoding. It seems to be simpler +// to just write them open in the code. #endif diff --git a/src/liblzma/rangecoder/range_encoder.h b/src/liblzma/rangecoder/range_encoder.h index b156ee7f..f66e955c 100644 --- a/src/liblzma/rangecoder/range_encoder.h +++ b/src/liblzma/rangecoder/range_encoder.h @@ -22,6 +22,7 @@ #define LZMA_RANGE_ENCODER_H #include "range_common.h" +#include "price.h" /// Maximum number of symbols that can be put pending into lzma_range_encoder @@ -87,7 +88,7 @@ rc_bittree(lzma_range_encoder *rc, probability *probs, do { const uint32_t bit = (symbol >> --bit_count) & 1; rc_bit(rc, &probs[model_index], bit); - model_index = (model_index << 1) | bit; + model_index = (model_index << 1) + bit; } while (bit_count != 0); } @@ -102,7 +103,7 @@ rc_bittree_reverse(lzma_range_encoder *rc, probability *probs, const uint32_t bit = symbol & 1; symbol >>= 1; rc_bit(rc, &probs[model_index], bit); - model_index = (model_index << 1) | bit; + model_index = (model_index << 1) + bit; } while (--bit_count != 0); } @@ -146,7 +147,7 @@ rc_shift_low(lzma_range_encoder *rc, } ++rc->cache_size; - rc->low = (rc->low & 0x00FFFFFF) << SHIFT_BITS; + rc->low = (rc->low & 0x00FFFFFF) << RC_SHIFT_BITS; return false; } @@ -156,32 +157,35 @@ static inline bool rc_encode(lzma_range_encoder *rc, uint8_t *out, size_t *out_pos, size_t out_size) { + assert(rc->count <= RC_SYMBOLS_MAX); + while (rc->pos < rc->count) { // Normalize - if (rc->range < TOP_VALUE) { + if (rc->range < RC_TOP_VALUE) { if (rc_shift_low(rc, out, out_pos, out_size)) return true; - rc->range <<= SHIFT_BITS; + rc->range <<= RC_SHIFT_BITS; } // Encode a bit switch (rc->symbols[rc->pos]) { case RC_BIT_0: { probability prob = *rc->probs[rc->pos]; - rc->range = (rc->range >> BIT_MODEL_TOTAL_BITS) * prob; - prob += (BIT_MODEL_TOTAL - prob) >> MOVE_BITS; + rc->range = (rc->range >> RC_BIT_MODEL_TOTAL_BITS) + * prob; + prob += (RC_BIT_MODEL_TOTAL - prob) >> RC_MOVE_BITS; *rc->probs[rc->pos] = prob; break; } case RC_BIT_1: { probability prob = *rc->probs[rc->pos]; - const uint32_t bound = prob - * (rc->range >> BIT_MODEL_TOTAL_BITS); + const uint32_t bound = prob * (rc->range + >> RC_BIT_MODEL_TOTAL_BITS); rc->low += bound; rc->range -= bound; - prob -= prob >> MOVE_BITS; + prob -= prob >> RC_MOVE_BITS; *rc->probs[rc->pos] = prob; break; } @@ -231,72 +235,4 @@ rc_pending(const lzma_range_encoder *rc) return rc->cache_size + 5 - 1; } - -//////////// -// Prices // -//////////// - -#ifdef HAVE_SMALL -/// Probability prices used by *_get_price() macros. This is initialized -/// by lzma_rc_init() and is not modified later. -extern uint32_t lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; - -/// Initializes lzma_rc_prob_prices[]. This needs to be called only once. -extern void lzma_rc_init(void); - -#else -// Not building a size optimized version, so we use a precomputed -// constant table. -extern const uint32_t -lzma_rc_prob_prices[BIT_MODEL_TOTAL >> MOVE_REDUCING_BITS]; - -#endif - - -#define bit_get_price(prob, symbol) \ - lzma_rc_prob_prices[((((prob) - (symbol)) ^ (-(symbol))) \ - & (BIT_MODEL_TOTAL - 1)) >> MOVE_REDUCING_BITS] - - -#define bit_get_price_0(prob) \ - lzma_rc_prob_prices[(prob) >> MOVE_REDUCING_BITS] - - -#define bit_get_price_1(prob) \ - lzma_rc_prob_prices[(BIT_MODEL_TOTAL - (prob)) >> MOVE_REDUCING_BITS] - - -static inline uint32_t -bittree_get_price(const probability *probs, - uint32_t bit_levels, uint32_t symbol) -{ - uint32_t price = 0; - symbol |= UINT32_C(1) << bit_levels; - - do { - price += bit_get_price(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } while (symbol != 1); - - return price; -} - - -static inline uint32_t -bittree_reverse_get_price(const probability *probs, - uint32_t bit_levels, uint32_t symbol) -{ - uint32_t price = 0; - uint32_t model_index = 1; - - do { - const uint32_t bit = symbol & 1; - symbol >>= 1; - price += bit_get_price(probs[model_index], bit); - model_index = (model_index << 1) | bit; - } while (--bit_levels != 0); - - return price; -} - #endif diff --git a/src/liblzma/simple/Makefile.am b/src/liblzma/simple/Makefile.am index a37f1eb5..f8cd4888 100644 --- a/src/liblzma/simple/Makefile.am +++ b/src/liblzma/simple/Makefile.am @@ -21,6 +21,18 @@ libsimple_la_SOURCES = \ simple_coder.h \ simple_private.h +if COND_ENCODER_SIMPLE +libsimple_la_SOURCES += \ + simple_encoder.c \ + simple_encoder.h +endif + +if COND_DECODER_SIMPLE +libsimple_la_SOURCES += \ + simple_decoder.c \ + simple_decoder.h +endif + if COND_FILTER_X86 libsimple_la_SOURCES += x86.c endif diff --git a/src/liblzma/simple/simple_coder.c b/src/liblzma/simple/simple_coder.c index 078f1b95..3ab56582 100644 --- a/src/liblzma/simple/simple_coder.c +++ b/src/liblzma/simple/simple_coder.c @@ -33,7 +33,7 @@ copy_or_code(lzma_coder *coder, lzma_allocator *allocator, assert(!coder->end_was_reached); if (coder->next.code == NULL) { - bufcpy(in, in_pos, in_size, out, out_pos, out_size); + lzma_bufcpy(in, in_pos, in_size, out, out_pos, out_size); // Check if end of stream was reached. if (coder->is_encoder && action == LZMA_FINISH @@ -91,7 +91,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, // Flush already filtered data from coder->buffer[] to out[]. if (coder->pos < coder->filtered) { - bufcpy(coder->buffer, &coder->pos, coder->filtered, + lzma_bufcpy(coder->buffer, &coder->pos, coder->filtered, out, out_pos, out_size); // If we couldn't flush all the filtered data, return to @@ -195,7 +195,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, coder->filtered = coder->size; // Flush as much as possible. - bufcpy(coder->buffer, &coder->pos, coder->filtered, + lzma_bufcpy(coder->buffer, &coder->pos, coder->filtered, out, out_pos, out_size); } @@ -210,7 +210,7 @@ simple_code(lzma_coder *coder, lzma_allocator *allocator, static void simple_coder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); + lzma_next_end(&coder->next, allocator); lzma_free(coder->simple, allocator); lzma_free(coder, allocator); return; diff --git a/src/liblzma/simple/simple_decoder.c b/src/liblzma/simple/simple_decoder.c new file mode 100644 index 00000000..72f8ee16 --- /dev/null +++ b/src/liblzma/simple/simple_decoder.c @@ -0,0 +1,47 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_decoder.c +/// \brief Properties decoder for simple filters +// +// Copyright (C) 2007-2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_decoder.h" + + +extern lzma_ret +lzma_simple_props_decode(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + if (props_size == 0) + return LZMA_OK; + + if (props_size != 4) + return LZMA_HEADER_ERROR; + + lzma_options_simple *opt = lzma_alloc( + sizeof(lzma_options_simple), allocator); + if (opt == NULL) + return LZMA_MEM_ERROR; + + opt->start_offset = integer_read_32(props); + + // Don't leave an options structure allocated if start_offset is zero. + if (opt->start_offset == 0) + lzma_free(opt, allocator); + else + *options = opt; + + return LZMA_OK; +} diff --git a/src/liblzma/common/raw_decoder.h b/src/liblzma/simple/simple_decoder.h similarity index 67% rename from src/liblzma/common/raw_decoder.h rename to src/liblzma/simple/simple_decoder.h index c0e626a8..7d1f3d35 100644 --- a/src/liblzma/common/raw_decoder.h +++ b/src/liblzma/simple/simple_decoder.h @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file raw_decoder.h -/// \brief Raw decoder initialization API +/// \file simple_decoder.h +/// \brief Properties decoder for simple filters // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,13 +17,13 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_RAW_DECODER_H -#define LZMA_RAW_DECODER_H +#ifndef LZMA_SIMPLE_DECODER_H +#define LZMA_SIMPLE_DECODER_H -#include "raw_common.h" +#include "simple_coder.h" - -extern lzma_ret lzma_raw_decoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_options_filter *options); +extern lzma_ret lzma_simple_props_decode( + void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); #endif diff --git a/src/liblzma/simple/simple_encoder.c b/src/liblzma/simple/simple_encoder.c new file mode 100644 index 00000000..15d888d9 --- /dev/null +++ b/src/liblzma/simple/simple_encoder.c @@ -0,0 +1,45 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file simple_encoder.c +/// \brief Properties encoder for simple filters +// +// Copyright (C) 2007-2008 Lasse Collin +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "simple_encoder.h" + + +extern lzma_ret +lzma_simple_props_size(uint32_t *size, const void *options) +{ + const lzma_options_simple *const opt = options; + *size = (opt == NULL || opt->start_offset == 0) ? 0 : 4; + return LZMA_OK; +} + + +extern lzma_ret +lzma_simple_props_encode(const void *options, uint8_t *out) +{ + const lzma_options_simple *const opt = options; + + // The default start offset is zero, so we don't need to store any + // options unless the start offset is non-zero. + if (opt == NULL || opt->start_offset == 0) + return LZMA_OK; + + integer_write_32(out, opt->start_offset); + + return LZMA_OK; +} diff --git a/src/liblzma/common/raw_encoder.h b/src/liblzma/simple/simple_encoder.h similarity index 65% rename from src/liblzma/common/raw_encoder.h rename to src/liblzma/simple/simple_encoder.h index 4e148489..be4ca9fc 100644 --- a/src/liblzma/common/raw_encoder.h +++ b/src/liblzma/simple/simple_encoder.h @@ -1,9 +1,9 @@ /////////////////////////////////////////////////////////////////////////////// // -/// \file raw_encoder.h -/// \brief Raw encoder initialization API +/// \file simple_encoder.c +/// \brief Properties encoder for simple filters // -// Copyright (C) 2007 Lasse Collin +// Copyright (C) 2007-2008 Lasse Collin // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -17,13 +17,14 @@ // /////////////////////////////////////////////////////////////////////////////// -#ifndef LZMA_RAW_ENCODER_H -#define LZMA_RAW_ENCODER_H +#ifndef LZMA_SIMPLE_ENCODER_H +#define LZMA_SIMPLE_ENCODER_H -#include "raw_common.h" +#include "simple_coder.h" -extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next, - lzma_allocator *allocator, const lzma_options_filter *options); +extern lzma_ret lzma_simple_props_size(uint32_t *size, const void *options); + +extern lzma_ret lzma_simple_props_encode(const void *options, uint8_t *out); #endif diff --git a/src/liblzma/subblock/Makefile.am b/src/liblzma/subblock/Makefile.am index 8f2daf59..2577e0b4 100644 --- a/src/liblzma/subblock/Makefile.am +++ b/src/liblzma/subblock/Makefile.am @@ -18,13 +18,13 @@ libsubblock_la_CPPFLAGS = \ -I@top_srcdir@/src/liblzma/api \ -I@top_srcdir@/src/liblzma/common -if COND_MAIN_ENCODER +if COND_ENCODER_SUBBLOCK libsubblock_la_SOURCES += \ subblock_encoder.c \ subblock_encoder.h endif -if COND_MAIN_DECODER +if COND_DECODER_SUBBLOCK libsubblock_la_SOURCES += \ subblock_decoder.c \ subblock_decoder.h \ diff --git a/src/liblzma/subblock/subblock_decoder.c b/src/liblzma/subblock/subblock_decoder.c index 39ec35c1..faf198c6 100644 --- a/src/liblzma/subblock/subblock_decoder.c +++ b/src/liblzma/subblock/subblock_decoder.c @@ -19,7 +19,7 @@ #include "subblock_decoder.h" #include "subblock_decoder_helper.h" -#include "raw_decoder.h" +#include "filter_decoder.h" /// Maximum number of consecutive Subblocks with Subblock Type Padding @@ -78,7 +78,7 @@ struct lzma_coder_s { lzma_next_coder filter_flags_decoder; /// The filter_flags_decoder stores its results here. - lzma_options_filter filter_flags; + lzma_filter filter_flags; /// Options for the Subblock decoder helper. This is used to tell /// the helper when it should return LZMA_STREAM_END to the subfilter. @@ -239,7 +239,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, // if Subfilter isn't used again, we could leave // a memory-hogging filter dangling until someone // frees Subblock filter itself. - lzma_next_coder_end(&coder->subfilter, allocator); + lzma_next_end(&coder->subfilter, allocator); // Free memory used for subfilter options. This is // safe, because we don't support any Subfilter that @@ -276,7 +276,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, coder->helper.end_was_reached = false; - lzma_options_filter filters[3] = { + lzma_filter filters[3] = { { .id = coder->filter_flags.id, .options = coder->filter_flags.options, @@ -406,7 +406,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, in_limit = in_size; if (coder->subfilter.code == NULL) { - const size_t copy_size = bufcpy( + const size_t copy_size = lzma_bufcpy( in, in_pos, in_limit, out, out_pos, out_size); @@ -480,7 +480,7 @@ decode_buffer(lzma_coder *coder, lzma_allocator *allocator, } if (coder->subfilter.code == NULL) { - bufcpy(coder->repeat.buffer, + lzma_bufcpy(coder->repeat.buffer, &coder->repeat.pos, coder->repeat.size, out, out_pos, out_size); @@ -586,9 +586,9 @@ subblock_decode(lzma_coder *coder, lzma_allocator *allocator, static void subblock_decoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); - lzma_next_coder_end(&coder->subfilter, allocator); - lzma_next_coder_end(&coder->filter_flags_decoder, allocator); + lzma_next_end(&coder->next, allocator); + lzma_next_end(&coder->subfilter, allocator); + lzma_next_end(&coder->filter_flags_decoder, allocator); lzma_free(coder->filter_flags.options, allocator); lzma_free(coder, allocator); return; @@ -612,7 +612,7 @@ lzma_subblock_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->coder->filter_flags_decoder = LZMA_NEXT_CODER_INIT; } else { - lzma_next_coder_end(&next->coder->subfilter, allocator); + lzma_next_end(&next->coder->subfilter, allocator); lzma_free(next->coder->filter_flags.options, allocator); } diff --git a/src/liblzma/subblock/subblock_decoder_helper.c b/src/liblzma/subblock/subblock_decoder_helper.c index e8063e1e..ca8fed93 100644 --- a/src/liblzma/subblock/subblock_decoder_helper.c +++ b/src/liblzma/subblock/subblock_decoder_helper.c @@ -40,7 +40,7 @@ helper_decode(lzma_coder *coder, // We can safely copy as much as possible, because we are never // given more data than a single Subblock Data field. - bufcpy(in, in_pos, in_size, out, out_pos, out_size); + lzma_bufcpy(in, in_pos, in_size, out, out_pos, out_size); // Return LZMA_STREAM_END when instructed so by the Subblock decoder. return coder->options->end_was_reached ? LZMA_STREAM_END : LZMA_OK; diff --git a/src/liblzma/subblock/subblock_encoder.c b/src/liblzma/subblock/subblock_encoder.c index 01e8007a..e78ffca6 100644 --- a/src/liblzma/subblock/subblock_encoder.c +++ b/src/liblzma/subblock/subblock_encoder.c @@ -18,7 +18,7 @@ /////////////////////////////////////////////////////////////////////////////// #include "subblock_encoder.h" -#include "raw_encoder.h" +#include "filter_encoder.h" /// Maximum number of repeats that a single Repeating Data can indicate. @@ -398,7 +398,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, assert(coder->subfilter.subcoder.code == NULL); // No Subfilter is enabled, just copy the data as is. - coder->subblock.in_pending += bufcpy( + coder->subblock.in_pending += lzma_bufcpy( in, in_pos, in_size, coder->subblock.data, &coder->subblock.size, @@ -480,7 +480,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, // otherwise the Subfilter's memory could be // left allocated for long time, and would // just waste memory. - lzma_next_coder_end(&coder->subfilter.subcoder, + lzma_next_end(&coder->subfilter.subcoder, allocator); // We need to flush the currently buffered @@ -728,7 +728,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, break; case SEQ_RLE_DATA: - bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size, + lzma_bufcpy(coder->rle.buffer, &coder->pos, coder->rle.size, out, out_pos, out_size); if (coder->pos < coder->rle.size) return LZMA_OK; @@ -767,7 +767,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, break; case SEQ_DATA: - bufcpy(coder->subblock.data, &coder->pos, + lzma_bufcpy(coder->subblock.data, &coder->pos, coder->subblock.size, out, out_pos, out_size); if (coder->pos < coder->subblock.size) return LZMA_OK; @@ -791,7 +791,7 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, return LZMA_HEADER_ERROR; // Initialize a raw encoder to work as a Subfilter. - lzma_options_filter options[2]; + lzma_filter options[2]; options[0] = coder->options->subfilter_options; options[1].id = LZMA_VLI_VALUE_UNKNOWN; @@ -817,8 +817,8 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, // Now we have a big-enough buffer. Encode the Filter Flags. // Like above, this should never fail. size_t dummy = 0; - ret = lzma_filter_flags_encode(coder->subfilter.flags, - &dummy, coder->subfilter.flags_size, options); + ret = lzma_filter_flags_encode(options, coder->subfilter.flags, + &dummy, coder->subfilter.flags_size); assert(ret == LZMA_OK); assert(dummy == coder->subfilter.flags_size); if (ret != LZMA_OK || dummy != coder->subfilter.flags_size) @@ -833,15 +833,15 @@ subblock_buffer(lzma_coder *coder, lzma_allocator *allocator, coder->sequence = SEQ_SUBFILTER_FLAGS; // It is safe to fall through because SEQ_SUBFILTER_FLAGS - // uses bufcpy() which doesn't write unless there is output - // space. + // uses lzma_bufcpy() which doesn't write unless there is + // output space. } // Fall through case SEQ_SUBFILTER_FLAGS: // Copy the Filter Flags to the output stream. - bufcpy(coder->subfilter.flags, &coder->pos, + lzma_bufcpy(coder->subfilter.flags, &coder->pos, coder->subfilter.flags_size, out, out_pos, out_size); if (coder->pos < coder->subfilter.flags_size) @@ -912,8 +912,8 @@ subblock_encode(lzma_coder *coder, lzma_allocator *allocator, static void subblock_encoder_end(lzma_coder *coder, lzma_allocator *allocator) { - lzma_next_coder_end(&coder->next, allocator); - lzma_next_coder_end(&coder->subfilter.subcoder, allocator); + lzma_next_end(&coder->next, allocator); + lzma_next_end(&coder->subfilter.subcoder, allocator); lzma_free(coder->subblock.data, allocator); lzma_free(coder->subfilter.flags, allocator); lzma_free(coder, allocator); @@ -938,7 +938,7 @@ lzma_subblock_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, next->coder->subblock.limit = 0; next->coder->subfilter.subcoder = LZMA_NEXT_CODER_INIT; } else { - lzma_next_coder_end(&next->coder->subfilter.subcoder, + lzma_next_end(&next->coder->subfilter.subcoder, allocator); lzma_free(next->coder->subfilter.flags, allocator); } diff --git a/src/lzma/args.c b/src/lzma/args.c index a4764032..30df4522 100644 --- a/src/lzma/args.c +++ b/src/lzma/args.c @@ -39,8 +39,8 @@ bool opt_force = false; bool opt_keep_original = false; bool opt_preserve_name = false; -lzma_check_type opt_check = LZMA_CHECK_CRC64; -lzma_options_filter opt_filters[8]; +lzma_check opt_check = LZMA_CHECK_CRC64; +lzma_filter opt_filters[8]; // We don't modify or free() this, but we need to assign it in some // non-const pointers. @@ -61,6 +61,7 @@ enum { OPT_SPARC, OPT_DELTA, OPT_LZMA, + OPT_LZMA2, OPT_FILES, OPT_FILES0, @@ -108,6 +109,7 @@ static const struct option long_opts[] = { { "sparc", no_argument, NULL, OPT_SPARC }, { "delta", optional_argument, NULL, OPT_DELTA }, { "lzma", optional_argument, NULL, OPT_LZMA }, + { "lzma2", optional_argument, NULL, OPT_LZMA2 }, // Other { "format", required_argument, NULL, 'F' }, @@ -141,6 +143,7 @@ add_filter(lzma_vli id, const char *opt_str) break; case LZMA_FILTER_LZMA: + case LZMA_FILTER_LZMA2: opt_filters[filter_count].options = parse_options_lzma(opt_str); break; @@ -301,6 +304,10 @@ parse_real(int argc, char **argv) add_filter(LZMA_FILTER_LZMA, optarg); break; + case OPT_LZMA2: + add_filter(LZMA_FILTER_LZMA2, optarg); + break; + // Other // --format @@ -445,7 +452,8 @@ static void set_compression_settings(void) { if (filter_count == 0) { - opt_filters[0].id = LZMA_FILTER_LZMA; + opt_filters[0].id = opt_header == HEADER_ALONE + ? LZMA_FILTER_LZMA : LZMA_FILTER_LZMA2; opt_filters[0].options = (lzma_options_lzma *)( lzma_preset_lzma + preset_number); filter_count = 1; @@ -463,13 +471,15 @@ set_compression_settings(void) my_exit(ERROR); } - const uint32_t memory_limit = opt_memory / (1024 * 1024) + 1; - uint32_t memory_usage = lzma_memory_usage(opt_filters, true); + uint64_t memory_usage = lzma_memusage_encoder(opt_filters); + /* opt_mode == MODE_COMPRESS + ? lzma_memusage_encoder(opt_filters) + : lzma_memusage_decoder(opt_filters); */ // Don't go over the memory limits when the default // setting is used. if (preset_default) { - while (memory_usage > memory_limit) { + while (memory_usage > opt_memory) { if (preset_number == 0) { errmsg(V_ERROR, _("Memory usage limit is too " "small for any internal " @@ -481,11 +491,10 @@ set_compression_settings(void) opt_filters[0].options = (lzma_options_lzma *)( lzma_preset_lzma + preset_number); - memory_usage = lzma_memory_usage(opt_filters, - true); + memory_usage = lzma_memusage_encoder(opt_filters); } } else { - if (memory_usage > memory_limit) { + if (memory_usage > opt_memory) { errmsg(V_ERROR, _("Memory usage limit is too small " "for the given filter setup")); my_exit(ERROR); @@ -494,12 +503,8 @@ set_compression_settings(void) // Limit the number of worked threads so that memory usage // limit isn't exceeded. - // FIXME: Probably should use bytes instead of mebibytes for - // memory_usage and memory_limit. - if (memory_usage == 0) - memory_usage = 1; - - size_t thread_limit = memory_limit / memory_usage; + assert(memory_usage > 0); + size_t thread_limit = opt_memory / memory_usage; if (thread_limit == 0) thread_limit = 1; diff --git a/src/lzma/args.h b/src/lzma/args.h index c6098558..b60e7545 100644 --- a/src/lzma/args.h +++ b/src/lzma/args.h @@ -52,8 +52,8 @@ extern bool opt_preserve_name; extern enum tool_mode opt_mode; extern enum header_type opt_header; -extern lzma_check_type opt_check; -extern lzma_options_filter opt_filters[8]; +extern lzma_check opt_check; +extern lzma_filter opt_filters[8]; extern const char *stdin_filename; diff --git a/src/lzma/options.c b/src/lzma/options.c index c82cb1a0..b2ec200e 100644 --- a/src/lzma/options.c +++ b/src/lzma/options.c @@ -299,9 +299,9 @@ extern lzma_options_lzma * parse_options_lzma(const char *str) { static const name_id_map modes[] = { - { "fast", LZMA_MODE_FAST }, - { "best", LZMA_MODE_BEST }, - { NULL, 0 } + { "fast", LZMA_MODE_FAST }, + { "normal", LZMA_MODE_NORMAL }, + { NULL, 0 } }; static const name_id_map mfs[] = { @@ -317,9 +317,9 @@ parse_options_lzma(const char *str) { "dict", NULL, LZMA_DICTIONARY_SIZE_MIN, LZMA_DICTIONARY_SIZE_MAX }, { "lc", NULL, LZMA_LITERAL_CONTEXT_BITS_MIN, - LZMA_LITERAL_CONTEXT_BITS_MAX }, + LZMA_LITERAL_CONTEXT_BITS_MAX }, { "lp", NULL, LZMA_LITERAL_POS_BITS_MIN, - LZMA_LITERAL_POS_BITS_MAX }, + LZMA_LITERAL_POS_BITS_MAX }, { "pb", NULL, LZMA_POS_BITS_MIN, LZMA_POS_BITS_MAX }, { "mode", modes, 0, 0 }, { "fb", NULL, LZMA_FAST_BYTES_MIN, LZMA_FAST_BYTES_MAX }, @@ -334,7 +334,9 @@ parse_options_lzma(const char *str) .literal_context_bits = LZMA_LITERAL_CONTEXT_BITS_DEFAULT, .literal_pos_bits = LZMA_LITERAL_POS_BITS_DEFAULT, .pos_bits = LZMA_POS_BITS_DEFAULT, - .mode = LZMA_MODE_BEST, + .preset_dictionary = NULL, + .persistent = false, + .mode = LZMA_MODE_NORMAL, .fast_bytes = LZMA_FAST_BYTES_DEFAULT, .match_finder = LZMA_MF_BT4, .match_finder_cycles = 0, diff --git a/src/lzma/process.c b/src/lzma/process.c index c180caf7..b4387709 100644 --- a/src/lzma/process.c +++ b/src/lzma/process.c @@ -63,12 +63,7 @@ process_init(void) } for (size_t i = 0; i < opt_threads; ++i) - threads[i] = (thread_data){ - .strm = LZMA_STREAM_INIT_VAR, - .options = NULL, - .pair = NULL, - .in_use = false, - }; + memzero(&threads[i], sizeof(threads[0])); if (pthread_attr_init(&thread_attr) || pthread_attr_setdetachstate( @@ -169,7 +164,9 @@ single_init(thread_data *t) } } else { // TODO Restrict file format if requested on the command line. - ret = lzma_auto_decoder(&t->strm); + ret = lzma_auto_decoder(&t->strm, opt_memory, + LZMA_WARN_UNSUPPORTED_CHECK + | LZMA_CONCATENATED); } if (ret != LZMA_OK) { @@ -185,36 +182,6 @@ single_init(thread_data *t) } -static lzma_ret -single_skip_padding(thread_data *t, uint8_t *in_buf) -{ - // Handle decoding of concatenated Streams. There can be arbitrary - // number of nul-byte padding between the Streams, which must be - // ignored. - // - // NOTE: Concatenating LZMA_Alone files works only if at least - // one of lc, lp, and pb is non-zero. Using the concatenation - // on LZMA_Alone files is strongly discouraged. - while (true) { - while (t->strm.avail_in > 0) { - if (*t->strm.next_in != '\0') - return LZMA_OK; - - ++t->strm.next_in; - --t->strm.avail_in; - } - - if (t->pair->src_eof) - return LZMA_STREAM_END; - - t->strm.next_in = in_buf; - t->strm.avail_in = io_read(t->pair, in_buf, BUFSIZ); - if (t->strm.avail_in == SIZE_MAX) - return LZMA_DATA_ERROR; - } -} - - static void * single(thread_data *t) { @@ -227,10 +194,11 @@ single(thread_data *t) uint8_t in_buf[BUFSIZ]; uint8_t out_buf[BUFSIZ]; lzma_action action = LZMA_RUN; - lzma_ret ret; bool success = false; t->strm.avail_in = 0; + t->strm.next_out = out_buf; + t->strm.avail_out = BUFSIZ; while (!user_abort) { if (t->strm.avail_in == 0 && !t->pair->src_eof) { @@ -239,48 +207,36 @@ single(thread_data *t) if (t->strm.avail_in == SIZE_MAX) break; - else if (t->pair->src_eof - && opt_mode == MODE_COMPRESS) + + if (t->pair->src_eof) action = LZMA_FINISH; } - t->strm.next_out = out_buf; - t->strm.avail_out = BUFSIZ; + const lzma_ret ret = lzma_code(&t->strm, action); - ret = lzma_code(&t->strm, action); - - if (opt_mode != MODE_TEST) + if ((t->strm.avail_out == 0 || ret != LZMA_OK) + && opt_mode != MODE_TEST) { if (io_write(t->pair, out_buf, BUFSIZ - t->strm.avail_out)) break; + t->strm.next_out = out_buf; + t->strm.avail_out = BUFSIZ; + } + if (ret != LZMA_OK) { if (ret == LZMA_STREAM_END) { - if (opt_mode == MODE_COMPRESS) { - assert(t->pair->src_eof); - success = true; - break; - } - - // Support decoding concatenated .lzma files. - ret = single_skip_padding(t, in_buf); - - if (ret == LZMA_STREAM_END) { - assert(t->pair->src_eof); - success = true; - break; - } - - if (ret == LZMA_OK && !single_init(t)) - continue; - - break; - + // FIXME !!! This doesn't work when decoding + // LZMA_Alone files, because LZMA_Alone decoder + // doesn't wait for LZMA_FINISH. + assert(t->pair->src_eof); + success = true; } else { errmsg(V_ERROR, "%s: %s", t->pair->src_name, str_strm_error(ret)); - break; } + + break; } } diff --git a/src/lzmadec/lzmadec.c b/src/lzmadec/lzmadec.c index 1fc561b7..ed5947ad 100644 --- a/src/lzmadec/lzmadec.c +++ b/src/lzmadec/lzmadec.c @@ -65,14 +65,7 @@ static uint8_t out_buf[BUFSIZ]; static lzma_stream strm = LZMA_STREAM_INIT; /// Number of bytes to use memory at maximum -static size_t mem_limit; - -/// Memory allocation hooks -static lzma_allocator allocator = { - .alloc = (void *(*)(void *, size_t, size_t))(&lzma_memlimit_alloc), - .free = (void (*)(void *, void *))(&lzma_memlimit_free), - .opaque = NULL, -}; +static size_t memlimit; /// Program name to be shown in error messages static const char *argv0; @@ -116,7 +109,7 @@ help(void) " MiB of memory at maximum.\n" "\n" "Report bugs to <" PACKAGE_BUGREPORT "> (in English or Finnish).\n", - argv0, ((uint64_t)(mem_limit) + 512 * 1024) / (1024 * 1024)); + argv0, ((uint64_t)(memlimit) + 512 * 1024) / (1024 * 1024)); // Using PRIu64 above instead of %zu to support pre-C99 libc. exit(0); } @@ -148,7 +141,7 @@ version(void) /// Finds out the amount of physical memory in the system, and sets /// a default memory usage limit. static void -set_default_mem_limit(void) +set_default_memlimit(void) { uint64_t mem = physmem(); if (mem != 0) { @@ -159,10 +152,10 @@ set_default_mem_limit(void) mem = SIZE_MAX; #endif - mem_limit = mem / 3; + memlimit = mem / 3; } else { // Cannot autodetect, use 10 MiB as the default limit. - mem_limit = (1U << 23) + (1U << 21); + memlimit = (1U << 23) + (1U << 21); } return; @@ -272,7 +265,7 @@ parse_options(int argc, char **argv) break; case 'M': - mem_limit = str_to_size(optarg); + memlimit = str_to_size(optarg); break; case 'h': @@ -309,19 +302,20 @@ parse_options(int argc, char **argv) static void init(void) { + const uint32_t flags = LZMA_WARN_UNSUPPORTED_CHECK | LZMA_CONCATENATED; lzma_ret ret; switch (format_type) { case FORMAT_AUTO: - ret = lzma_auto_decoder(&strm); + ret = lzma_auto_decoder(&strm, memlimit, flags); break; case FORMAT_NATIVE: - ret = lzma_stream_decoder(&strm); + ret = lzma_stream_decoder(&strm, memlimit, flags); break; case FORMAT_ALONE: - ret = lzma_alone_decoder(&strm); + ret = lzma_alone_decoder(&strm, memlimit); break; default: @@ -344,50 +338,6 @@ init(void) } -static void -read_input(void) -{ - strm.next_in = in_buf; - strm.avail_in = fread(in_buf, 1, BUFSIZ, file); - - if (ferror(file)) { - // POSIX says that fread() sets errno if an error occurred. - // ferror() doesn't touch errno. - fprintf(stderr, "%s: %s: Error reading input file: %s\n", - argv0, filename, strerror(errno)); - exit(ERROR); - } - - return; -} - - -static bool -skip_padding(void) -{ - // Handle concatenated Streams. There can be arbitrary number of - // nul-byte padding between the Streams, which must be ignored. - // - // NOTE: Concatenating LZMA_Alone files works only if at least - // one of lc, lp, and pb is non-zero. Using the concatenation - // on LZMA_Alone files is strongly discouraged. - while (true) { - while (strm.avail_in > 0) { - if (*strm.next_in != '\0') - return true; - - ++strm.next_in; - --strm.avail_in; - } - - if (feof(file)) - return false; - - read_input(); - } -} - - static void uncompress(void) { @@ -400,44 +350,65 @@ uncompress(void) } init(); + strm.avail_in = 0; + strm.next_out = out_buf; + strm.avail_out = BUFSIZ; + + lzma_action action = LZMA_RUN; while (true) { - if (strm.avail_in == 0) - read_input(); + if (strm.avail_in == 0) { + strm.next_in = in_buf; + strm.avail_in = fread(in_buf, 1, BUFSIZ, file); - strm.next_out = out_buf; - strm.avail_out = BUFSIZ; + if (ferror(file)) { + // POSIX says that fread() sets errno if + // an error occurred. ferror() doesn't + // touch errno. + fprintf(stderr, "%s: %s: Error reading " + "input file: %s\n", + argv0, filename, + strerror(errno)); + exit(ERROR); + } - const lzma_ret ret = lzma_code(&strm, LZMA_RUN); + if (feof(file)) + action = LZMA_FINISH; + } + + const lzma_ret ret = lzma_code(&strm, action); // Write and check write error before checking decoder error. // This way as much data as possible gets written to output - // even if decoder detected an error. Checking write error - // needs to be done before checking decoder error due to - // how concatenated Streams are handled a few lines later. - const size_t write_size = BUFSIZ - strm.avail_out; - if (fwrite(out_buf, 1, write_size, stdout) != write_size) { - // Wouldn't be a surprise if writing to stderr would - // fail too but at least try to show an error message. - fprintf(stderr, "%s: Cannot write to " - "standard output: %s\n", argv0, - strerror(errno)); - exit(ERROR); + // even if decoder detected an error. + if (strm.avail_out == 0 || ret != LZMA_OK) { + const size_t write_size = BUFSIZ - strm.avail_out; + + if (fwrite(out_buf, 1, write_size, stdout) + != write_size) { + // Wouldn't be a surprise if writing to stderr + // would fail too but at least try to show an + // error message. + fprintf(stderr, "%s: Cannot write to " + "standard output: %s\n", argv0, + strerror(errno)); + exit(ERROR); + } + + strm.next_out = out_buf; + strm.avail_out = BUFSIZ; } if (ret != LZMA_OK) { - if (ret == LZMA_STREAM_END) { - if (skip_padding()) { - init(); - continue; - } - + // FIXME !!! Doesn't work with LZMA_Alone for the + // same reason as in process.c. + if (ret == LZMA_STREAM_END) return; - } fprintf(stderr, "%s: %s: ", argv0, filename); + // FIXME Add LZMA_*_CHECK and LZMA_FORMAT_ERROR. switch (ret) { case LZMA_DATA_ERROR: fprintf(stderr, "File is corrupt\n"); @@ -452,6 +423,11 @@ uncompress(void) fprintf(stderr, "%s\n", strerror(ENOMEM)); exit(ERROR); + case LZMA_MEMLIMIT_ERROR: + fprintf(stderr, "Memory usage limit " + "reached\n"); + exit(ERROR); + case LZMA_BUF_ERROR: fprintf(stderr, "Unexpected end of input\n"); exit(ERROR); @@ -479,23 +455,12 @@ main(int argc, char **argv) { argv0 = argv[0]; - set_default_mem_limit(); + set_default_memlimit(); parse_options(argc, argv); lzma_init_decoder(); - lzma_memlimit *mem_limiter = lzma_memlimit_create(mem_limit); - if (mem_limiter == NULL) { - fprintf(stderr, "%s: %s\n", argv0, strerror(ENOMEM)); - exit(ERROR); - } - - assert(lzma_memlimit_count(mem_limiter) == 0); - - allocator.opaque = mem_limiter; - strm.allocator = &allocator; - #ifdef WIN32 setmode(fileno(stdin), O_BINARY); setmode(fileno(stdout), O_BINARY); @@ -531,8 +496,6 @@ main(int argc, char **argv) // Free the memory only when debugging. Freeing wastes some time, // but allows detecting possible memory leaks with Valgrind. lzma_end(&strm); - assert(lzma_memlimit_count(mem_limiter) == 0); - lzma_memlimit_end(mem_limiter, false); #endif return exit_status; diff --git a/tests/Makefile.am b/tests/Makefile.am index f9f15c54..2d087e12 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -48,6 +48,7 @@ TESTS = \ test_filter_flags \ test_block_header \ test_index \ + test_files.sh \ test_compress.sh clean-local: diff --git a/tests/files/README b/tests/files/README index 4d0ef8bd..7c7f4e18 100644 --- a/tests/files/README +++ b/tests/files/README @@ -14,259 +14,192 @@ 1. File Types Good files (good-*.lzma) must decode successfully without requiring - a lot of CPU time or RAM. If the decoder supports only Single-Block - Streams, then good-multi-*.lzma won't decode, of course. + a lot of CPU time or RAM. + + Unsupported files (unsupported-*.lzma) are good files, but headers + indicate features not supported by the current file format + specification. Bad files (bad-*.lzma) must cause the decoder to give an error. Like with the good files, these files must not require a lot of CPU time or RAM before they get detected to be broken. - Malicious files (malicious-*.lzma) are good in terms of the file format - specification, but try to trigger excessive CPU, RAM or disk usage in - the decoder. To prevent malicious files from putting the decoder in - inifinite loop (*), eating all available RAM or disk space, decoders - should have internal limiters that catch these situations. - - (*) Strictly speaking not infinite, but if decoding of a small file - would take a few weeks or even years, it's an infinite loop in - practice. - 2. Descriptions of Individual Files 2.1. Good Files - good-single-none.lzma uses implicit Copy filter with known Uncompressed - Size. + good-0-empty.lzma has one Stream with no Blocks. - good-single-none-pad.lzma is good-single-none.lzma with Footer Padding. + good-0pad-empty.lzma has one Stream with no Blocks followed by + four-byte Stream Padding. - good-cat-single-none-pad.lzma is two good-single-none-pad.lzma files - concatenated as is. Fully decoding this file requires that the decoder - supports decoding concatenated files. + good-0cat-empty.lzma has two zero-Block Streams concatenated without + Stream Padding. - good-single-subblock_implicit.lzma uses implicit Subblock filter. + good-0catpad-empty.lzma has two zero-Block Streams concatenated with + four-byte Stream Padding between the Streams. - good-single-lzma.lzma is LZMA compressed file with EOPM. + good-1-check-none.lzma has one Stream with one Block with two + uncompressed LZMA2 chunks and no integrity check. - good-single-subblock-lzma.lzma has basic combination of Subblock and - LZMA filters. + good-1-check-crc32.lzma has one Stream with one Block with two + uncompressed LZMA2 chunks and CRC32 check. - good-single-none-empty_1.lzma is an empty file with implicit Copy - filter and no integrity Check. + good-1-check-crc64.lzma is like good-1-check-crc32.lzma but with CRC64. - good-single-none-empty_2.lzma is an empty file with implicit Copy - filter and CRC32 as Check. + good-1-check-sha256.lzma is like good-1-check-crc32.lzma but with + SHA256. - good-single-none-empty_3.lzma is an empty file with implicit Copy - filter, known Compressed Size, and no integrity Check. + good-2-lzma2.lzma has one Stream with two Blocks with one uncompressed + LZMA2 chunk in each Block. - good-single-lzma-empty.lzma is an empty file with LZMA filter and no - integrity Check. + good-1-block_header-1.lzma has both Compressed Size and Uncompressed + Size in the Block Header. This has also four extra bytes of Header + Padding. - good-single-subblock_rle.lzma takes advantage of Subblock filter's - run-length encoding. + good-1-block_header-2.lzma has known Compressed Size. - good-single-delta-lzma.tiff.lzma is an image file that compresses - better with Delta+LZMA than with plain LZMA. + good-1-block_header-3.lzma has known Uncompressed Size. - good-single-x86-lzma.lzma uses the x86 filter (BCJ) and LZMA. The + good-1-delta-lzma2.tiff.lzma is an image file that compresses + better with Delta+LZMA2 than with plain LZMA2. + + good-1-x86-lzma2.lzma uses the x86 filter (BCJ) and LZMA2. The uncompressed file is compress_prepared_bcj_x86 found from the tests directory. - good-single-sparc-lzma.lzma uses the SPARC filter and LZMA. The + good-1-sparc-lzma2.lzma uses the SPARC filter and LZMA. The uncompressed file is compress_prepared_bcj_sparc found from the tests directory. - good-single-lzma-flush_1.lzma has a flush marker in the middle of - the file, and no EOPM. + good-1-lzma2-1.lzma has two LZMA2 chunks, of which the second sets + new properties. - good-single-lzma-flush_2.lzma has a flush marker in the middle of - the file and just before EOPM. + good-1-lzma2-2.lzma has two LZMA2 chunks, of which the second resets + the state without specifying new properties. - good-multi-none-1.lzma is a basic Multi-Block Stream with two Data - Blocks and Footer Metadata Block. + good-1-lzma2-3.lzma has two LZMA2 chunks, of which the first is + uncompressed and the second is LZMA. The first chunk resets dictionary + and the second sets new properties. - good-multi-none-2.lzma is good-multi-none-1.lzma with Total Size and - Uncompressed Size added to the Footer Metadata Block. - - good-multi-none-extra_1.lzma has the `Extra is present' flag set but - no actual Extra Records. - - good-multi-none-extra_2.lzma has two non-empty Extra Records. - - good-multi-none-extra_3.lzma has an Extra Record that has empty Data. - - good-multi-none-header_1.lzma has very minimal Header Metadata Block - with only the Metadata Flags field. - - good-multi-none-header_2.lzma has all information in both Header and - Footer Metadata Blocks. The Size of Header Metadata Block has wrong - value in Header Metadata Block, but this value must be ignored by - the decoder in case of Header Metadata Block. - - good-multi-none-header_3.lzma has Index only in the Header Metadata - Block. Footer Metadata Block contains only Size of Header Metadata - Block and Total Size. - - good-multi-none-block_1.lzma has Index in Header Metadata Block. The - Compressed Size and Uncompressed Size fields are present in the Data - Blocks. There is some Footer Padding between the Blocks. - - good-multi-none-block_2.lzma has Index in Header Metadata Block. The - Uncompressed Size field is present in Data Blocks and no EOPM is used. + good-1-3delta-lzma2.lzma has three Delta filters and LZMA2. -2.2. Bad Files +2.2. Unsupported Files - bad-single-none-truncated.lzma is good-single-none.lzma without the - last byte of the file. + unsupported-check.lzma uses Check ID 0x02 which isn't supported by + the current version of the file format. It is implementation-defined + how this file handled (it may reject it, or decode it possibly with + a warning). - bad-cat-single-none-pad_garbage_1.lzma is good-cat-single-none-pad.lzma - with 0xFE appended to the end of the file. 0xFE doesn't begin .lzma - or LZMA_Alone format file. + unsupported-block_header.lzma has a non-nul byte in Header Padding, + which may indicate presence of a new unsupported field. - bad-cat-single-none-pad_garbage_2.lzma is good-cat-single-none-pad.lzma - with 0xFF appended to the end of the file. 0xFF begins .lzma format - file, thus the decoder has to detect that the file is incomplete. + unsupported-filter_flags-1.lzma has unsupported Filter ID 0x7F. - bad-cat-single-none-pad_garbage_3.lzma is good-cat-single-none-pad.lzma - with 0x5D appended to the end of the file. 0x5D is the most common - first byte of LZMA_Alone format file. + unsupported-filter_flags-2.lzma specifies only Delta filter in the + List of Filter Flags, but Delta isn't allowed as the last filter in + the chain. It could be a little more correct to detect this file as + corrupt instead of unsupported, but saying it is unsupported is + simpler in case of liblzma. - bad-single-none-footer_filter_flags.lzma has different Stream Flags - in Stream Footer than in Stream Header. - - bad-single-none-too_long_vli.lzma has 10-byte variable-length integer. - - bad-single-none-empty.lzma is like good-single-none-empty_3.lzma but - with non-zero value in the Compressed Size field. - - bad-single-data_after_eopm_1.lzma has LZMA+Subblock, where the Subblock - filter gives one byte of data to LZMA after LZMA has detected EOPM. - - bad-single-data_after_eopm_2.lzma is like - bad-single-data_after_eopm_1.lzma but Subblock gives 256 MiB of data - to LZMA after LZMA has detected EOPM. - - bad-single-subblock_subblock.lzma has Subblock+Subblock, where the - Subblock decoder is given End of Input in the middle of a Subblock. - - bad-single-subblock-padding_loop.lzma contains huge amount of - consecutive Padding bytes, which isn't allowed by the Subblock filter - format. If it were allowed, this file would hang the decoder for very - long time (weeks to years). - - bad-single-subblock1023-slow.lzma is similar to - malicious-single-subblock31-slow.lzma except that this uses 1023 bytes - of Padding in every place instead of 31 bytes. The Subblock filter - format specification allows only 31-byte Padings, thus this file must - get detected as bad without producing any output. Allowing larger - Padding than 31 bytes was considered (so this test file was created), - but it seemed to be a bad idea since it would increase worst-case CPU - usage. - - bad-single-lzma-flush_beginning.lzma has flush marker in the beginning - of the LZMA data. - - bad-single-lzma-flush_twice.lzma has two flush markers with no data - between them. - - bad-multi-none-1.lzma has data after the last field in the Metadata - Block and the `Extra is present' flag is not set. - - bad-multi-none-2.lzma has wrong Total Size in Footer Metadata Block. - - bad-multi-none-3.lzma has wrong Uncompressed Size in Footer Metadata - Block. - - bad-multi-none-index_1.lzma has wrong value in the Number of Data - Blocks field. - - bad-multi-none-index_2.lzma has too short Metadata to contain all - the Index Records. - - bad-multi-none-index_3.lzma has wrong value in Total Size field in - the Index. - - bad-multi-none-index_4.lzma has wrong value in Uncompressed Size field - in the Index. - - bad-multi-none-extra_1.lzma has incomplete Extra Record at the end of - the Metadata Block. - - bad-multi-none-extra_2.lzma has incomplete variable-length integer as - Extra Record ID. - - bad-multi-none-extra_3.lzma has incomplete Extra Record at the end of - the Metadata Block. - - bad-multi-none-header_1.lzma has empty Header Metadata Block (even - the Metadata Flags field is not present). - - bad-multi-none-header_2.lzma has Index in the Header Metadata Block, - which describes only one Data Block, while the Stream actually has - two Data Blocks. A sophisticated decoder should give an error when - it detects the second Data Block; all Multi-Block decoders must - detect the file as corrupt at some point. - - bad-multi-none-header_3.lzma contains too small Total Size in Header - Metadata Block. A sophisticated decoder should abort decoding before - the second Data Block, preferably before the first Data Block has - been finished; all Multi-Block decoders must detect the file as - corrupt at some point. - - bad-multi-none-header_4.lzma is like bad-multi-none-header_3.lzma but - with too small Uncompressed Size. - - bad-multi-none-header_5.lzma has Index in the Header Metadata Block, - but the Total Size field is missing from the Footer Metadata Block. - - bad-multi-none-header_6.lzma has both Index and Total Size in Header - Metadata Block, but Total Size doesn't match the Index. A sophisticated - decoder should abort before decoding any Data Blocks; all Multi-Block - decoders must detect the file as corrupt at some point. - - bad-multi-none-header_7.lzma has zero as the Size of Header Metadata - Block in the Header Metadata Block. - - bad-multi-none-block_1.lzma has wrong Uncompressed Size in the first - Data Block. A sophisticated decoder should detect this error before - producing any output, because it can see that the Uncompressed Size - doesn't match with the Index in Header Metadata Block; all Multi-Block - decoders must detect the file as corrupt at some point. - - bad-multi-none-block_2.lzma has too big Compressed Size in the first - Data Block. A sophisticated decoder may be able to detect the file as - corrupt before producing any output, because Comrpessed Size + size - of Block Header exceed the Total Size stored in Index in Header - Metadata Block. A sophisticated decoder should be able to detect the - error before the end of the first Data Block; all Multi-Block decoders - must detect the file as corrupt at some point. - - bad-multi-none-block_3.lzma has only the Compressed Size field in the - Block Header of the second Data Block and EOPM isn't used. + unsupported-filter_flags-3.lzma specifies two LZMA2 filters in the + List of Filter Flags. LZMA2 is allowed only as the last filter in the + chain. It could be a little more correct to detect this file as + corrupt instead of unsupported, but saying it is unsupported is + simpler in case of liblzma. -2.3. Malicious Files +2.3. Bad Files - malicious-single-subblock31-slow.lzma requires quite a bit of CPU time - per decoded byte. It contains LZMA compressed Subblock filter data that - has as much Padding as the specification allows. LZMA is also used as - a Subfilter, to further slowdown the decoder. Every Subfilter instance - produces only one byte of output. If you can create a file that wastes - notably more CPU cycles than this file, please contact Lasse Collin. + bad-0pad-empty.lzma has one Stream with no Blocks followed by + five-byte Stream Padding. Stream Padding must be a multiple of four + bytes, thus this file is corrupt. - malicious-single-subblock-256MiB.lzma is a tiny file that produces - 256 MiB of output. It uses Subblock filter's run-length encoding - to achieve this. + bad-0catpad-empty.lzma has two zero-Block Streams concatenated with + five-byte Stream Padding between the Streams. - malicious-single-subblock-64PiB.lzma is a tiny file that produces - 64 PiB of output (if you have patience to wait). This is done by - chaining two Subblock filters and using their run-length encoders. + bad-0cat-alone.lzma is good-0-empty.lzma concatenated with an empty + LZMA_Alone file. - malicious-multi-metadata-64PiB.lzma is like - malicious-single-subblock-64PiB.lzma but the huge amount of output - is in a Metadata Block. Trying to decode this file may take years - unless the decoder catches that the Metadata has unreasonable size. + bad-0-empty-truncated.lzma is good-0-empty.lzma without the last byte + of the file. + + bad-0-nonempty_index.lzma has no Blocks but Index claims that there is + one Block. + + bad-0-backward_size.lzma has wrong Backward Size in Stream Footer. + + bad-1-stream_flags-1.lzma has different Stream Flags in Stream Header + and Stream Footer. + + bad-1-stream_flags-2.lzma has wrong CRC32 in Stream Header. + + bad-1-stream_flags-3.lzma has wrong CRC32 in Stream Footer. + + bad-1-vli-1.lzma has two-byte variable-length integer in the + Uncompressed Size field in Block Header while one-byte would be enough + for that value. It's important that the file gets rejected due to too + big integer encoding instead of due to Uncompressed Size not matching + the value stored in the Block Header. That is, the decoder must not + try to decode the Compressed Data field. + + bad-1-vli-2.lzma has ten-byte variable-length integer as Uncompressed + Size in Block Header. It's important that the file gets rejected due + to too big integer encoding instead of due to Uncompressed Size not + matching the value stored in the Block Header. That is, the decoder + must not try to decode the Compressed Data field. + + bad-1-block_header-1.lzma has Block Header that ends in the middle of + the Filter Flags field. + + bad-1-block_header-2.lzma has Block Header that has Compressed Size and + Uncompressed Size but no List of Filter Flags field. + + bad-1-block_header-3.lzma has wrong CRC32 in Block Header. + + bad-1-block_header-4.lzma has too big Compressed Size (2^63 bytes while + maximum is 2^63 - 4 bytes) in Block Header. It's important that the + file gets rejected due to invalid Compressed Size value; the decoder + must not try decoding the Compressed Data field. + + bad-2-index-1.lzma has wrong Total Sizes in Index. + + bad-2-index-2.lzma has wrong Uncompressed Sizes in Index. + + bad-2-index-3.lzma has non-nul byte in Index Padding. + + bad-2-index-4.lzma wrong CRC32 in Index. + + bad-2-compressed_data_padding.lzma has non-nul byte in the padding of + the Compressed Data field of the first Block. + + bad-1-check-crc32.lzma has wrong Check (CRC32). + + bad-1-check-crc64.lzma has wrong Check (CRC64). + + bad-1-check-sha256.lzma has wrong Check (SHA-256). + + bad-1-lzma2-1.lzma has LZMA2 stream whose first chunk (uncompressed) + doesn't reset the dictionary. + + bad-1-lzma2-2.lzma has two LZMA2 chunks, of which the second chunk + indicates dictionary reset, but the LZMA compressed data tries to + repeat data from the previous chunk. + + bad-1-lzma2-3.lzma sets new invalid properties (lc=8, lp=0, pb=0) in + the middle of Block. + + bad-1-lzma2-4.lzma has two LZMA2 chunks, of which the first is + uncompressed and the second is LZMA. The first chunk resets dictionary + as it should, but the second chunk tries to reset state without + specifying properties for LZMA. + + bad-1-lzma2-5.lzma is like bad-1-lzma2-4.lzma but doesn't try to reset + anything in the header of the second chunk. + + bad-1-lzma2-6.lzma has reserved LZMA2 control byte value (0x03). + + bad-1-lzma2-7.lzma has EOPM at LZMA level. diff --git a/tests/files/bad-0-backward_size.lzma b/tests/files/bad-0-backward_size.lzma new file mode 100644 index 0000000000000000000000000000000000000000..07c4695af5bb815198ebbf597349bafa6cb51655 GIT binary patch literal 32 kcmezG6Xom3z`&TPbkB?d2xRWND4I?@dJM#7V2q3c0F=52IRF3v literal 0 HcmV?d00001 diff --git a/tests/files/bad-0-empty-truncated.lzma b/tests/files/bad-0-empty-truncated.lzma new file mode 100644 index 0000000000000000000000000000000000000000..15b6fe3650a9344012bb507022b72418ae40b3dc GIT binary patch literal 31 jcmezG6Xom3z`&TPbkB?d2xRWNC{A#i$;$}hFh&9Zh6e`+ literal 0 HcmV?d00001 diff --git a/tests/files/bad-0-nonempty_index.lzma b/tests/files/bad-0-nonempty_index.lzma new file mode 100644 index 0000000000000000000000000000000000000000..2a612c61904a12fa93fc591710ca7c78260a5909 GIT binary patch literal 32 lcmezG6Xom3z`&TPbkB@|k%2*bYn#FZr$2sFUm<#q^21P~=1_p*3{K~?N46GihIXU@UObl${`9(P?id+ou^*RLS b+D|*pz{tsazvyef{I<`vj6n4aERj(F&#Dws literal 0 HcmV?d00001 diff --git a/tests/files/bad-1-check-sha256.lzma b/tests/files/bad-1-check-sha256.lzma new file mode 100644 index 0000000000000000000000000000000000000000..e47609cdfc1ba610dd2d36041c0058560e6590bf GIT binary patch literal 96 zcmezG6Xom3z`*tJH_t*Q21P~=1_p*3{K~?N46GihIXU@UObl${`9(P?id+nRk*3cd z8fToH*72|3W5Lv*$>~{4qDc`tbuGQ?AKGtlXJF*#jl8=4ro?RTS&Tq)7`P&%0F(3{ A^Z)<= literal 0 HcmV?d00001 diff --git a/tests/files/bad-1-lzma2-1.lzma b/tests/files/bad-1-lzma2-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..0296e5ff76bb69e1a6b9eb915200a181a5367173 GIT binary patch literal 64 zcmezG6Xom3z`*cd=%ynRgCZjbkbQ$+S(u4|)gv`0C!dRnfh|10C?`dci-CcWoi{MP ROpvW;UKJxy9RovT6ad}h4R`vH<%0_2e@^`p>hS)*d z1vSlsrF;^l1>2ClFBH>pa+auyeS}tVjcOv8tDZfOfGx*f3=&e)KL-eV9He*i6Sx<^ z?va5xvxo5i0JCaRLG3QE@?@8!nx12!t{HRDzhLwWI`Eq@>)$`vuccLzMg>?R36H2?2n(a>KD@LR+sC7#z!udM zry}St=(Nyx7sZR@^1Ex+@0x)$P95CrHs(ZfzbUmm1uvH<%0_2e@^`p>hS)*d z1vSlsrF;^l1>2ClFBH>pa+auyeS}tVjcOv8tDZfOfGx*f3=&e)KL-eV9He*i6Sx<^ z?va5xvxo5i0JCaRLG3QE@?@8!nx12!t{HRDzhLwWI`Eq@>)$`vuccLzMg>?R36H2?2n(a>KD@LR+sC7#z!udM zry}St=(Nyx7sZR@^1Ex+@0x)$P95CrHs(ZfzbUmm1ufCOZ6IWCY;STP zb7^!SVQpn}EFfcVZgXX0bY*mPav))3X>e(CX=7<_XCMlo0h0kF05Y2iY*g1Sm2CO< zH`N^CMgoF$`$=|T@N9l2BVfO zx0jsod;a1Qj4e^{+je(-qH7ud^_P!g>Dw^7v~WEZcmsDji+a_#_(~@w^^U3} z6s9RCYcF%)p>ec(q<~OB%J18>mHoTHjE-QJ+sl6;?Z6Om8hy*n3DDX9ZGgaMeLu#O z&s*J55z))kAlEB|PN+q@LUfCOZ6IWCY;STP zb7^!SVQpn}EFfcVZgXX0bY*mPav))3X>e(CX=7<_XCMlI0h0kF05Y2iY*g1Sm2CO< zH`N^CMgoF$`$=|T@N9l2BVfO zx0jsod;a1Qj4e^{+je(-qH7ud^_P!g>Dw^7v~WEZcmsDji+a_#_(~@w^^U3} z6s9RCYcF%)p>ec(q<~OB%J18>mHoTHjE-QJ+sl6;?Z6Om8hy*n3DDX9ZGgaMeLu#O z&s*J55z))kAlEB|PN+q@LUvH<%0_2e@^`p>hS)*d z1vSlsrF;^l1>2ClFBH>p5`0a~5vk*40WO9~hLS=uQj{(bU$>5r{q-BhCIRrXySr}1 zdY0ke$TY+CPlE<1QGsOnfNF6^(Bh`n4P-ITiZEjPb2Um31;ozpwEokDHsI-J!0@%9 z9#U9T-9T3OoAX)m*5G;uRy;ku6@j}t>xB*ysa{hlP)@$!-3+>l!|)^*_VDD)+r1|%x2wUGVz9q}m8b{;f6>+}ca0ow&ZXJfUDM`^ zwdZ+3rqiB%C%j3Sg$wZxMIQSq_ty@5&+zn8NEmw&OYym>f$1j-H^WuEv{l=!x8Hf6 zMzxhod3zDrEfeDi)uDu-K|5{{(>D2fRa6k)2H{NCS4q8QX?+`@;e%)h9iaU*hMo<< zARA?M-8)}%%EE*H|LAH_*Z=^s2Sn)?GfRX30bI!g0000LIjK&u#Ao{g000001X)_j Cqrjs8 literal 0 HcmV?d00001 diff --git a/tests/files/bad-1-stream_flags-1.lzma b/tests/files/bad-1-stream_flags-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..cd0d6f43c97ada3f487335b15844ebab9e0ba0ba GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^5L6MPzfq~%$zp^kR1FJ`BPEI}-69Zd#eo;<}A{T@6;zgni Xj2yi4oZn5;5}q)H5vZ1dDKZKG7y1v2 literal 0 HcmV?d00001 diff --git a/tests/files/bad-1-stream_flags-2.lzma b/tests/files/bad-1-stream_flags-2.lzma new file mode 100644 index 0000000000000000000000000000000000000000..63dec85bf01edac4d7d0211627f40688348c6e5c GIT binary patch literal 68 zcmezG6Xom3z`&TPbgztwL6MPzfq~%$zp^kR1FJ`BPEI}-69Zd#eo;<}A{T@6;zgni Xj2yi4oZn5G;53t$5vZ1dF)|7OD4!4E literal 0 HcmV?d00001 diff --git a/tests/files/bad-1-stream_flags-3.lzma b/tests/files/bad-1-stream_flags-3.lzma new file mode 100644 index 0000000000000000000000000000000000000000..3b306ff74e5e08949f2c83486e93be4b95c94328 GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^5L6MPzfq~%$zp^kR1FJ`BPEI}-69Zd#eo;<}A{T@6;zgni Xj2yi4oZn5G;537m5vZ1dF)|7O8dVRV literal 0 HcmV?d00001 diff --git a/tests/files/bad-1-vli-1.lzma b/tests/files/bad-1-vli-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..7bcdd568fc157074ba5aa9bd7df41e7c34c06371 GIT binary patch literal 72 zcmezG6Xom3z`&TPbkB^rp_c&$HZBg#U}WI&NX^N~=L*j+%1Ke=0?9cqUL?xE$jN)Z R=xhH3r$2sFUm<# h^6te`!3<1nY;5cd3`-_9blVy5b~6FZU|@`l0sv!U5}5!1 literal 0 HcmV?d00001 diff --git a/tests/files/bad-2-index-1.lzma b/tests/files/bad-2-index-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..cc6ba6dd2bf82d7d5837394c2ba7f1dc99a93dfd GIT binary patch literal 92 zcmezG6Xom3z`&TPbkB^5L6MPzfq~%$zp^kR1FJ`BPEI}-kS}IB%@9?NEj+&{Cq)q` gdH3R}UglDUTYFo literal 0 HcmV?d00001 diff --git a/tests/files/bad-2-index-3.lzma b/tests/files/bad-2-index-3.lzma new file mode 100644 index 0000000000000000000000000000000000000000..de27f55f1a12bb79087c19370aef0dea5f415a51 GIT binary patch literal 92 zcmezG6Xom3z`&TPbkB^5L6MPzfq~%$zp^kR1FJ`BPEI}-kS}IB%@9?NEj+&{Cq)q` gdH3R}U5>8ik4I`wPQF5Teo;;e7e7N}6a&M50`mU=vDXwW diff --git a/tests/files/bad-cat-single-none-pad_garbage_2.lzma b/tests/files/bad-cat-single-none-pad_garbage_2.lzma deleted file mode 100644 index 26595aae3a073937823a1903043bc9c43674d9ba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65 qcmezG6Xom3z|i}i>5>8ik4I`wPQF5Teo;;e7e7N}6a&M5Lh=By*c2`R diff --git a/tests/files/bad-cat-single-none-pad_garbage_3.lzma b/tests/files/bad-cat-single-none-pad_garbage_3.lzma deleted file mode 100644 index 73c8744992c90e5e9b64011d6425e286ecd6074e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65 rcmezG6Xom3z|i}i>5>8ik4I`wPQF5Teo;;e7e7N}6a&M50`jo{u{INo diff --git a/tests/files/bad-multi-none-1.lzma b/tests/files/bad-multi-none-1.lzma deleted file mode 100644 index 208e510052c29b56e37fe5f364c5920c4850680a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 54 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tQY#XE)PS7u{Gyx`MJ|B_44nMT(!5N( Ld~EDI0+CSw)a?w| diff --git a/tests/files/bad-multi-none-header_2.lzma b/tests/files/bad-multi-none-header_2.lzma deleted file mode 100644 index e7e66a7ce7f0db1f648f2e16f0fbacbf7396f2e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 61 zcmezG6Xom3An@}4*7FM(SU4DY`8XI<85kHmQgd?hxdb>E)PS7u{Gyx`MJ|B_44nMz P(!5N(d~EDI0+CSw^>hsc diff --git a/tests/files/bad-multi-none-header_3.lzma b/tests/files/bad-multi-none-header_3.lzma deleted file mode 100644 index 37648e3e937e79029ae569731ec2c3b7129c839e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59 zcmezG6Xom3An@}4*7FM(n3z~O7*rV;7(7yQa`L$ZI2hD`obddjoD@Yafdvel{H)Tv NOuT$->^uUIQ2^sN4C4R* diff --git a/tests/files/bad-multi-none-header_4.lzma b/tests/files/bad-multi-none-header_4.lzma deleted file mode 100644 index 33cf425d0b104bd7c9d5a519b46d7d1afa1d0fb9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59 zcmezG6Xom3An@}4*7FM(m{?dj7*rV;7(7yQa`L$ZI2hD`obddjoD@Yafdvel{H)Tv NOuT$->^uUIQ2^tS4CMd- diff --git a/tests/files/bad-multi-none-header_5.lzma b/tests/files/bad-multi-none-header_5.lzma deleted file mode 100644 index 313661f9312d129e8b5333391b957c753da10505..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 58 zcmezG6Xom3An@}4*7FM(IQXP_nRxlw*f|(f85kHmQgd?hxdb>E)PS7u{Gyx`MJ|B_ M3`~sNtOAiy0K)?e+5i9m diff --git a/tests/files/bad-multi-none-header_6.lzma b/tests/files/bad-multi-none-header_6.lzma deleted file mode 100644 index 4055256b5f10d35cc1e96867a4551ec216ea98c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59 zcmezG6Xom3An@}4*7FM(IQV3EnRxlw*f|(f85kHmQgd?hxdb>E)PS7u{Gyx`MJ|B_ N49v{j(rf~eQ2@{44Eg{7 diff --git a/tests/files/bad-multi-none-header_7.lzma b/tests/files/bad-multi-none-header_7.lzma deleted file mode 100644 index 66b2d4b5728996746d350bd362c6801f5c59fabd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59 zcmezG6Xom3An@}4*7FM(m>3y27*rV;7(7yQa`L$ZI2hD`obddjoD@Yafdvel{H)Tv NOuT$->^uUIQ2^pD4BY?# diff --git a/tests/files/bad-multi-none-index_1.lzma b/tests/files/bad-multi-none-index_1.lzma deleted file mode 100644 index b1bd0ceebeb385fc685b59927c780c69d7d158fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tPk?W?nuvc20rF FC;)?~3;_TD diff --git a/tests/files/bad-multi-none-index_2.lzma b/tests/files/bad-multi-none-index_2.lzma deleted file mode 100644 index 59d92c6cf51a3d31ee3413de5b5c405fde3a7f28..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 49 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tOZiCSE>vfygKT DZ14;3 diff --git a/tests/files/bad-multi-none-index_3.lzma b/tests/files/bad-multi-none-index_3.lzma deleted file mode 100644 index 5b94972a109bb177ddb600fedbed1052ad0e1a40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tPk?CSHCvc20rF FC;)?|3;_TD diff --git a/tests/files/bad-multi-none-index_4.lzma b/tests/files/bad-multi-none-index_4.lzma deleted file mode 100644 index 880878a5e1fe35bd0647fc12ab91d36340a42bef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tPk?CSE=^Hco-a FC;)?+3;zHB diff --git a/tests/files/bad-single-data_after_eopm_1.lzma b/tests/files/bad-single-data_after_eopm_1.lzma deleted file mode 100644 index 3c1e90f2e23648f6fd9253b8ba706b827c976159..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55 zcmezG6Xom3z|i}i=@J)%L##Zb8Y2S(gNmeQ+w-eWABDZmdivtxp`!o)k4$1 L2=HhyL`DGsHd7Pp diff --git a/tests/files/bad-single-data_after_eopm_2.lzma b/tests/files/bad-single-data_after_eopm_2.lzma deleted file mode 100644 index 008e3941cee928b5069c89c8f81b0a3fbe2dcef4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 56 zcmezG6Xom3z|i}i=@J)%L##Zb8Y2S(gNmeQ+w-eWABDZmdivtxp`!o)k4$1b!b MC?~+9$q*R@0AH#W2><{9 diff --git a/tests/files/bad-single-lzma-flush_beginning.lzma b/tests/files/bad-single-lzma-flush_beginning.lzma deleted file mode 100644 index 1952c04325a31843e9da3f7bb471d5d96a79ff06..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53 zcmezG6Xom3z|i}i=@KV{L##YQ`~Tnn{~urg0u@Qm8LNBVC>&tj-8I`I>hAx4E6oyk I)EFY807)zsE&u=k diff --git a/tests/files/bad-single-lzma-flush_twice.lzma b/tests/files/bad-single-lzma-flush_twice.lzma deleted file mode 100644 index d71dac0073b0035183bdb97b64a2be29c7fbba5c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63 zcmV-F0KotMOj=Dr0052e0@4WpKwTdIBpFGVZv_=6Ck=LZW|d3p|Ngxo004-_3;+L} Vg@eEVgnfK=|D?oo`V24tSz6|X8MXib diff --git a/tests/files/bad-single-none-empty.lzma b/tests/files/bad-single-none-empty.lzma deleted file mode 100644 index 3007e9a9acecaaa24ce82f77c87fc504fb4aada8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19 acmezG6Xom3z|i}i>5>5hBLfRVWE22F>IG~7 diff --git a/tests/files/bad-single-none-footer_filter_flags.lzma b/tests/files/bad-single-none-footer_filter_flags.lzma deleted file mode 100644 index 1257ce389df2a4b1b48a133c20b0aeb8fe24442f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30 lcmezG6Xom3z|i}i>5>8ik4I`wPQF5Teo;;e7e8ZU6ab5>9N4+9i5c%5>8ik4I`wPQF5Teo;;e7e7NJ0GN&mJ^%m! diff --git a/tests/files/bad-single-subblock-padding_loop.lzma b/tests/files/bad-single-subblock-padding_loop.lzma deleted file mode 100644 index 7f0f5f782ae3fbb07d01c30c0a9b19a42d6696e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43 vcmezG6Xom3z|i}i=@K^sBO{~z|NsA4K?DPXDgy(90uT!b2r$SnL`DGsM+FR1 diff --git a/tests/files/bad-single-subblock1023-slow.lzma b/tests/files/bad-single-subblock1023-slow.lzma deleted file mode 100644 index 842defee2278d4627b3a1fd35e402ce988fb83be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7886 zcmezG6Xom3z|i}i=@J(MqeHAb14I7b|Nj?nuYDONT)(6Hhqaksxc5oLEw3uVG#WV6 zE7|jQ*tL9nv7IyfmkQUG2fveKrmb_>RlZ|`v90IggZUd@`0kLrKB2d?q{BJTER#hu zTHWHGqQ-ikn-gD_q`X{o=H^*z|6{L%3w-=nOqjuPWTMetgO|0|0FJ4n0atX#B$)6cuX zwm&peIISB_YvtVEYW?BJUcLBL{C3`HP7{Rr{tHb0QCJcAgZu3NsTywlEC1Bo`#sA* zFYG{n>*wai1F54{j|TW?G8@fYqeaSSK{{G}j8>YXO@+}0(rB}Av>iU$`xxzWjrKH0 z``4qR3Zr8wqa#S8<7T6yg`>l>ql40;L-C^%5TnyBqmxaeQ*5IXi`1S@l|P)Yp_{d2 S{{KJy2?`Ah_U&bei~;}x$d6(G diff --git a/tests/files/bad-single-subblock_subblock.lzma b/tests/files/bad-single-subblock_subblock.lzma deleted file mode 100644 index 4b6987d8d676a4b4f0d0d9eb1fe9aea04bd0ae51..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26 hcmezG6Xom3z|i}i=@J(MBclQX1B0Rf12;ot6aZs#1)l%_ diff --git a/tests/files/good-0-empty.lzma b/tests/files/good-0-empty.lzma new file mode 100644 index 0000000000000000000000000000000000000000..3256fe1568d3242943e33dfbb32708d6f42d3477 GIT binary patch literal 32 kcmezG6Xom3z`&TPbkB?d2xRWNC{A#i$;$}hFh)iJ0F7t|VE_OC literal 0 HcmV?d00001 diff --git a/tests/files/good-0cat-empty.lzma b/tests/files/good-0cat-empty.lzma new file mode 100644 index 0000000000000000000000000000000000000000..ef8e106a49fce4ee98a579b15c66e142fcd85fa6 GIT binary patch literal 64 ncmezG6Xom3z`&TPbkB?d2xRWNC{A#i$;$}hFh)lGCm;_12JQ~Q literal 0 HcmV?d00001 diff --git a/tests/files/good-0catpad-empty.lzma b/tests/files/good-0catpad-empty.lzma new file mode 100644 index 0000000000000000000000000000000000000000..7e5a81d4c96d06b1a9f8159f2b1c24da08a79f70 GIT binary patch literal 68 pcmezG6Xom3z`&TPbkB?d2xRWNC{A#i$;$}hFh)j!sQ(1i001O%4#5Bb literal 0 HcmV?d00001 diff --git a/tests/files/good-0pad-empty.lzma b/tests/files/good-0pad-empty.lzma new file mode 100644 index 0000000000000000000000000000000000000000..3bbc241b66d4b6c40341b3b54d3cb6d69a8b1c22 GIT binary patch literal 36 lcmezG6Xom3z`&TPbkB?d2xRWNC{A#i$;$}hFh)j!C;+PL2VnpJ literal 0 HcmV?d00001 diff --git a/tests/files/good-1-3delta-lzma2.lzma b/tests/files/good-1-3delta-lzma2.lzma new file mode 100644 index 0000000000000000000000000000000000000000..2724ed4b7e8c977153034a96f960601a4438f339 GIT binary patch literal 528 zcmV+r0`L9*Oj=Dr000E$*0e?h0|Nm70|5a80RkZb2mk=Q6=lW&0mw`vw}j#eaZmcX zq4!Z^2&}I1SV9eE?x+FHU@nMQ*KXTACIAKT`>!--7r>uC`TPwI1El7!X|ZCSN0&px zFGr)J^#oeLn>_){TRQr)!c3h{SN*%MQ~w74^`}u@q?}{4v|{&1pipkF;|o&5r#HY< zNddXE|2tmwZ8yUJ^Q+|Uc32$NxO8Zp-%n0rcCj_ zRA%p_tx+K35AF;Cq+>h(sK-_Uuiddw!DshN@3dM+2De|MO{(J;WxJ>_2DMUPr}Xg` z75PD}w>lNL>YrxEQPE);?FhX8TXFGkdS{LWrd678{+yx&U10mCw(?F7M=dFx>EAL@10)4R_I;( z1fpA0{kN)Hu{Krk>y9EkKRQZh;&ACC{yaY4=000282Sn)?GfRX30eZ;;0000! S-&~)u#Ao{g000001X)@tqx!=D literal 0 HcmV?d00001 diff --git a/tests/files/good-1-block_header-1.lzma b/tests/files/good-1-block_header-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..0eeaf464c2dca66ba450869efa9f50a43679f8ca GIT binary patch literal 72 zcmezG6Xom3z`&TPbkB_W01K}oBL@QrT%NK|i;;oHBQ+-{pDR4SC?`dc3ncEmc#$Xr VBPZ|uqObiEoM!Sef&>{OqX3^#5)}Xd literal 0 HcmV?d00001 diff --git a/tests/files/good-1-block_header-2.lzma b/tests/files/good-1-block_header-2.lzma new file mode 100644 index 0000000000000000000000000000000000000000..7afa25fa3454142a629369b84f0f34e0602da5eb GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^*fkly#gMmTl{^U)J3_KpGIXU@U;rT^5DT-VSK;XQ1kthQr V2k$)Rche>~&E#bS2{J}T0RUj55d8oE literal 0 HcmV?d00001 diff --git a/tests/files/good-1-block_header-3.lzma b/tests/files/good-1-block_header-3.lzma new file mode 100644 index 0000000000000000000000000000000000000000..85a1f8e8edfb2bf7e3fb298a945cf91a9cb4bb48 GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^bfme}{gMlGXurZR6fyW~?CnujPJijO>MUjgE2%Hx$5@lfI U;GO6EZrTK=nY@f3LB_}^07GUFOaK4? literal 0 HcmV?d00001 diff --git a/tests/files/good-1-check-crc32.lzma b/tests/files/good-1-check-crc32.lzma new file mode 100644 index 0000000000000000000000000000000000000000..b586af1466e0857d76a1fb0b897b852401749e29 GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^5L6MPzfq~%$zp^kR1FJ`BPEI}-69Zd#eo;<}A{T@6;zgni Xj2yi4oZn5G;53t$5vZ1dF)|7O8eb2g literal 0 HcmV?d00001 diff --git a/tests/files/good-1-check-crc64.lzma b/tests/files/good-1-check-crc64.lzma new file mode 100644 index 0000000000000000000000000000000000000000..9b8cc1f65230709e9fe5833d66c717cefd386d10 GIT binary patch literal 72 zcmezG6Xom3z`*kC+7>q^21P~=1_p*3{K~?N46GihIXU@UObl${`9(P?id+ou^*RLS b+D|*hz{tsazvyef{I<`vj6n4aERj(F&y^HW literal 0 HcmV?d00001 diff --git a/tests/files/good-1-check-none.lzma b/tests/files/good-1-check-none.lzma new file mode 100644 index 0000000000000000000000000000000000000000..7b7ccd5f98e0468b72c05cfdb227e4009243bd11 GIT binary patch literal 64 zcmezG6Xom3z`*cd=%ynRgCZjbkbQ$+S(uT5)gv`0C!dRnfh|10C?`dci-CcWoi{MP ROpvW;UKJxy9RovT6ad}34R-(l literal 0 HcmV?d00001 diff --git a/tests/files/good-1-check-sha256.lzma b/tests/files/good-1-check-sha256.lzma new file mode 100644 index 0000000000000000000000000000000000000000..0919afdb71d5b152ec7d27ebc8f9789df61f8411 GIT binary patch literal 96 zcmezG6Xom3z`*tJH_t*Q21P~=1_p*3{K~?N46GihIXU@UObl${`9(P?id+nRk*3cd z8fToH*72|3W5Lv*$>~{4qDc`tbuGQ?AKGtlV_@Xxjl8=4ro?RTS&Tq)7`P&%0F&Py A^8f$< literal 0 HcmV?d00001 diff --git a/tests/files/good-1-delta-lzma2.tiff.lzma b/tests/files/good-1-delta-lzma2.tiff.lzma new file mode 100644 index 0000000000000000000000000000000000000000..d52b70d436aa11c7b67a6dcc4479b2fb2cbe82cb GIT binary patch literal 51312 zcmV(nK=Qx;Oj=Dr000E$*0e?f0RsU7ApsD_O|Gr(Epf;%T>vEDrEb4(Zly)j0n5Cy zv{sB`NaRg1FVvR`1eym}Bqe1D&aJsCmF<&5rrIN$l);~&P2Qw0_UGwqI@}ytabE<_ z@45NkfK*v&DqdqAuVZ&vR61D5XjiGC=4_BL(YhgT=Q0?Z!q2+g*x!tCWs z2YtBT#$Fg5;oG{TcLjd=>CVQp?BxN-t7GYkp$zE>)>{K)c?hebkj{jV%@)pkMrA=^uARoT`n9ZFQ#YxHs!wx_Z9G5E7uH`O@?Xp)$&kNm$ZnjzddT(Ce!2)lOdOwp`7P1o zX2y1D8E3^ZK_(pxml{c2R$Wy=$OlG~CAO2y%icd!zjzk=MaLo|7N1xpyxg13%@rMTW$LArLQ^l4Q=XRofF#C1x++e1; zA%?U8#R-{W-3pVaF{oB#F|`VH$(Qrj*9FyV5GBY>V|TK)Zajf6^(f+t66?99rnX*S zfCDdAVKfc;K5yC-CJdWv5J#G00?*6%qac4KAMXMf$v*xn|?TcD#o+&K#Uy( z(S?;{kcojFW19B2X$So$i={r|2F*!ZZ`m0!tU;zvNryH5U(F`sH^uaqvwYvuM&Sj1 z4#v&&MKQ`AS;3;o)t!T~&qiQb2$25BrmTJ80rGa_0z0nXI6}3f@D= z3y{s5m`aWQk)D5c-e8V0{g-^EiCp+l z`)r_gAFI+ggws1qg=i1@_m|0P1}EwJFn6oYkYuIU4h{zO^dXOBhB~_Cw?)etTW5hI z!#k=s^I<;yEt*RfsCoyBaCI9~c4&`m-@8S|q?PBtsz7XTlD8Zq&0xlq*&owepsYlR zwd9lKuQ@wVMb_VoGsqrr1Sv@{Qh;f2v(j3Pp8PsS_`0hJ2 ze}s@W1ir~Pvuit`VI$ha-bQ*Pypt73#C&*Nc4*|()|RgK6z85i{F_z&vwV__l2Oo+ z&<@#kwEE`8Pu-<$HXAWqQ3Q;$PiwIIQpys^{P$-lW}Xo#suM5*Q`$#o$BE}hZ(4C8M5eCpgLc&bILW2L6IHwE zuiru9kPy&{X9f#2PCkyc--5DEv|tPuIhp#>3bscRmT34VYq2|iU3@?{1X9;zHBq`l zJ_6A*k*IAZ?lC_q%7$0L{UK#|I##W~hk#z*O^brj`@xB3d6cA5LG0P%Ar|ihNPr}# zywSYp{Fnjr&uSh!#d{uETolOj<$({ZLvxBfIGPv9m?Mz>OG-Nq6eq=|+?m_k4l6ascmV94w|v&8|9-H+$pc+$G)0x|OcS3iy7i(#UulSXr6$t7X{+2T z(=f3@h&^Im$NF#p^=A*v-#rU^WTtDW@w?AOw(Aa6Y16q*B++Sgb#S(rS+%}*jnmILjsa#Wobc2Te)c2p$9k!OZ>JOd$ zsVG2|Q2PZfY1K&`fqP^rpwOuUgA(V~t_J%<-Lh{zMqrj?_K?99WN%S8W~hmx!Xp1@ zW*y83#CXb69&3%;l6;<_^ApE;tzZBVPPGU`l-;K2gJ|r`!D4s^ zLcksXI`()i4?g|U1qSTJUf0Lce^!8K&{k8{KWs7!#*X`Hub=uI@QH|nt)JssO)Ogw z?snH+GXY<-Jto$#N|&+}1vEX!PVdpw`oW6?ljb|lYZ#~x z>Tg}Dzd|JQFXt5&8^kThfa2qHUbfptNpJ1?l#eJk*|qz@04$qI&ohNYAng0;!d{Ec zJBbi2KN5h3Nlz}Ky{y!gPfEKcL_0#kf#oy-f z_dk$kKoj?~+LZcfXZbOSN}u9v))S~{B2XHHxDZriWI6>0ueQELXkMncpkKPxmsW{jn&|n@& zDdt_oRpuv)$3NWjNXq-3^#Q_?fEnwE2ejK`9Q&jK^LU`OlMN<})UsJE3h$w>g%a4} zl57kq(W({ z&Gb0lKUPT}gDMt@Xd(+|y5zt-jQw1zCj3uu{q_ck&lZuf91ztIfVGWWRnt6}D}HH^ z3mxw&mvU6Ei`3RYuQJGOBvIO7$&uPAk$6-J!$_bg}b5?Yv(b2HN$Nj>BHjV*^saf3;J@{+Y8Ja83i0$T6jFe}5x>ZwEG7?JSR%X3f0 zw0z?4wxpyQ=GdtIZKd(C+y?=9(<_Zi@WCHG(6%g+u$tNFbIv)%#DW(6FYAPEu)2Vu zluW!qYmDa(3@aZMe2uYK#(~(Q@rp{;JlNl)V=;?m)*Dth5nf4FLbhT}je5k>!Hn%d zkq=54r$*LIVI8P3K^15RRLzllxUD`hTAapTRG@LXHLM(j8VO+O7XjGlJ4ydEuk6SijWp-*A*1s>2c_?|~lLx6Vg@z!)L4wbW@ijd{W-s^)_> zqWbE|?)|5mu)o4Tn%h$|iZr8;gB2s-+E(xRd#!YM^5G}2t`TyrvJZ?+3&#s zo$M;SNx7T#xZ1X7)>&)WvDceJg%^Ou-O+SONFW^HAJ+>I@P<;wm0qIOC36a-mjMVJ z)E@%QctU}a^hnMXo}yM*ap$r~y#Gw#wV;Z}v;$MJt~5m4tV^bf72~0)gmmEH9bO)~ zQLI4K7Clo{yYpX&1ZMG$5zNe;e*&lwAls8c^{Sxv0K9rvMv zo6NeEnsn(p1v@kwzuY>k@gSm>nntnRiz|gmsZV1xqrG~`pd&5*Ctb9~caQDd_U?M- zq!jOB=3Mv@C6=CYxPhHY9<8f+#l!T)1|Eb?Y9|~NU>%z-4lj`=;IJ&f%}^+X*3=(K zqa>*N4hycKjG_?b-$w;uzC3Y-7_Tq)34XE4z>XC|s_4%&QhNz@95?Ou`tR&q+HU2p z9oXK}Z!f;8f&{z}z#FZOuVy9lQt{HMj~J84YW(mwL>ums9i;h*Md zRZ5vbpEUVmLJT#|;S;=UQ%07|6k){#EOm^(_grPmT6!l~Kgy3eFCEDGoh{MBzs|-^- z`d#9?vG5VwUdV=_Q;;;m&83dPD^G6Q-%{qa$1Y&V8k-7f=Thdnf-Hvp&+~lpl(xOv zC+~`Y>T1)!Wj&&XD5C$|rI0eL`ly{{b#x1PCQ7G%)H&3KapT ze>X$|sNKhBG7~OgE);EZ;3s{Pq6!yn8wcKQ%mksMYKbN#3#vjG`+B2~5O`r9w7_Xk zMFI)v6w z5jPrGi`h-lCbsjMiW=_F>LbTN<^`hYs#)z9ONP6eX;64$sLulVU8lPt6uMKx+EmUjKF9mGUVrzL7tk1+CTG300@>Pgg6yptD=^HDEPpm<6qK;NAyw|3K! zMNzdLPpW3>ZFz3$0z#R!FaA@^N2Y6W58~7XV<+yuFKFhNg>L_pY{Fv0c@WQZ#t}4Y zZV{46blshyf4~oC04Gp|9Jie^@X+jxWU4GL$K|a~2#s%}EIjUO_48 zs6;N{c>voime^`O{JF^q^*T^po8qTlRRvJDyP_4*oY6uX^WmduY_k^#l8^ggyegw! zq8q}ek`}2J;rA~`9`c@`lEQ%}ej|%~RbM=KI|Cn2%_<=^&Xu3-06vHd{QVG+276AB zZ^`F&dSA69)#TpNN^xVcX9W;f>2WyMxTr8bYx&%Fs90h{0FSyO?NK&kv_E;PE+Vel zx`nT6OcnAUcX3f?wfaErt0n9<||@~vM& z3bF9n@_uzhA4EM7RkgrUI~JOJ2&piL`_Q&wg1Xaak)D+j3lV$*mF>q5^vCO?E=n!l zPDxNUn`a?Y_N_*6D&;;rV2B74BTeFc>`ZqVWkN3!rFt(2HoiPM;Te7lwN4FWfvYH$ zc>g^{GLg}V;W-t=#+I_)o+sAL?RL-?6$})OSdjGlW<1Msx&AG44N2NXeq69&`P%Fn zNX(hRO)DOZW!gFwI?_c8Z7RfQbsx-vqz!X9_(-CRM;%!J-(Ml(%p7^!6C@(D=C1lW zMol~CJ|z6rhHEIPsu@@hw+D~N**%z-OEQ3&BWt6EPmIwV`^RLBL{87wP$>vx5~NW+ zm^En9QzOtP)szqAbLEqm_@7ma%T|(D{B@JD8tv%^KN6?@X4X7Hm?MevX$bf^1t5e~ zzNE2^?4&+LSep3}H}AozAU%RQtZ2m!q(_|Ni5bBDi%5j1zc{s2s#8#j1$B=@vzX~~ z*>B4)4%cL}sW!T@sGLzObo!TC4=dSyOD*ebcqF93=Y15?pyr$gY@&Jt!+*!}n~Mrb z!bFuAast)X`(Pffr1IZE)m0w~adhjD0Wh381jP*EPaZjvfRi_54~>7+_#y6$giRHa zY^+HF%0AjGUB#)u2t*oW?IgJczwgrLYUUsG)N?YrY^Oma;>vqYj^)UD{cu+9?9F|yb2XU@kS_Gk?=2%Ok3S&#&%MnR2r^@$(^KF@0xkXcz)}(v ztmsEe$b*Mo7My9fhk;ywrSL0eoS&Q z29~Hx4csaYud;J1k{u;Yf%5n=pw_DKIvORo!wc8pK!9>98QPDQ6<~W;hd=0^mmBym zf^>oKTk}Ajy5@g+IfV*A|E}rB=&!gZ#(WQY-uUqtZgFx?o*f$w@mDl*Zsyss!|K{<oBCyv>4rYEjJ1b-o{)^3E>euM^qF)fw=DOxgI8A6v@pYqV&e-#UsjvzK{KA0S|Ng zVTT9fq_=f0vAab!i#$`u*w+FrAnX@X_|=f>&A(-xZs#LblAIhrFFzBx6yLf`?^_;? zdD&S>;7B=O4VwSEnX=!f)$7_XC*~E z4LO&*;h=`obIH$pjuS49k?MZI4HR>hvogU`Aqft-FbPyd5LOG3&CcK$?YS;NfsxR} zv5hMDcsF>dY)GEhz)t1w-`Vr?YRo41^Y5WxT8$oTJWCNllEMK;z76PLql=oX-mNdJ zGhfI^R^)Ms=yuAj?$_StTQT7%q6@p<2;x*P>`Co?W%4Fp+r^Eeihcv#h?GhgL|8{> z-(*(rzD=u8nh7jeZD;^>g4{clq(j2X&~H&@K6UI?)=cV&(bMm=l(yV00|PN#^$UH`cmu}h&q+d$bUpEyf0#3 za(65X-)0`C21NB}1zF9hzRWy#7kXF2?5Y4a-ntB8xbF-oNlF4bFk#pq-!3<7 z6ceqlK4%58kPCO?cu}*wC`Q3vHxJ(VMC!RIzyckc{V(>iUpSY)Kz}av^H5Pvz*b_DfX4NhMzkh$-TPN7c1+mg$yMWzJ8<9 zKTT;&P0HPA!{F?=JpjQ_F7U=Umnc})>yS(BC& z6V5%(#^+J(R4|-Y4XS7v?7rl`sOR5+cI@Yit8IfY> za;MYdHz?NqXt94kHV6^`DFD8c&_pSBGfe0{UnJ$Tu08y<gGHtRdcgWzW67;NHj)zbc4BXVZ4F8)Mu zX$qd_07^)iMy&W>ReUEau|gb)F8p65;C2A;_z8nK%Eye|Bl!ho#Pvs|L<$c3w#&|@ z1qigJ97|D7v@rybbv#)u3JJ0rOGyU z{yDt!0%+Z=h!lUapDbJu>!w-JAaE`-E8|(j8}b&RSiE_noy5IM5YKs(w5_QJG$wTT zq)Mjw4Bml6k^O75hY;5$Of)&{@^33wqv!Rb!YKU^U;On^c#W{oHDi10JQD1Y;Mi&J zE~!exr(Q%!?R+zLoPu|f2CAA-9fa^Xnub6@p^gE>cNj4FvC<1HDIGUqqK-x3#3cYV zCL0c33&jw?6i;;iHX9t)nAR_UdvP5}Ddy08ey1gjf1_34NOh!|i#9YsF~}hC$Y|1@ zCiCNDnW|U{D>zZWO^HZnkJ~P{E#bF?MdCRmghPnsC`ruOKY%v_KvAj}2+TJ5c#JBl z6>~B`7-Y1kW3Az&4L=vd$GIe*L_rqCz=ATR)w#kjJB13|!RA< zeU?I=@jRD>GPVan4}$nUge=U)zp%{pD7h_{fLuEBVB-|UgLx@Y3qr`WR15C(()k)x zBBDxR-vlW(qeaxkJa)2MPF4cWKe(Nh;$BOD{ZutKjKSVUdb(wix0} z7)t#6S9dooqW#8c47Mx>z!hvjHwjt}Z)g*U^{?7HT{XwlcW(cdxwi^al9@gzmLAOZ zKiC5gw*$&`dRp*>BRhwX`H=z2G)Rw3eG8^e3ok1_rW>sVUhUPs_iRNu(1}LJ21a{z zD9+>)kR=K>7BEkDQkU z-nDCZQ(|1iTxbDI08Yl(#vx5&-H6ianFEhSm<1{(tObQ!;$2G@*=?jsRFvjiy&`;E zvDP|A5PXb`NDDOc09uj^y~K@rN_?jKqd7(KnD_~Y5UykeD#pYD9YIZX0u@g?Zk{A4 z@^gvZ8pKzo^Y4b+>HQ4^C4Qq$n>BW^sh(t6>s*T@=ud!{fs?QDXG}73z`bRn&T}|z z-SA>`>Y4U&LiRTbu>vCeoKo!L_l|R=v1H?{<0Ys~x?GW1N=mw#U+seNrgpMm#oM@d z1Y$WHw<0TC_^td5w|Z>Mmoe|UQy975aB#y(EX?P9p@@4U z*Y2}q^JH@wE-R3S9;c=0sp%RSGx$naBXAykq^ZchP#5`lin^|tm;HGHR==&!@f)gd zC2_FHI6ULtut4FAI%V*st&8^XT1hs{}c0b5J-*j$@YMd z&6Qk=8P(%Kr&*U8TR@M9b3nzx6MxUUa!$Jec)A!A7oZTPDWO!swp>6;OTZ7NX&(b| zEQ~4?(uW$OyqXGnaV{RHYw@`f`EFaU>?{BzIA<3O1COTsbUz`tn?}lnpRy`@zw=)Zc;Xd0BnDJpSv#LhA z7%bV;;}0DYo2x*}=)p-Ab!J)%T z(Lf!;Bc?0`PHxPAdqSzgX88ya+8;+{oDLb77fsPFmN`GdR`!oodhq;orH_5`D8k&T zTUqB|ebyLpk#}(k-=<{3FzEaPjyS%zjx$0a2$r^* z8gA*NjJ}&KuRFU-Qz%f{P2)H7LJ*`cV4?6h?VjW6TI?wZUV(#n) zrG{%9BZ6tru#d|LXB^AIx#y{z|34Rm=p@~#rueLfzWh)R|1>Ks3L`W~hLqhu0;fN& zZK~1FoT02Umj=Zts$vJVmYAFB!-}z5Qt&QdL+o5&4kd_YD20iVG=zQnz89u`tbhNH zs=eIuiJ)}n1v36EZ_aK&kZebA+%^h-ZwGxOV~%XGgu7<6h0?N* z0tM^8n|mOD@DzJQj-#ZFJIEXL8HIJ4l+IIfsqV};B*p(1bmf%^=AE5n`sF;K(~SQ# zj6hs^bKFWZAh_}od|xA=oWluDMILJ8iQDA)s?9Dw&bD@Kf2;KD>o^=k$3bq?X!786 z+(g-*kfQt>rGw4784%1)Qwgo~*Ap#S%FkL#2=OE%DYwehasbR^nTWo=YV+m*l?dhK zrq7iqLVt+69}E0RbH{SP?3j-yLDA6 zEmv=dXE3Zq-TpPe%D`M$w1A;7pFL;M%0>;D(c22%w}MI*(3 z8^Tl4lbRqU25#0A)YE{(O&Q{O=w$N{GLZELaSMEx1nBA|c)wt0OK zPGndg=puwL7AZ%EJe7B+zIJ{(-ZQqAD+knBNi^DUE;j@UD3b|_zH02ZQ_-Xu=KGB# z+JkRGZ~<2_O33IstT9pX54}77ob2ze)2N3oXKtNad!X#nUB-7r3U|k%_k;e7O<*LI ztxwx0sT;j)Ad}8D9to>Ch@JQ^N3{q*t&XAX-}=1tUDH@O!XE1w#^YD)?+mI8>=rnm8R*ywrHR;4YalZ8s7rC-kI7`_P)a6+5GvVEW>{~+B8-CZReqTHxgxfap9Z#JfZnjg7fEaa8U>Tx6i=-3bJ0Z zb9RY}!c^p184cQ^bDi5lTxb;|y2T=mxS2GV^tl%@eW>zX@h?C`p;UYzUhI>G+^2J_ zN;atttep{TDytaX6ZBG^OG1YX5z812X+Y&PB|=qOtLp_66=_j%0$Yu73+9ZKJe(}{ zRGpVg43*y&AkQCR68L8nqx;^3Pb=cUp;TJg%XyQ1eqi+?JybM?F3ne#qex`}ZIZlO zi>%r-wYb zD1?DV*1T_kGT`K~BM!a#UBjF+>d{Sd3FM#&4Fc6yGrDcocT5~Y6bpG`XX9e69Ml}8 zTm{%|Ej2Jh^*fCP;Z0A~^&Me&gGOoMH1Xj}1e~u4TNTgjPU|MNQ+a_P4-pl_Q*OzL zFdqrSUFH{6Cz9F!TLRWsQo}z)gAy%?Hj^^PLDfm8rAO}0!Q$}y#A$dM2QUH8rtTy2 z2(luLz9c?#da)+}nr0LE=so|_jojzVMZ;9u0(gDFD*7FD1M5KxDMI z+l176stI3kaon4Kaw)_njZg0sYRhK|jj?6k-c0g7Ig|CPVqJB2O}1RANFpoMJ0CG4 z*HhO|V1fCt(}(%vefb~{V!}PBm{W7Gz}@9OgF&r~H(zy~-U)m}ZuGjxgAEE{y15t( zO7xsngi&*!B)@ndDsr~^64R127X?vF=jkcN<%#RnUA62KE1R*Jyh@`7QKTtsEa%F_ z6r(3usUG>|r{VHl#smB6kFq`evGdazg59+#LBgoA%Ug11%z*W2<}#C5t}uf<6FM9j z^xwSxShX4jQd$D`>62?V!qsV~=n%Hd+*QREX@9QfLrVeuM0k%GWhUG5Rre?6sh0-9 zq6Ss8u%f~YLWbV`2IbD5tKa-)%Mz)w>%WfbAe`>kobUQt@n?cHSGOwm#Qibfx_Z)F zEe%9CJ0cpGa2Pyy4x7VFoZG9!;HN9`x^02< zT~kywA?zPym3{Jdj2T6FMN$lmf;h6j1%IBw5yNvw5uUW_R}UP^fWMYimQuzpS{d`S z6w2h>sXi;vjbq>QQ@fZbR-!ZqiCKNi@yrwhPkn@4U==6js(NjvF)@O*Nv?GDa1`#S zrc>RKaIj$MNiZ*!;=zXo0KU%s^`6vHjPB7Dui6PwI9UOMe@{it- zk`@s9J7a*kL-5p%FmKnywgXeM*%-2?oMRvThzyf#ExDZ2KcT3e@@nEWWQPZfikYGU zQGA{z)+=$*O(9(+%=pJ!p9`kT3{maWbTw_<^?x9Uxhwl{13eckM8Hp(jpIK<4%P~^xm}4_)c-ql^HvH)JA=(( z6@I9;#Pj8p;Hevz;fJ=>oCfjy>yy%Ccp+G@LgW6(0U+{+xIjpt)1qbkx>cYZqs*~NiOL0w!K$*gLwyIkk$OMx`3dYi4CiENrt|ebJdJ_y*bq}FkXLc*tYE{@eNI^+Do1_;eqX$7#^6Yq0`;mD)l?y zE0qcUuv@2iNZ0k93@tZA=i4dD-UN*a(U~hRBUu-N7(?&=$dsbJ1zMvu=d!@XrpeN; zH4DPgJZ>!PWQZuBs8^v8h!KJ#a*1jHzT0<}Lq=sWV&o;!*L*LLRb^c$In?boUy_;$ zxO~>_wlb-Us>){@siPMc@KrUt6Yc_WR=PQ<(D zdt^M2${5mt53)c}jZNlrQsp4aW>cC;>C-3Z`oFdl5}{#EK@o3cCLy%TnasNuP`dXY z1~l;TQtJu}Gw5g5!Uk0LO zz)2zS&5P1Xn;O`uVXWX*ew<*g_+n zA;{ADeuQ1-Y3M=FA!fL~6S0UOjKuy#S0DP}Q>MF34%Y8Kxa+q59sX#jmVc<|aS{;A zjfGQ(*~+NGmmW5}t&jZ&pJA(Id%$ZR%^r+bmXBEPn>2qL`QCRk5dG7-tz)wjZRv(- zZtd&{1Qx7dQT2o~r@Orms1~9;fU<{Ds0L5-iG~W`0X={X3?%{ z6$HUt>U5Q);Didh4*ywsxHNU&Ma74_eKUbyHJ;hqUwKFRSccbSq4uZxY^&0swbU;w zj_Z-A*<9Bn1AIUS_?)qg7TjYKCCYN-IjO7t+@R|}Ybori_ zbE%GSR_2YoxXy|X%%(%>f}K+3r)%d5KcCMPNP5Yx4T|I=XP4? z;W)}I?-O;t4C*8~q##*VZ({f!7=4^C8uZ0QZ91{H$L*-;DFeV0Bhr!R_@W(^wq5&d z?B;;o?_W~CR71TZCugH_2Z~i+g;ua<<#vlj!KQd0EdhzVU5*y9eCL`XS2WivJA1v zmVNBT{_^#ZZxZ6Fqh@b=7K#ve$__Th1X|O7hQ^oy1M)(lHU|RmidWG8?KIaYHRhLJ z(UqMNmRw`KvXg>|K{fGEw;G{y^w6~_x$|3iUhS!h*_FN{EzsGKruEMAELd*bH&MmshOuw>;Ofelg1*TxtAtA_&2`(bDe54k; z)hOkEbEB#R??N)F;g;lW&K?TJA3*Nj3}oTWB&y1hf>2$q;+7`&Z3)N55=8h^-a_(F z1d}<*!$HEJ4~#@dh-#TE$>Y`(GsyI~6pd;jT`BExp8%#KK&(C-W%H zuj|$Xz2^ke^~Xfy1<1u--k=dp;|~+vB#X}ramz@e=)=sydAX!cQ!6T(hR--TcY0wSr$G2b)ajF2-5s{Bpd%=YO7u9T%aJi_fnILiGk8otbGF=*2w z+#<_ZN?H#>(@p!@#O>rC^KiSW3e#=?hxkG(CEyFk1Gel4pcl9~meZGrHw+Q}l3c#v zZ@9jxv^Dc7nl)VZLFXj)c_=#<91>|9bqElV6#c}MODR@7+ng}OPzxYZfnw;&p;@6s zY!~f!!%1qD-cLU}8fB7HXh*cXP|&swxnih`tgmBAdAPq=gZ*F@&(a6Ct?7r!>Hx5j zc=zRR+Vi`{XuKb(e$pBbD|NGZ`sK5|C6Uaham^V`11(h>M~gH6YSP8j;jcwuPB%A}6&VZxWp8Z80LN+L+Z`tWraBoqJx$wno z`SZa$W$XO0s!@htBs$acyW20c)uz%*Za-N-Kz0{~3L+qu@}=dvxx6uwcWvUT0r|k3 z$8)4Jg4UizG9?%Gx(wdnD}y};vwiV+%JO_5HXZfWZaA`v6xHI$TsEtkw9}gyL^Tj0 z4o74G*ko!YGF9WAc3EI&RA4_Q8+kJ13T=n3(=pn;h^1JT)zb{Amh)o}Z*{KP-qGRV z+Sl2IXu4BYM2C3G{>EPt`2HRCnldp11=}mhhV^I!5`(vmJ8<%F_R zq~N@jyl85quP{jn-4hX(4*eZp0P!TgGMxqoJZ32Kc|qCLeubayP=Zj{hglAW?F{?w z{yhmc$Ak@#kC3;X;6-9_l-Xs}kr@@Sx>H)_vPdEAFTv>#gw3|Bq4f}VYJ+Mk@Se^q z@jw-48Y;LdU$lN%*;h&X7(?*gGVV9VQU~&^X1!D*{9LAA6O%GQK-NrS-T1VR4C5Mz zTg6o>5vT2PGBh233AD3)A?f>Pax#ICprUrApfRHp48p`L5GmH>Iq73;j@5C|J*+6q zEPXG{{kx-Zcn&`;By_!JUkmZwJwHs%(BF0VD(V!!jK8kDn77l3H&!Zge(-Kn+fbn| zU!2i(*)2bv*Qat*Yu1@_Y7*iiUOKNec<*s~j7dc>M-}Hffm1f_Dh5*VS-`Q(W}?|D ztT?*mF^1q?;!emI8>;~od!t66B82&5=vGx$f^rn{_{J;#u@`*Z=^4HD5CDh6L1*|dFC=m=5c+gF5^Sw9;@%Gww)`Ou9$i>!?iChYabcxa#^Kv z5Bv_b`psvmb18%X%^y3O#9-s~Jc=rAR!t~u^h%>49wyM<1);l{gP4K}&Fkz(4O5Ct z{J`s`f@noj{+}!#%B0y$7B72jJdTWcB;DosJ85*if|Bl5 zq&n3NMR?KGKwM(QMhhUVeuJOF`cqbJqn>qQC55W$*(|*mzvhI}nnCRHhTd`u2L!r6 zB%v{A0sJzF+rh|@j;WM};=}Y1n*+b|wpm&*)3!In5vOp1=XdDeAI&lj-9lbum%!gf zEX;e$j^VON?2s#|tYtX~x0+TlO&Q~xW&S3xB8phV#dzzHn>{E5u$g8mV-f>W<0=I= znjBJ-Rve$?r(=64gxaMh<#Y9r1Mpb1at_lc&!*7 zep%U`goI<#F^?D=$CV?bcK#Z~1FdR~ET*%e9(pKvIoHWFa4zR)`-O<42@zw}wpLBMly zd)i9Fj^cZ(D}~Vfck@d5cS;LnA8$fo$)~^M@4x{p@3!d)j_3A?&kuYgR1d(>Gujg% zD6Xi=)8I4!>%q)vMpq3VFIDax@3zZ`Kg*ho558vD(w`;StF~J)JZ($&HGKNPWde)V z%P?L9K|*o3!NMeZx6rqUy@TIUM13{r$N{vCvy|k|loPyMIW?p}&3nf@-2$N!IJ{=Y z+O}S7)mFtIAyNoNm%iwqb{11;04SErop)7gzE(ujRe~Zh(6Fos-??1vVt9 zvzZ#YCGMl0-$g9cIy{lB&^4$nY#v#I|J*V_S1IT8<28Dup(ar!y1on=H5Q5GrviB( zpipvbOSTDmHnQndMLq8b_62o0zBt_vG(XD5MA1Xi`?^KyytaYoV>JIOtv4@r5WUNY zOx41!uFw_(%=zkhwc;{pi`C!Gtk`8ZOM0eCi#N{fY(RhsJ{Q&?Tb0y;_}VR8IIQNU z4=1k$-Q6?%uCB}YdA~P{4y3CS<0ovslLTED8cm_WGk?|7Q!u`|P<4belG3(TE3piy zI6T7LlFt<16H?5|HlvByruWb|_#2BhzWsL0Up&;{_Rd+geTippz%GRPR$kD1B6z zAqUL<_Tc((BVtpne7@T3@7B)4-NQX`q!d5{Jf$^gjbKta=rf7Qjk$GY7>opC@W`= zX0azna}$~47OE~xogR#ZJW52#4(+Q5=Hbt0qx1ygaP&H0`CFl3F~axB;b|_3d3|IU zMV0+5fgyK92fTxejY8!XC4!7|O=FHl#OQXj2+#m1!P}niBkDbglz@VYAgOjD>z0z`Bvl+1 zK#Xg^*ukqyjDV?m?1r)Aj?N^w$F44;O&4nOP+LD7Pn@HkMamhb@E$;2AwR}0JB*I8 z;bjIxuf0_mmSMdjZfJI$ThuoLd&LNkNn-n~m6rg;$m)S)SLNv(M%taNS69wHFC>cz zhg~OtFE%kSe~CP>7M&IKNX&)aTejWyB;qOH`A{Vr=J*Y_G-}B7k>_Z0#C|7v?912S ziPvu@Vy)L*F8t#oj-E+@SXF89T73thdeHFE*lCcj$D3@+tl1qMI}v{)l{$81S)z$P z@?4xiava0vsVZIfze3$DFXPLpvuGu2wG0qF7nWT-EY3{X_#Qt0ImWFQDPt~#miB3r zl(?9ik3j~07|FmA{rH_@W)8tF@^0@TGi$Me(PQ0`)@Z8zBwM1#+NDZ&M(M3_+&U;> zr44op-A8V?ZS{y9CjqnJk)yIf`AdFoWN7geR_=WPN(jVrg7fj`ug;{GiyW$;HO!20 zXEUwswNK->wxMPAwRHvsXnknq1OYPmRZiHvi-0|aq$(0Z4eFHt?BeWA4 zW$fJ@<(VZVPMo~_mYl&r4-==L0#X?>07XE$zfQXZ_4NXyPX*F^{rmqF7Y4M69`tT@ z<2<41%`S5!d5szZ%9oXBHzzPdexHtW3*TO!G_SMdm{>}ho+C~nn=dfRV23C0EX=n) zLSMnk9~sk+ADBA=yRUuNarUvfDniK|F8eIG2iCToW%O4%p2#9&=U$Dwy>LC6MOEmU zlbo1u(Rhb|kZ0~6UN^Cf4n~=XW7UQjnzq@`n}n1wW~621?m?<&Tx)_|dPg)a_rHLv z1f6H@#F)_u&4qL3(qBx)<}EI#L#JEXF8d?*zix^Z_84>jJ(M~6e@Ha9SxNl-jFVwE z-6n+y7xakPoUA<8^U~F z-uguok@pRRpWSQ7+;Aw0N)5KG!TKWbJO{(!$MtOnNr*JzwD^Fme!Sb(BX6Q0=jCJ* zR4&qjI#C6PFov$vaOec=`K+HFpV)R#*foXcRn;FJH7{=MRu}_QM2;8p3}oOWc35@l zNPA-RV^7tluKx^kQAShuR~mMv2S~{zhBL0W1$?JQrftJM19I%01Mu4dl1J$vzo`%3 z4Cq60y{oWLR5Euyeph$pV1@PBt(Hl4Xr~58E z;@0M#-xx?VM>ZRSdT3+5z!nSp!Y^6R#B?+Vs9{>S2NFf(V4J7D!ubbRmC+#yaB;G- z+{-~xc*gE?A9Hx)=JjC6n+#Aqw!a&-{CqPSWT*4$yFZGL0(Gw+X$cP&u-Z81!Mirz z+E(tZ#vk2>Qqty`P1NH}d^{BxXMp()8aDq6Qqz5-l3P zci<7AFMJ4RAX#mK)T&3Ph!Z42D}ZBs#|1%84P{D1hV3wr0Md!N;}2lqBjs>lDsU*? zIeVF=E{6%h90yO!1MA*fv8XyML%R)DtVy#>m_eL_Nr1yz#8T$*^<+~2Rw%{|>m^8c zvrx2`vUw%u4^4q^dfHHwk-YLvea`Hv9<4!XP4o&jzMM%NZ#FU^(TaY4KzdW=xn!05 zngpYRj3J36vpjPU3^YK16BDncZ)6>yosi+fKfUzF(3Byam{*yT2XtI!-x5;Z}g8xXrJo-7X~o$eH5w0y;XOzy4{VN$OJvu0LnXO|8?YIui7 zg%Lc{1XnOkt*2KbdQoU#6~pV=lNEO#Yk1*^808`yzzD=c_)^pTxU+Z6cHfb1b%xKu zRB+L0yXMZ|@Yw1Bqr9ui1836EDp|}y|0uhFj&EeZRKcs^`G+B_ad-BxB(|?1hKKX{ zLdmpngJZ;AR0|9PRSDUCfgfBUfu@qv<49rCS&x`x(X+8WUut4 zt7^2em?DPBp~y9-2&Nh{;`NPYRFHHmMR zj#~S}C>deaF?~4PvhTgHWl3)Cy~aN-_e^Fp?|dHV1SA*Y!l$Wi>UeqjR+Ackhl`sp z{U1%Q)LsPwT(G>XL3er3mMOwmKziqz(h6diQ($NrQ>Sv!hU_*rA{3h~1N0j56h1KB z+v=?q1ciDdqHvSw+GvH1S^LB?;rvIOe!e{CZXFmIaLhp~V#m@`BMPdw>rw5()>&Uo)++B$}%V5`cG&1ZV*-;1A zL;=$<7cApia8Y9Xh$-DT>|BXg!)aU+059h~f;!4|xUE6krX5y2D(F9MN|Wlp5=|is zALbSNi`ol&lzA(J2LItB|L{a>XCPifJaX`j)X+#tvV#niSNzYVO|mZ632Vc#45veh z`u%uNh<&98h-|tc=lEhVJnw+ra*FiB{Wq~SnY7dt9IZM-qwbhE-Hu_97nhY)GWEcwf7$@gAJa*BU%MrGPttDmHk9CuTGwLP z7s?gv3W(hv*d7ipLaDk+9d)pc$)9WB<1ve)bEde1Olt|qTC|Ds!kD4 zwZ7HZYT8k14@#cqDMkA#?>*QXt8W2d6@FLS+~zDqTR92!ig9nw%F@AtiZeVTMJaTt z=N5AP$~QP-(5P@|M%To!v~t%mUu~UNpxSf(90IAh3b*ar1O3VpR=6?(J~z332g-sVIG9B6JqVioG>T3Xxb*h*}2 z$b3jG@O1ghXny$H=$-iBSjEV-`e~0$)Ae&Yppb7TGkbz8El^0UycTJB87B6mzP%!y z5+1Xdm+R?U!}hyD1*r#d@S*dPjlf*ZzKgX-%GrXD9mEx6F2`2NjN zGZ7ws=p83eA?t*rFZn@c2By}L^594lGSPL!ql_dUI0p zK|)sU9rlDAV5_Vmn zgN?()r@5)o(24v9w(;LACM53jmcqX>%TC#!3B0{-;e0pG+1`ljq@Fc{sr;={_Q)6% zcg3TYrXl4f%IZG8spWrjSeG5Yfv&xwhwXPqmPktLYqpxoB=xw4@1j;#_!TJzCbh7+ zD~vJ&l@rXsZkh2q$DHA2T0IItv7p?|SVI-In+D+N7n$nPG9RS>Nt4+=I--`zK0%e1 zMhB^ceVid)34j9}=-G3v6`&%GPX=&pRvf>@evevYz@d% zUh|=cw;ph0*ijCY%T8-fMv?y!k6);Z&Lz6*X*=U`7b&^xnfl8e)6ywAw7cAh#+Q5J z78#EjS2wOm#WzpG6YoZlnBc&&nH7Fm40&ARQi2w4bpg^r`Co*`6Z}igB~L0$tS))o z{b`#_7mg1%@nVmMJ(dGDvmT=Wn`-doWPer_1O)lyolltkOjZ47y)$zIoOdO5I*hP( z!wgg-?1U;^Ysi5R=<{{*0_p5H#dqFF!WzKoENvtZhE{p*0>IhCdQ%J`muSA7xRvte zlzv1H4KpUsQ)j3lJ?CH;+};PkC)vv!(9M_#ag#>XHzPsqfHQdWeHq7xv0y<=Wf^L8y*61eZ!|rlF69I1g(>Nt#HMvSN9+Di2>l2df=8R= z)wN$wdH6d_Ap``%)HN|Hg(MwFEC&8{s#XnuzeqGodTlVA*O-VvW2*gZMn$y0%Z6@= z_uzcL|FjS?DAL*@3jRm^r+Iqg)r^#kVKW>6xUW1JW@a1eUS?m7bm*372$|l)#$0hU zZ~lwo1yW0dZg=*|ub^C?2jt*Yv!ds69sj@2$sH_~?&ZM6Q8F{_riTD!g~FgLEyYP& zTeBnP7VevAwWlx`%*c%Uq5im+7oz6#f-hSs=C<* z6|S3QTh5}ahz14h@+X> zjlGQ3e44y*t%|$;+}X55Zy%UK_RDzCdHBh3LwtQ7@IOK+o>)8Y61L!3RZh)PYarNn zJ`;#Zh&pB*_le2@B3)K1gKg(3Xh1Hb9MW&6{ihNDhya!9V6gQL&-*t(TfkMs2UilE z$}?p~xY_Dg+BHfev3YmH=Gs2v!3i?nT+?uOCEY{;bPI))09Bk^5kf|6!BPeP7)TBT zV$WGYRjRTELni~F&>u8S!zfE=On9VfVY^Zsuo5lRjLB1hr?pe zi3jZlMCH<;(d9CHOc-5#f^y&e4_uthd{piCRYUGe>~KG?9p`}^fb(p({HAkh0-Is~ zKuUg(2Vjneblz~lrDmE9X8^1lkp{CeDV1`hj=kQ;A2LuV~mXZOs2pl%ai@ zX*Fp8x{1adNHPC@z;EUuvn&mhn8%2VDAjesx(o3xOL`VO>m)r-mVh|HVV8Z=AsU8= zEnVv)AOGOW2GG+;+6@Ro{;)#E>K!jFuF0;O{0f0jZ$U7{_JfLlG}~tfKzXtmqERK~ zg%Ud94?)gL)H6L0W>N6{QR@WA3SDR25(B{jh|$Lb89QwUNMycQ+p8Q^fW%bryGd4D z)KT5OmwUAF+q&3NKz&zI>w%XQnCY1|$v;iEgo2GxSN&F&+T;Ub3+KsSLM#T4q3^VOg@zp=L9Np2(;fmhhU&;@&F1%B zwPCt=HZ5PTu1lI)WE;AgSfYec$Wk1HlAQGT~~9e7&I{AQ@yF~l-3xxPyk&px}lH7npVIK!>NQ29M)``s&doC!=*x^-Hu zp&B!=cpj77aGfDAgRon`Dhx_NhR= zQo%UZhsbo^x!hY~bbwtrD}&`F%6t5S22V1j0P-?z%o%V&7f|RW% z>01gC&5VV~X_?X2q0>2kzkmJ7WK27M`a9&E3A3)5Gz+&j9GUjaf7=@oC%aklV@UjuC(p z+q>2SXviAbyFMWQe(@-XcWfN9PWvbFOY?&-vT4(9G@fM>V`?h=XS zx{3th#%i1?RyvtPiPo}ocG%W7vqSH`JJ&R)BXXHohs)PGD*7zOhPX(zalYQ3(>gH$ ztzPV^yxd|8f0B5ah}<~mA|2N1N_$&Cx>Nw_QcIaW-R4!mWLFXZ;ik1S zatiTJ&tZZg2iStasNbZ>T%C9EJ41hR%?A zJE6R3S>i>WNC~E92sAJMc_xOR&CZ?EqARrr&l)5Oqr!f=F)Zt6FYMZf>O1;Miw;<$ z;m&oLl2CR(IzuV03_UWhN5v>dWS9U7Pjd?`|yx%%lOJ>S{N^c z2JX&Y$+}~2f$?WhlM;mzy%p~;67&k~v-}sQJEa6Qj29O1>kY6Eyxnr=Yu?_?J{N4- zCD``BhZQA97VVHMR|?+Q#hLMEM(nl2;Mk&;p;I}MgtmHrfSX1*h7jh#(8;HAbpx1R zBKUtN^^cJOst*HoPO-k&8xG2W{!c+XrV4EPnMwf@n{hC|_?I@MyCW)|5C?R5t5MS^ zB7??@{4+w}fWmF(i#qZNhew^{%etBeiMqqOK!)KQE8&CpaIw+uxT{h~9crFfcL3GUNNPBwGTi7cepPr83_quCPT_$nBTkpDmDowXTmM~wKI|9DUp0B;myQ_HT7j&WV zC+YtJlZ_cqam|=7?s2(wtRy8kV5Lq>Gg~Ni%Ptx9T5H?CtPz4wDN{Bp*_Erk(JI7+hk^dI9xfy3R+!g3FJfXG8 z3H2btO^|^i$>Iv{C>(}xb*O03%gU-!cxC*^cj!FcbQZgnh|AmKq^5jOrhH9?8{(l)n5_MAvOAV6oo_xh#9^Ju->hTAFuM# z;bQg@Lmm?2Hj?PAKMx2atL~Juhhm#cy~hF;V!Qmxq6o^KxZfABBB?HBTS)Kze{w-t z2g(eR@1z}U)##T?b|V;ODHc9-_Ijcww)>+0D+*6`lGg_m!0^re@HTwyxl`@nQvZkP zT7axeSW4;h!|e$$ZGj42TYWmxL9s zxJbh_nFup&rGYy6B{z=~x&*^Bi0Loy=c(a)wh%RP7!YP=6iaV>7pYBgM$Go<$<{l@ zErN{dC?U=GY6Z`zsTV3Vjk_EU9O!qg3YD6_(4Kpgd6z_w52bP~d|}!m$FAdL>uee= za#Da4qe;V!=tG&YJC#oL&<@at>^yb5Kp<2F&U^0gtb*GHtHH&;O!8Z{v0^&%!^4rE=o!q9zP%pzOnewwQX3j%Tpes zv|YCaL+_SZjl2P(e(>oZnbplDJO)U7AP4 zRXy+%OsSj=D0})SMZn9e(_aihO%kk;K5i)SP$bmK*$yO^mYCLzm15SXV6L#Yh!R__ zIow!Pn#K6#KPMrf?)j;VV~@{YC%R6x?rx^c46G*{FIc}44V}$lB38J~*o}h)%rDbbgh@yse&cB*CR1%TYL_;IrA2Ax-;iq80Ph4e4c#UJ$q&Qar$kJH75 zS50@69@*=zI^<%;VpTaEZhS$V)enj2Ck0JJH2y_kbWIp}qi0pZ54yZd{uC~W$LkdUNdWwTVc;%|pfMe1zo@4>kIdF6 zIOGK*eH+Ob?n8n57Ku!MIiV(%5q(-=&%ty%dNDJI&K+h*^$4Y1{6-$L`R6b46p8+- zm76KE3d#J`APu7%6yOL<4m1|ky7<6}l+{NbG_#0Mh@M2|G8!#14`xjNBb0-Nn;v?p zp{rk;m4?;HxdtfT%7*EpA2G;y2(s0bn;eRxV0IB#+H;*3v&S(mUJq}z zSe45fW6I1!xjS&)#4*Hb%8{R55NLx3a*F#O7Tuf%lIB{7Cy$(isf+1MSok%CF^M#nm zyQ>Vi0HQGT|Dnl98ijRCf%D3GYg;f;+3Cw?dbei}*xa0V^g{O(?XQf;wFju;x)HRY z5o$gRQadD@#8mnGZ2!d>;r-Qrr*y001`#MV4cB-G+{b}1b9;NjY;`nKd-kbol!c;S zl-uuFYi$}Ds$Rd-&7KhPlK%Z^>u@TLd^=b0WWi-5f_XQpB^@Z%y+fn`3`)iA?lfwF z?$GDXk8nTdOWReQcVYtqGjD12L*v=q+QSQ=@it|NG7no}mXXz!z2_;vgC%|Dx{ip-h{n#qf z?-1gxE!4Yz%`PN!;V&GaP^WmQG$KuOwLa2}l|zeNiqkAGpY4ugrf&C29yZ7~562Bv z{4`@i$HCb*q!?L?I73c(mq#HHggU20JT4xU(8zQhbky;?7Yp}xXzha(=dmsIhEiud zy2S0@{ih{E-Go(H?w~Pp65sUryl3%BY61e{jGt%|_b_C4=VINKG)Jloa#Xs7K3|(n z=+SK_mvWMhdUP2MR$wkZ+yfB5jnFi0+R7y&uysH*#Q_R6Hqw3#$ehi8j#)I!!PJ-a zze9rbD@VTUy0>G8O;5CWu+uHq$)H=>eQ)tv|-W@2_c z!WyiFOX-*0XOoR{X~fU+^(neu@6hSEP zDCa26%SRIHQyr7XbnuL?i_S{RNX7@V)+ zY2tACoZ3z8A+Iob-Yn$8>UApa;!iigPyIMk+^a1phS?D4z|y$bwK+!Yx}UfY)X|E% zBd4iE&>TcqAIFZfvhvzuPSy@))Jm(21M7&Hl zCL$D?{T!)8zPRak<1M{H$TZ1o5wzuvkQFW2*VCHcB*yvO(9O#}f(v5z$GD~TE0Npy zTrPi%ge0P1{TNKd^`|1NZ(S6XM;yHRN7NdC*Ok`w5X5Z2jKzr$49#H|t8!A2>AFaI9A zgVn<8Gr3MW;FcAg@=lxj0(c(N%k`gpko|m5xBZOsST6xbdIVW@(HhW5iPWfsV&*`7 zOCft)ZF^SUeti4{|L{V+wU7U>I4;UXG;EHg6$3IB7g~HFjhhb@?6?)}Xl^G)&oFU2 zGYS~=Gu~Pj8fI5GX)XCxA{y-yk0mC4!?*xystyN}Iuk$1xjT%rMk=ejr6KfX1^o*| zlNBZPVOHKMeloKWm5Y9S7b6~D?WmqN0i^ye(KF4?l}+lgpVNnFP}!&eposvI(6Qg6 z=W&ilBZ}&L1C70!JoqSF1uYbdt4mArQ=yFFSWlI()a%DI!KE!jX^x9b9p+)CP67D* zVEaw9^&r?c1?(66Duc?NpD+%R2V}E=LF*tI(@0~;r#irqD`a>**_VTl{IFq8;O!z3G?KyN7p8! z$qnO#B|O`X{MK8dZe92f(5dRj(3QO_4eMcB3oQ(@gt0wDAy@&ZrJB8m=dKPnrb)%ZVog5goPI$b}BUPf+tMJdauE68** zhmcUc90U6Em7X^rLR4Cu(iGg3U+=_EtUV2!IF$zZ&uguOJT+*cH`0;TY|Z|D1;j5& zx~ZSE3W$B{ro!UE_Ix4*{(?ktJ;n5DFS7Zum`aBDweU-#1zF*ka*mwYg57UI*$AlJ zE?QpJ;-GJI!Vt#Z^Te$@Hk%li;?Hj>~hp>k&~KM5VZa07UA_!d@F zVtIP5SagNPF1&nSR>uj?D^s!1xy$$NQu-Wq>Cd<`9`7z>N*{aNGXaOt&!GjdC=6_S zWxT#3{X4D|o?{ntIIQ(C-3O?kY)0s)=Rb*ZhJx(UqsAOoC!ZNRu^jo&u-;XKv8tlQ zOnk2Fd~qM`-#j^@2}I(798!r|H@Co$dZzWL02TCg=#|ZC{J-6NoHUx9)!rqkvGi8I zfJqRBGhJY;XP{V75^zwShKD^tcM@30Q=q|8!Q;{=Owtc&aRe9z{fs8igB9A znG6fJy0t=kHOY$;<-G_Bby5?7(e|i(F!#mC(Eqc^qfeG9_goI( zbIV8xI#4{37bV_U`eX7N%R}#14Sqvp<1*pIZkyD03k&Q2oJdX^YjoN4X&$4{?sW^K zA?}V2x4;~uw-W;P9~^3SR0Eq?#bcmS!DtyByz+`K1Nh7pSH%mupf(Z`sJ;KD+pxO! zRWh<+uW1+j~MS zN!`^_SNug$9!$?@p?N2~>co+;Hnz%AUDp^lYX)*cf_^*`BA9kZrKB4T>dv0oTL?>4 zpHb0`z#pS$wB~2$h$I$yks?+|L==Pvn7vf($+0n8_uX&J(RcVTJ_^j!3)R{VkV1pL z=Hp8x@$X_vX`2gPegj;2UCIMyAc_igs-W^PuOjs_2@{Kc_}aza^p*S{>rFxjVwX}F zk~j1E;C_-7J3CFO9@9BgvNE;st$~uFva6l*6Wzc6pg%U_O;%?Vu%`Fc%Ihv?wjHQ@ zNgL3Sab0p#7f0hlD>_j(eX9pbxcEnbjTu~psby0fPQYXd9zu`GXg-h z*`FtY1nq4tKRcBtk%kw>Q{iD3Hb{UXrk43ay5@xgW2?Y!-x;8?gDrvKXDf#uA?Yn7 z`ac*jeb^aMVfOGwByI#}nAlRL&tbtfXm&M8zRZvlNPVA&Hhd37p*=tnMV(B-5vNcYnG5H3`ui;=VP5)qjUUU*97eapoi(}e3+-=vPCac9fc}Gs zFV8SfzN87#M*wH>o7bP9n zsQHI(R}SK#Y|VAJ#UozhY`Hxb6XDMU)ylXGwt7cAu8LWwIv1Tpr`OH08NQrA6h6YomEX7p-kUWAalsd|X>pv7!{5p3<|@St@ktdqHe~)uj9OLm&^n zdkCa{)tSXgJ@zvv4L5cYp?e&ExRj*-KIIeXhYg=B}htR4}uYhlAUd1#_m<*KZ zK6}Qbp*(_BTvhYP)$6{-fu_tnwC4vG3Kavk{Cp%`mBW`#H^6ZE>4+MmT?ia|30Sn9 zt4Pzu-eV27z|;Qi*k3G;aJuAik;dyTDJHvnoeWcBg5L@G0Y*N9Bo3Lba=e`mKX|Ub zCzVD}jE2exY&)$Y-v1WpRLg&5fEX#Lt|{V&RHspSbJmSHH+HxaFMrTv51#U3@`hYO zN^q)ZHS_AEmp7_b!+(>ma=sBHtF$#E6}PwYdH?0noa;q-D_D8(5$r8IcP~;8QdIeM z#ubh3G>C*}_Vxt#VK{{pr4V!*3IHKq6N}2yd4Z~+eS9;WB zH7n|5X*>KIYM#>3uDZ0@8IP-&(?54@7C)MbcDvTpDeC{><1L!RiMeJ)OGtf|5(MCm zKkm7-5kmn%U+`ARm3!3kKB=TFH4~y@XPOx9&LCmfKx=~P$7*k!#sU_zd=&aUf)I5@ z)V)8aPJZ4jO&kHP3ep;JA^cSO6#ezs*I3=g;b5fAiV+|-tQr~#x>g;g_ zub$lHl6wSwm^2QIb9tGLOA=EGWyeryx03#KXPYS=oWkB#oldkVmvefvFO>#{gps+( zx?y5DedB_8r4F#8*^Ojnc#w!O(6*;bz@!c8yQr~kJ$B^6KX4A;p$kt{9(EW)(c$3B zl3ENoYDNEKpC(RrF_So_ii$l}eZ?Nv(|n`OXa1 zOL2^qALtLMO@-r5G8RJ!y0ySu9E&aa4r9ek*o`G~oOxkH1tz)>#Wy_Zt>5KplFT46skg`1(;42(%5 zmCS$VJf`6w54S#d%YG9cYzV80bKR)yN$d2R_OMb+;wKFFes4A8NP_zZ2IdB5P-S^$ zd=7*491yhN$TQZ-H;IfQQrAS zZ&P8SO|7SYg{I@s{TAA4wn$aH?5r=`+|b8PJQsy85?l|gbz9U8ybl7DD?+f`s~Fsv zq-1pPmzEj=@tV&q&Sf<=jp5L-){6+Rb<9Soy{G19agIVji-J2WY9|vaP{b%Bj9kkr z7rDbp%GpSv34EfLjv_Z$eiVkAre56CVF{y~gHA{ljy{)nvfmoA;vmZv$OfUOW8C35 zFQzgNZOBn$wfs|>a!$qDD-QALEtVuMfaLXUZxPQ(nZ>;1XY|&$zP|kz67E}UG9(a* zSzQg~1R`Q{O~1nYM1x@OeFDn7!38XTfH$tVZ>3{qu!}HlC09UmOs20KW--~o#o?op z9+%np!)G(qdvHbQ`E&-+`A7gEqVan@mnNI7#b0X|y0rp*BD~ugD>{&Jysg2X9B)4g zq*@QL12dLsouYL?0NLClc5K_T_uTRqy#ST*X=G_#6z?{8Z$VMVWZSL{E)7jp%|YU& zN&&f~Fl4}sU``JKTq9qDJuN*&`Wd{ype%81FOPtQqu^vQJaPC^pizH!7TdLXNjCNG zJI?TzR9t{-Y(uBLttP)iL986ZO7;xy6-lv~YC+hH*0s6boE_vn5)dOEtiD>}7l&=jVHm zXthgDa22H#1^6yH>bz;JPJ1r9Y1lEw#W``#MUD`qvz=flo=1Zc8DIhnw`DA(O$*!3 zA(2Z#M!}4jr@kP8x*41!QHu3Ed?73^pj(qOHKRf}-Vp(rs(T+CuUHWzLc6KxP*U8` zBFG><<$z#5Rxz?K8$5K5S&O~_3F9|!kMebY=nHb$^lC4C-*FP%6-gQnqvpa6%eU+;`*< z2sk;IY7g7EmRD(7%yJP{FOzTX(VEw~K>-C36SdH!7y=mGEu(fb~3p} zl9RmudKg?er7tU9y0O91Dt2({7Cu&hu~7wTbjJ*4#D1j6?i9`?>ek2(k_hS3faV%) zuol$Z$%MM=p)2%$k$>nHqsn*RNp3JeZ77ZV^29s-e?{P)EN=h#@E~PgR!Zbux(mWX znpqUy30g(L)&Yrz#&Q%6YDlDlB-o6#ZpbR~4K&h)F{W(v_6&MEv6H9CcKI{dO&W$S;urSOZg(hl@lf| zR##p@g_ejo)yI+c9v#*%ps8+OaYtr-&2z#+VE(HUK(`9|fF%(ublknmMW%pYZ6^_`|)Fo#Q!ePjiprI$j5tZ6f9Nqysn7ZjJVKUnVTv{DvgQ4RskLnEp{vY9 zj$Z}^<@)vMPh)jrEXR1ibMRWcfx~P|TzmhKsuz?v2(N(5;KNc!szP#Bq=BiBgC)50tw%+}j_B65b>3wPMeVHWIo1f8QG`aRlm z7}3y-BJ10b{@k3c3z6tXMAgA&dmCaXmEt_`Xx!gy6nx&QakWKLp|fgve}-6^OTl0* zuJYD+&g&FGIe^z7M4i>OcdhsisM-9QAM^fnt$pm+m=s6T(wO}coJ-zjeQ5aiVUZAt z>6sr>{0{LP@5Rv)PLnOV?maHNj!5Ho;ya!+)X0AUiM#z2erGe7OrTgGV$_!~BY)u6 zTn%TFlwAF^`D{H_=EyV8ASdomHI!5%N43}~yjvs-N=s}KevWND`8FGm5hPF?u-m-#ctQW`Cgn$ki57mv^3yaMs?bOqOIpXx>o-Gm$iwH@% z0jgg##L`vAEUxt3WUFr8UpMgUstV1=QahVEwbR@i5UWdFNlBq@51D8|bLDL&^M^U= zZeJ!$qCQhq3XEC~uFWxMhyS-Xj&i&R>i?RSdM8#uzt22;>=B~? zvT7Li81rL_A5DHkbiqBDXd)&ZmT$%;SEmJNqjZFTcK=aJ^FOZ4D^EQZ#zo`^K&tp2 zmkw27NAd%&n|NIwUnCDhEMUgiOWpTL@~w1_+HuuL>(BouzoF)rI!V zlB)A0%R^rJ2k(s)e>ylHx{ik9ivKJOrPws``4UqE6KTai4cCg-OP2? zVf*M~uGkAPzXT^5sN1Wp_T%f?Dixq?!*q={Jl*fXus4`Em4%1W@=N&EKu&|SIqOSS z?LAnO%OF2+Xow!_&;uJXx3BUrl9n|glaV|N1Q`HI=~2RHmH%P>O+7cI ze(H0}CBAUUaHqcVXmmx?4o?B%1e?!yDrA2lYrJ0mW=m<)|4_h)Gh5AZbJc<~H1XyR zwDnmI(Vi-gGLntjs1&z%K8j{E9YDd6N7GO9;3W+fjRmFiFjtd4`dl_*&_8i*rwoh* zlW4aWzcodr3f;Y7C!87i=-n<$P8Egm&h2@+bi0JeZtIN#_Rh+K2FNvhvJF%mU7ljn`RpIp>!r$564tlWB zD%hOxMu6~xJvp@^zBmzgz;f^>Kpl7C(ZZ3AGc+ILY;U zJKy9wiikyBbM}LSG)LVrHZl>U6birTZ@eHA+&feZogY`o6FzV+eB=V@{w?$tnh1*# zXs{~PK2!dnv;^m;0r9rgu>dvAT-$PcYp-}ngybHl*}Ha% z4?BHIE6L_@ACBCo{Oe}8m0edso!&U2O zwy}tILYydSN3>_&!tJf86RD@+Iz1JezVkY#ew+%^EVYGoW9yAoMe{}qM(-PD9p75O zYen}ve=Rq~xLm%Po&wsu(k;%GWpobk@M#g_(q_d&PIQx37{ZSs8ZXXKtuc4-%+!WS zH=R|yFfIU0c4Sd6b$3Qu6f z@Y}$854A{zGcb+EcQNwB!*q*pjQ|BzREq&RMpmI(YWd)fHor^N(1cAZ+%gyD5k2WV zuw&e3mpqp6w}}Nqbn)Q%eVrr(5h)Tv%Wytx7;_we#OEWCZUHmcde6GT%SE14ppeI9xN6WvrhGS~HfhN6dqUzow4M;JQki+>JaL&?| zHlW`6aSUi|9DMGJW16-INg=*Q?PBoQXI9dFnFifnxg)d1ssbNvO2ui=CYU$EU7+Ui zL%H(+GawV&BGlI@&BKRtnBcePN_WF}4q+4Lc5O+PST-LcR`vFOr28+D)n2dnu$j{7 zO%^x%A>|EpvJAAn*yJ`CPw!3R24Q?2u{>o%9j?EPAuO09k-P3@uF_D zsMH3JYDtd4=zcttIQJF!$%5p9q$v7YiBe(#s)sNk0Q3G0cVECF@a;B}1^A&n)3kTI(R*zmLeIm(Ye$FrRi_2FuvjsL2g0 zD*tWc&&LC4@d=qvYClVhpTwPYt!+>6>07re_9A$T*ohsp`Nd`{x$LH9G)@DcI+S znF;L6+LyRd{Jla#Sw$av1b1N470KcTy5~uQZA$lUxqq`hEr%e*c1RKBhuL`qc3)p@ zlBNwkJ&+PebL~<^$4bOF=_|8m&C)6_?2UZmTcCSg4Fb%761w=(msIl(jCpqV>Hd7u* z*ncd*$A=Iby0LSo)><5^_r_KZOc5*9O&i$33Di$I=2iJM>_Pr0Ax`l(o)S03L&kY= zl^Tnj17!Phkb$3d$?sPOp&55Kx4BgRApB3`uqTc5eaypjad3Mu9*X5W|5^r3L!hf+ z2NM(0Wz$7co@Qb}tBwLAaHZ_B;#;Wej-!H`XM@hW(Oo>}Qi(8=Aq#diXXs|LYv>#qsDN^m84lW{WUnoTgjmhZUT+9Jtc}usZm1w3YgYZN z*0|Ae$6$e2C@9IOXa##@JXJK5krcP=@087lcRZ3PU> z2Vq*9t5#MkWs@ASnj`1)-21@vlCCPXV8or^9dYI6itQ*)FjV+y0{jYc=X~gj{Rl=V z>2L4Z;ECzhGrna`xQYxW47+{a!yM|)I(n7tpN{k#oJm(W@zwh=)p;|_Eemw$8Ay=3 zCbvG#Z72nJC_j0vfds*;ExW=V1snr8tJjlqd9 zyo_dSOh^5sxL)i!>n>`(uA?lW8Jn~HwrM+1lTGpcx7yl62%$562V$m_nkGnZQAbEE zd(1mZQb=miKHw$};mvn96Nkn7#^Q zV*)3SFpPz&UGp_W03bqYbps>+C1c*$eHlra-Xi^Q{+v6Pb)j^YxkoAWvMS)iHo3#& zh@~y$5VzdASj!$~^sMiHV&nllQ7!ld`v3*&0;t~ut@v$}QAT5-|>>4?MaC#j&jj+q}r|0_jkqG;E676h$#ZRYNw*}S2J%xy43dmlOrj0iO&vS zY?Sjz$s&U=Re$V(zYgC6Vd+}3EI7dU(jC&r3paGVR>I^K*QIDfz7);qEu(=;-A7lr zz#LCAU0CS`ATq!4XSPVRs+6-_fNmTk%4B-eZMWjmwjG2==jnThvy()gB8(SF&79ui z1GQPgyet(IFps;bYB@&Jnl6W`CkJ>L?3IqhKCJntv+6a&2o~fm+&Im)|p>h6;VNCJ8R)sl)i?atC0?g1*0!n z;Vz!Ui|o-ZLb{s$hHjs;F?-^hxAC@Xyh&kLR=P@534#Z%1oLsqi!|?|);c)_)?Ojz zlMRyLX&m|!{B8gyFutd^3WKjq9@P^rtJSJy;>DjX=R8}1EJxNNm(@KuM=aJ8;I~*_ zwXT^{3JyjjgV=x|M&N<-Zyqr4eIYUOEQ$da(UdBiY@^lhd7+UR5Pm{{M#x>lYD$FQ zU%?d6UcO5b$0e21i~(VN7h&qcJFne4y5bH8I)>pbc-1Khmt@sFy`*8x!9?M5rzul< z8*+j#YY#Da&?-pTr0NgFaO+v?+09(h9#8$Psi7nkr>vyud|4I>A}DhWB{_R@S=)j+ z2VuR*x~vI^X5jq~1A@g`SVMPSSl-QRW_UtvJI(w5RLTveW*jGkIZtUno=OZb<5zX% zjSO79_>Slf2d@1~47(*1{d}9OGu2l4kFng&VqhHZu1)b2q_n`!zzqIz=a*vz3K1gW z63K#js1m?DJ&+Jb`?V7 zF1XU^fID!Y_}q$>Uz7D%hd0#e0XkzsKVDom;oTgM>s~%r)g3v2 zaKP9z+7ytxGtM;w#t!XXv6|ghAOTO;(w3Jvl!0 zY(Kf~umaD5mT6Y-7tj$MF)#}*i#z$Vpo@V9;Y^q)QFK?2ONp`7zF?g4l2WrdNRLT& zV&zvb6^KS6>a*Hr%)8ezlK5w?J5KNsi{893*php}jkc4H_9ea4($QZ&qidY`O=nK2 zuIw8{ae7EC#_5tjIUPGZ(bSx7xoT!x^rBC+3UcoqUI=e?$+|3K{N?rTaDsvlwD0H2I~N z@Zm`chxo&5I0_D>cfWd+;_%;Yephe9=oX~6u}PPf>lMt3Q;=jb6whzWea!{Ea<-WW znMA5IeFx(Z_ShK&gluZwy>WsY5Cgg?vx%Ph!YQwon(_8%AheRxw`bl#4uwpjfY}X~ zLK^Lb52CS+Dr>8lOOqV@q(%Qjq;|RgUYB)a6`GiobI>VDz2FZ1MKlaWkC7Q#(TudO zYYYdXGeQ`GYOx=PSHLunt$De%PLkhTq>a0sDq&YnuEg#BBn33gm5hy3F3SHTU)BSX z*rt#%9|UvrnkTw0*z>-boIC07iXlf#%St7VC2R|y*^{zU1`M3mLSrTH^Um)Yuy0+y zlbhWYdsRMEu}R7(c?L5&&rSPXrxSnLHcbC&Q6EcF8}`A>XROX=V^BTc%iqbK59K8e zp({Ujm0olG{4~k&$1n+=b7^-7?_i&LSuiN7mC*RPVlZy)l9sr;d%Dg=+BHYUnO{Fv z>gfdG0>vhN)=7&iar?Te-lqbGUob_RKsnI9J%S$uB@*^3uaa7YjGpW_&-iRA9Dq>{ z>{bt&u}*LT+eY2_QuCPT`wEf#7F|K|$%lkC1=@dww>Psn7H{~-oV?Gse=&)!y0VSH&exs@+Av=|_K;Y!n9(ac5Q5u^1$N_%qhv0;6~s`6EgowiWEn$ptD^M z7X6a(A?~=#MMl;9kLX+r@3x)EzBHSN40ouQv)n|dRH_huUVu5+?>3cchdIfeKBfs) zpPum(B;HphB-BmWY_p0jIouVH7VB;?_WZl~1{NGH^W*I^NU_q2JO}2QD1*?FE;0B^ znA?&HOPKMw1eCverH)IT}+AKXSGTQRT4^rDd7{zigVA_J$F4>R!#RGNe(e$ zTqJDT5pCFD1m3T_&9KSF3)Sah^K$&!xrc4!Qo?Ib1Cfx{=IrK^J0N22?I8rcK)?Pu zI4o)2MEW9WUTQob>R(M|>CiNC%F~8kPPtDXxZ9<7HoLTxBchTm*|8i3tGzWUhaHb9lLacRxGZM6yG>CV`!EWO?U=%yvSQ`_h`cmMfx$IX+d0ccvfBtA9paXanwr-dCf99vN z@l8+NPF7rYH3A2ZpI%r}#Rj^J#}2Mae9xreuH2eu8uB(+Hrll~t?w)!l4#J?QHb@e z`sbO#xvO|(kB*v=VCfM(8kV(@vK!)*xIjiB!_>*CBVxwB^DbYuhp6WKB=`0#I^$yZ zFDu)2wWS7LF<(+i9$TWq8{~7+l4e;$e^1Bu4IO#1-nX1Np?@ zwd*fwL`K}r>c5Lvls$5&)tT;Ku-%y!&shb;t8`LEnMGG@MSP)L_b3^#9gh`Cw}uIg zT?0uP}#~s#S#3M)hvS7_!1}*MD|fwyJ}afHysj zZHOCrG;@mK>PWgQFzaGlDOhC-!==XgDBDTH%u4+DKcgB3ci7%O)W&-;%G`vMaWA2F zcb;GoYC*eSQT*2)JXbsu%_^=7($;)rS%BIl!!^H4-{Vgu>i{PNj!XxvoBB_tu}Swo zj41XbQgU9EDg4?l#Nxp^Xq7C%LFxbwTuh3}^eZ+TEx7>8rBbNWHbLXm63nWw#o0k! zJ61xoi`^At7(YYX8X;D0DDPTfVbq&+gN6G$@hV-_M?ZLRaiGp~{?q%f8kM){Qp4p;0%okD;WLd?_InKQ`KxTvqktr>=AvO`^dtg)o;v75Km! zvj(vEgvIDC-jOBKp@^&M%g=i|u)1U$xza*9<>=!#L^zuG$(qd8W1S)4qVQL>BE@ZXeXU28Cwd$oxwD)b zz=}Pv_HBuGxLq9cNJOg*C<95=dKU4&L?&S#IivOwhQct>M~AWz16yf56J7n&T0RB+ z{Og`A1hPs^xt0G`R@H?Y+SRZFajD~;NEelM0Mc-&!bIpJbD(LXM2?Qw9AUAeWg4Zo zQ|mUcwpU+!@;q(uLt-t6EWeV-2EFG~E*OQvuaM=p_ZWFx_zfaSPb(+3}jJF?Q1bpS`|25Y99A8>d$2w(+f(PW8w& z!GTGTcHD0{RK#2c!ujHzh>SylR6iRa7hm^95drut%DP6}h;vVejnVt=jNj=UcAoSk8m@q@!}^HQ4A$`#2K82 zxo`5^;y2$W909PnvV*sdg|N|H1BLx4>Q!l^?VzbtXgKCxWe;{ne29$0x_vBk*x{|@ zB83Z=gIGH^(>H(M{Au;DbCgFdFC$jm^?jaUg!OVNE=Hk|zX1lF?++(=%Ovejk@;nE zWuc&rbIm*F0xJ+r8E^|>`LE~=sUS5z&*fEgk7WUei3@6_eGvJa(u*p)V=kxOzJ+0? zMBNB6<$+F6=kXP0(9&CIm?d4HXPG{m>-@J+WVQAZ?Om{lF5F!_no8S6oq-vz zF!btljQ5FtBCuGj@oMHVuQSsICov{A1m`wtlO6On;oWs?W7r3rVHbt|cf`jQ0#p6! zCt6Xw(p_!oCnuTE{V#!)4diu#@W-u<)kMSNKF}8K8Q1)rI``2%iq}bCI!_R%smW_@ zV~gV^VCAiK`8}j&dj<{{z#B(036Wpi2K>0+T7vR zX3bD5k*@nH)4Z$#p&4YK1e+GG-Ci4;gJf6shp(L$&_Lpq@brA8Uk>NC5H^2yEfxc{ zSo}Nb1r-(h2G54`X&(YDGZ%>m8!fyX%r3$L@(8r`eg)tJ?bJ<#0_A@z?=ue}jO4Y1 zhl5lmSF^#$B7y9jBcltdHfQJp2|htfjVR@Q**i#|+JsC-#m_QL-uaHW*?d0kD}MJ< zPAgo+RdT8YVAN7UOt0pU({12A=qb7&R(OlKN2izx;RvINoy+<_?TkoGE~MNad@2T( zVY9_;RkZqEV{Uw-pM95RrlWMZSxA~Uq;kG%EejT({Wzp!z=V4T``y|>`HdP(=J~(%qbk2^H zJd`ODPR{0M-@7=5tSk?00jXMY0ENy6y~`=eqTe@&*?5jJNose!RgN(doAgi-V{AH- zck1rmReuQchpveh&OM~D9FE>z;9gjTN-x825s9X1E8fNi*5q)@aN=Bu))kD+mYtNh zP%jLIdrfj%YeQE;Oh58_Y&D;#c58Wt_>MPMcH-bQ9a7@S_?UtCW=gBVrhnFbQD>w9 zVmpfK+P@%xP;DE67g=$&PoF>7*#5JJVTeHnCXK7r6S^70s)wcI)N(9(Azkq{MFqym z*=z6`U<&<^v$DFBuyrKvs!OS$qIuDN1$Z%K4f{-a0vGf%arB)t+@9a=o6urN2 zfRLbUy+7{Ent()?sz6Ni!#E%E47)Bi`!G zyqba#8qcusK#Z^DysH<6MS|L_Nfo}sUJ;^lMmp~gZk(MdHRlu=f>hRDRDQH!;L#VZ zr+4pAxYO+&CHDKRzZMmhd#nMvSRLpM;<$WnPc?zoZnc8YB?sm;Zj)JUwU$^Xj8Zg` za^Os^xz%a&j^V}cR;p~}RHZ)MF9w=41ZK#Fe&)|?m*egfj5Fss+Xu1=%s@Xh#p|Ba z^TJp@Vuc!z+UGWdTu`!))wJ3V#wQ?T5W#|~-_n7g=bE!!Fv)qtVBD6})W=vR^;_tb zx=g6?j0m2W(m64qB)|>oXbPUQFXmm~lU&P@n!3%&{Vk_b_!f-jg&mB6Fo)PpNGB%` zk|;e8DZ;W8A6zQTEV&vC{zC#axVJ7H762s%lGrC9rgUe+2}X%;2U{i01$2?--73-b z+Hqm}UrwnJC9wBemZDb}@q(tBo7cDm8YWJCOmNmyswIda5M&WwRY73UgXMN#O+N`) zP0o~R$Ddd1g+0DTC{&7^u&6;x->jS_B%t`g;C!BOOsj_i z5xk;uF!x8XRdg&*fK)tT-{%x#eVN1L8o)s8F->>DoF8h#kpb#eCkO2teecH=Y{H%J z{Eu8-2Inin#0^<3W-%`zA>?e}o3N2}@I8pD6m|1jtO0gHo+K6Xx_XDGl#9Ri|ykHBmJsavi+uMkJyG)0Db_AF(igX0XphD1QOT7UW2)a%kY`JdvR zItbieE%KRk?`7y?ahQlTuXab!wCrsi8U24vWU0P8R*HJTg%qKk?X51vvBEwR5)LG^ z*~nErPhTJDL_sET6zemAWO#*3U8t2m(INg&yP~3vCxGrXVG=#1tczdr;iclGg#sp% zV&D-DIXQE&DAC|VYs1;BqJvfyx`5TOhrg=Wkiw9~wRi>FrHZQZP7%h>!y_&q)t`{X zsnsNuC(@0I&vJC%~mCUVQ6dXn6WCLoS7BmF9MF3q|Yaxc`|^$l7h z3qhPzjidFgB?RECZz!p`_z3%apn_2eV!Mu3kPQ~~QHW*G#z?g49r1}=0xaQ?tP-UzK z1`Ce7B4nw5*>g131=66un=FL7&bcpX6MoY_S9DYX9^~+f8kP$R!@Nx<$bLuqQ`8Z4 zpL3Z00x*btPZTA=lVvnd^$v%LrG)9Md`@HP6Yg^r=w4Is3J|XFnM^MMa-R5@K3g|1 zXudY;!?%E%Am!(1^!r_|+|=RU#QGAwu^~vE>N93EXlQVu*1Gz5)WZttXMmj_`cUe{ zTd`ozfiDwHpRQo)qU zXEH7#>phMF{YnFT28#{ribSjFQ%abCdo%ZRevhHF1FB|6J#>Bp(7^Q$S$A1O_T-9* zPLi`!!+HgSIM_L9I-yHA2wA=|FvL_~Rp-XMmWcwRE_43AzfVbThtvFQPA+ZA2orHi zbAaJCaCaJv3rD37(JF>oVA9_t)DzbGhBxYdeq20loaV3>i$C{{bPTm@bsCxIk2?QMR!bH(MBa?OhqFs!iXsDvC>vk* zCJD076v)V#s7|yNvE+-%{dIEh!D2Fb|Bk7!dz zSNrB{?)GT$iN|=rW3z6g7RMa9@IhhSW_b^_O)k-IzFDB+ZwpYnai8XUdNlxnxF^^O zmRooJ>2yb2z6Zn>evZ?1RosN{y9@gBP5(==dp}{4*_BaOYP@^xfbh*Cim7``=HPxI z{s+*X+EgiL51MZbd%Hd9GsfQ-jHCG=ZhU^h`ai=|1xzbR5U5N&#B=>iKO4jZ^D(`^ zc%Vq53ty%w8sPTG~LoP<5AvdS2{d=+3 zm717zmTUtou^GH!kg*A+FaY$sDoKtI+l(%5onA7u+h_7V)2t!Aalhl%k)>~qKW7g3$IWOxLEZz;{-n_Y^f|MTL_CM0#{{?~2OL*~M6M&?_7r2-;a zv6`P_-|n^#b|Yt}k`F>j;@Y3oQ$SMwg!V;-agSU#F1e<_NpddV&>r8FF)2*woep-+sMkPUXJa#a z0p759J_Gy3@&MDmfNh-ylr(m39*M2?Ys4Q}f2rXJ*oyv+s+u5KNE%i)*R6sqZ++`B zURnvhhaWd$-N0|a0{t5m2+O)jpZVG);3()&5KB{#cs{@*p{q6#8S z?}}H(g|dNMi7U6II)`dY{*u%fERBsLenkjo!=}3*AoWgK zcCdfSZE_cihJ(tWx_4(na0iSA-Xr7ynuQ8^P4;B9(`2R}x%3U$ArxWeIEm0~=2dMc z(Cq3J#In>)+ln(p`}M`B`gpUu1&PQ~_FgHOgNp-xg}Rsh|?a zWd;*k{wkCaVc;Q4v5#Z#)+^Mx_{GdN389vYt%pkP5h~eRK@l5JLYyV|@73eaN;oxW z<#crQK=7)#JwNN68`F-0J1@nE3~QvysYR?%zE<-9LAW}k?i_Ooq687%WP$_EIuH|2 zg_oJJ(l!iLJ`^eg1b#u_Sr!eAyun4rKAn7=V;T%$ufpJ*?%E2-%-Wasslr)iNOK3i zIfoow(W6!%GZ^|_B;v&mLi#uS{k>ge`dYh3appS!eY9>vKm2^(Uu=sv_)&2@?c
=j6$ zeBgdpNh(Yzje55GSc?3=X#CpY(M;}`>m*9jgoG%UcLavE-6X=@(o%nz#wWPF6@-E% zi>jV@#H;T5H5!mDwLn9vq0N*k`0Cd=I=<$VDJ3j|eW}T{jp$NCa|7^(bo;Uf#|G3D z*q?b7t>hRh#mub+QNt}Mx4H7Ra>6^kt$H%z4JE)v9>3#Jzd=YRhic6n&BxX_I?l=4 zEsW_duNou?u>ouL`S(L{*T4VgPyGROVM6JuHxMM-uwFtRsA!ke_N6cz?U6`Ou59A5 z^3usBg@4f0wJL&zL+oo=RNwHg5_Z|a)+sT4D#Xy$YQ$KMxhm4%649)q4v zEEk=*v(5l294f%o5CVp{v2fDr7_av7ogk$;o?f1M*mUWQlagP#U5SM4_E^uDP`rok4R^!TC2oVG=9*tt5{+{j5 z;&Bn#JrnEpH$>|1Qkir#+FJSdCG{Cf7s3J#Jp^3PKxZ%1oaPl*L@PdBExoUchg0nJ zg|E?#>=CMCpGn6vyzH=Ywl4MV6uUzDcjcmF)GVwNQKpO~#!#B3OF>XEOogx>ixlacdB58?L|l*wM1`9r6mcBaI+} zb&07j5uho*ktPNlC74o%Da#Ar`)bZ>uc@AppVY}!LdPr!A_hA_%*YAI`D4&cc z0n^rkKVW(h!gYW{XQeouKTva=Dj=5*^oB9`6|G2XhXypw(l5&lkC8lB& zVcK*E&qfnu=_kA|18@m#Z+w3WGu2z)3#PBZCQ`2<`-sajFy^!71&%6@oV~yyVGt{4 zfZFAbH!*%Lt7i*kmQcOxA??7VktPrfF2}{;bj0Y~RKcT$H8Zs5ETJvqyWkfd6UwAm zqP=!|SEr=h#Cfs}u+WZ~`mePLs=u2V!B8!Zn6w00flSqD6+?Htxt9Un9>CFJuszqzejewD+=si&+B}Ka2<%wRi2!NQ+_YmbHB<(551;R@@M6yin*L`aY_u zq?n9#XTK({FBOu(P<=|cGi=DHRPBr>uxTIHAyyUpsJd>;`nNsgP|}wio|Dt5z41InA=9tg zuSpVdA7P#SF0t}fr~ZPmz`7C-WgVs9|1rR1@-TqZPLA}^>Hyz}qIenL?{1j^`?&%o z-{SlS2syWx*LPlHv|Uw_*gX80B(-ASTp1ZkdydISwXop84TlqF3-9qS(QSFp220O9 zI*Bv$gs}&R#nTl9B%8oSr8ef?t(@dJUtRm+E!9&79^DlwISyo`?bS=KhzByBs-!xq z%p6^OsIfVkm}LW`7Al;*gD~w`)om$d`h{z!%s?6xlU}F`Lf0F%X~y zLcd;mG<7P9;d9c9C(ja8E>eM&to>ecgTu?G=(S=^hJ|`3)jEL%`2hBU01p!tIqWKZ zQ@t+96G_36JQh%Shv>hlcX{LpZbhcbK+fVX_?5Ki^n4iOfsJBVc4*c=2}qx1eCDwt zdSR-s;mXwJRSu)~cLT|ViQI_tpsg4Az!5XP(f-mZ!U2!_Xpe$(w&_c{V*XynI5I z56A?N`aJC}R0RQ=HHiC?)OZ?AdjeBR+|22^q-bMq_{l}W&%E4152 z0J0R+?=?prt>}}4+1q2$Ai^BmzzIHqz}tK0`L0^;uV)R{u7r1QV`dn5F{;%NW z|Amu|CXz80&aMjy$L&VjF~Vs2l7g5UxhnQxfwKeyqy;Z_EH2j3(Vh+h!U6g@q7~D9 zg-aC2{&joapyH}##_Dyv;QZf-nnukAXT$I$)RHa`SjrIFFbhy7|I;V;BOt~) z+hl2dS@J(Q8?^;av5)3IDX=xYteRv@FTsvQ_LKP;M4b7Kf%QQo^1-b^XA{zKZS6K_ z&u$x_i6Ps0Y8a^}zjpvy>w*$jTY~!+bVhlZO3Lrwy8;rp;@*Y_KwjSDJZ_k|A#vKG zBNA|_JR&4NgjD7KfeCQQ%3usDdTWAHr@m?Q6UPOcFWu|(SdQt$VQSzpj0oQxM+lo; zgRcAKuQ+1r0wamNGra1Qxd){!t+nIbrlh-N=Q1|;x;w%F+qY&&3tBACB^pYhQL!RW zP^x1?m87an+6FE{L{#5jJ{hXkVj(R{<-0jRf%J1x^Yl52PqlVg3T@9nCPDXUENKu1 zvi=N{H5|I)9|O_0fVIEngl@PFC0G#|<;LvxWKbRtGi{dfP(UpBow@BQi?z3xz%kOG zx6lx-*z;<5XR1?3_70g~X~kky^A=A)K}(K8RiFS9sHLhYh?M{2w1*VoFfppSX)_ z_W6~#0kWss^Dzm<6SYM#m}?hZyq1Jnx}_7mrI01bDHv>R_T#ZwHL`ql_Ylqg-ha4Y zsRmzE1*+=_HDo21BzEqwe$ftiNuJrgfcs5GLJb;I8+?}$m@LcVTT({DQm?xy*ypd> z&ADwwzBttoa(&E@`|Nup+jp}cG{v7)3|v*Otsjobk|efFtQxq;7vDsm*GDEnDg^*> zF$WbOpyJS42CJu|l5T>1iYf<*oerLUG22UvXRckwW{T^)CVlFmX?*>S%0y@D*-TtA zfsEsm`&ozUQ~Oen+njj_xuscOt9LCLw=GcBVDNrE?96Kj<#^O<{s$8dQt5F3=zxuc zVi#82EuL-NXAD#j+qL_O(OLXC_9`HmoffgivE)T^Sfq-oxb5H|TEP;O4`}eDAxIyx zTbgtm*m>5kIIFlXNCPu%mv`UU^&2Y8u})lvNGi=>M27H-~i!? zHv2FOQuTiFixHleHX7l$;K*745E-o?M{)M%^k_TUX@?>(cJlzGaXhlR8Re~1^3?Qx zjlHI;6}e$*U=YGev9Yx@2OSmJymLFc>2?hd1Z-DLM1GM3z5|jWbzD7MQ8eDsbe87J z3@uk+(ail8gT;cl;g{yaQb+=kzWolxhL1#Eb(?K3f?XErO94PYbc;4?XLmum;DludK@v&we=s zPFAl4=eMGeR^Tq8F);$+auVQsvJ8}``e;D1r9%j?N-n)*5Q;ulh)n-g`uB&-01WKDb@Sk*%Z- zg8%t-%IAY$QhmJ)0}DLq-~5nLs6|@SVczqhfOF8hASNnuTNd~rV#x4pTrf&73Q_x8 zRfFFgM#h_H9&+CO# zHZ7xmZGlI-hui_%jOlEkL@khu!Wy2IK! zBbdeH1)945{RGrTmaVquK4)QpxJ9XS(`F71zm!2sqDQ~j_Z;HUlEc)`do>#o+Y6GV z6d*C*bbD8Z^tB8SeL)pq(n%r1;C(yLaW|z%EDEaHce*p~pvv@g;OYrurCV&bUpSEi z8RRs;PZNMB4R}XSN7B?dVix)S##FBvVOl!N(!4@j)=0?q+650*V!wBy7s{Ob>~{%! zJi`(|eI)9uhv*1l{LFeT1ndHT#!z+KGd(?p6I?7c{%54oEP+#MD_K+h1nfO z;fg-gp=|L5bi=2_4MEYhSna|mg6PJ)IN%*IV+rfl#zxbj-0ZncrP_QZfNqrSfooG^ z7wYg;Po?dJKJ@C&r#`SMN1g_q9RpD?&-5lZd+}~m zr$hlrGFM1r8bh^iOOTW;Aqm22Fd<>Ppso5MbnpKex#H+Y<=Q=}>S?=Ba0M!cEcd@UbYArYn+YZWh6AM^1qik3n`5bP)%5j(p1g{X=(00Xjktl1snkzbc^{C;Bt{F5 z4dTqGAwTH}M=27rWS=@oWjhgxWR8}ZKoeiLYUIz|z zA>1liA{F169`GslmvG&YH$1u}{G>sZ+*#eB(n72BbWa>dfr@|AUdBv4pL`a8D(g(d zaaqvf0~!u61*ss!$<~K6@+F%L3$*Ghuq-loaotfywcoU%_-d+*XH6f+=&x6&i+*{hcPD78PIVE9r>Zo{vEhJ!Z&!BJKD ze1%RP3D|NG6$qlKR!+^Gp7_}BHygE+eWvdEP5N%;w%5|E7hg~?Yil7O2On}KEAV*0 zc{dtt?j(D4DP7$Wa}tESDRl&HkCmg@coSH|6-)j0BP5=e5x+Om?K{KB;K8hq*gdD$ z#`neid%7I`jkVyY0;;rlJi8lD2RC&%ZAbPZvS6nBf}$~vK2RWj`vM0S=3;?E?>+v(PE z!FyYbGxX-&bkmM8@=Vu(E&C8iR_Sd3EQ@!tTW(lhiATVpK2<1(XXYHNC3LoE3Y>PS zw+jaC*PR7!j50TJ+VTBN8*Me~Yo6G`TG#NE;L?p=gS|9S+E)8MW%zTXOFqbp0RpHZ zcd>n5FTo_vA_Zoc`lpGYcNw-sM6x$lTysnbge4XFI`hw~Fr#g^rKUu7ciUFVfeM++*lZC( zA}EMf?o@lT`x6rmP3~iR_eX z8c0-Ex)U!(#pLs^tZbkem&e3%JCK-nG8}r1vPTDxz9hM>gRWREnuz_7X=8Nd&j*vn z>y2WJ$sry`G0Nq+}MiH$a+pxLL7hs)w8#35#yfJ7`V{vg-2E2k<|H(`~{Fk1& z*f->gZPft=+&i#ny1pnQx_xuB=2$^rABo`ct3m-a_xik7aYrL9jEyX?v2s#sh{?bc z3d&}OO*2Ad`n5IQg?E(Gv={TXV}89ei?po>EC?Bc8XAZjTmbynZGTMPo_2MzkD@9{ z012KtnF6f6DLEI(Ej=KA1(zL?q`!umI`Fn~2xJXLk=1m;;UaV67>pss#zIO<3Ryp@ zLPsSnEo*F}ju?9!@euP$8SER|hXZJj0{Y8Hy@yPy%W

j6k{^kGYlwx9=OErb_K0vxXM@heGKYJn`0>$dlbZei-Z__~2^_u!a!rCblZ;K5 zeDpHf^{H%jZc%cLPGwHyhWuATF<&4sG5N0uiTvD(He-$hW{=m~rRE;N!pfvD{545R z`9*p(3lXh*9=e^M8TYNy>Z}i83d$urX`_FY!^u{fdWdc26VU{eD3SY87<3=_4FzP= z(d{&FQWoK8&)BKnn%fmI_2{^^)|9wfP9-qXArKVnQe$#D-Iq2sYUNpFO1E_3W%8^R z8pDxVy`3gfsv_?9d7D@l3DW7_m0NF8F28aQ{D;S`ON1dFzZjm6dBmcuXc6|sU(E~N z3=4fkZXV61h`pt z1HiUbXbkx7tPy2iuG|o+gcd{@lJVy;%~Yd0Xp--O8WsFv%-E;OqC|Xds03+{R_`}0 znycrO6e0972ms4B-@QKlg~k-GJmzQn!;U3b&)&{UHe~Y<)m26Bg@y-R;%#oi5DczM zoZ{S;V!$H1?AGR~fCj}l4=BgCRCG%=6!nqmN-Q~7ykK}X;*Xq-aBqZGZ^4VeoxypJ z(#)BWFV6NiJ&`rM3`-)BI%~DI;eG3KE5c{MfZQKGg3ff8p;aGI8GiBB)51nn2}g5i zcPPyUv9t+0Jw&58UP5N{{Cj^QdHpJH<9BK#Q4{Ty@b6>u9%9+<#a z_nNmgheH^2@z6nHlT9(s$}Pi;z6R&@t#ZSL=dHt=C#1=O8(TFK9%dOen$uVpP*YZ2 z|8lWbK7+cK@28Y3`Iq}Q=n^kmZz|*}o6cM{Jg)$j2{`BPnIeKYNmEe%ZJR*J{*A%M z8P%^tyW&xZP{p_no8d2RyrgV*e+dXqtWnJ~Ljt!i@}e%Th5O}>Y3Q3@mV?r)!w3!Ka`8t5Iymxjv>w(;K#qNk<1N3Fn2BO-A?MPN9&1afDg z+AL%)*|2jVho-DN?->ZE1xAfK)*f5_PoLC(QF)A)RA!8PAf~EFRs*YWqb3bn50vA+ zNkfm3(yr(H)9vM567ncG*$j#_lf`jw1b7nb+nMH@gScY?x{dkdcf+i@|BJee#+Eg# zfPZXp_?@}=RjJMkaUkBXDY__hdx@w+DSA=h3dlN;Q;%XbN$t zTlTG)^wdF{N{mo>^z#4U!+u&htOpCulgl?9p*$DRX~f!+k}Ifh$r8umW@jH_TayP> zu)X`3`kE%%j%h+&Jw%ZuM#EDV-0L9D3&iIwVVat8K6+pfj>CN1RU(kvM1ut^MyOF* zLDr(1g6yZ{Z*Uu+euP3@TzBrUSN^sE`C}r7vRnk)t@?A97nV#O0!j1cmk=U%h$hTm z8ccz#zUy|rWQq4dvG?ytKU~o8y`)1k#Qp4Ecn(v5+970vW!eA){4(rS6cL}W1_N2~ zph5_R|8*iDM0hcLDtrJmAb9oNPfOYRSjKq{s{ob&DF^N-TOAQz<0OvAmb0E$R278^ z=vWGRi;76la$%hdk+&hF(5O6rZltii^ok_wr%Y$vZ#7!u&bF865C3yO=3|`y+nlO4 zmE*dUgojCxX3T5>9iyHwsnQJ%FLH2j3tBxza%#4)D)`gLcp-@K&MLYQ1C#+*gnV8T zJ%*}+W=+n0Hs&JR@t5gjF(r%mO5K1pR23a!7WCWwT73j=gZ!*Qjs;#9~cso4f; zG_}8yS?CWV0qF-8_$cj>oWi?W9N2OGR&uDD*R-d?=pKWhD5ZdYKnd?XMbH8ffWD?6 z8kbK;!5b?*BysQjoujv$S67e-@r_}(2t-15K+e6@my;l+uveER>|>>Yr{nwMga6Q+ z()`{EWHY3?&qM*_epcV8@eVN!YPvw#KtZ@8i|^rhK0POqCmj#+c-g3RG(<%^c248f z2=q*Pbo0ygM|A37tsK2(@%NOP8qwISmWc$c$eRllL`ln;PmYEZm+2*XO&>P3mJHxT- zQ(&C2A}h)fyAWDy>hiOS1y|?*&fvJbThEM)_|hEJs;XauI1uOr>bYV(bXVGS@sVRp z&zE$87K=67nZUlR(${W{XH^+@{Q4XTI;0BwC3bDj9fs^;`-aq@>nHa45S|!;Vp)@@>gIY=j)Yt_C9TX5-!d z=;;0{qOwd*LH<`Uou)0;nrSATEL_a6ORDCn``pj9%z2=_5)TpYFnsHAJKC*uRVdN6qnmI zD_Z7K<>dWU+)ng#tigz;4bVdbon(`{0!VNOIR70na}WvQffH)(RRUwvhhaJP0aJ)K z)B6A?#x56hw}aR!uk)C?asxJ<+refpiD#mUeUptTp08~qmq`c+GVJ*^|61UzwwQZ8 ze@Z#YRiW*!o-7MOSX@g9!5hfZbR>}B(BJfwD7+>1OJdZp6-mwPifTQ8DT6G^yIB5^ zVJC!*J02~diAiUbYf%9efE>mnkfV4B* z!eCYx4&rr1u4w53dG_6h-Z@7|-zC^6Ccv4gtTVu!;^=V2?{ChTjF6*7 z?CFrO%w4Xvn?4*e+3a4jE0XG2tn-}$2CDL0eEc=9tZ52Lez2Lw5eZRHGNiu;s(+nj zZoqKmi%?Vbi>DT(*Hp(APy+v;%7V3MzjPlK0}2+} zD;B`IuvuJ;MBSl9+Oo>r-Xaa0(mqko^nQ~U9Xhb`J5C z#lrx7ug}`^B7IM5^V}y4birkA!4bbLy&3Q6cE~zeV8asMWoN6iv@pCZ4N-GzYzf!t zOKQ{4re+>kBTk(lH0QT5`xQfW4n9feNRp7jSnE-bmR}agtNi6+T#N=Xbt{iMxS}&w z1%PTW7>D%8d9^mOD5T@N4P^}O;g+Kt4hQdXY?3A!=n)du4SM^V>^Ge7p?ik^2?2LO z_MX0Ml}ca$N_g9sG1nV{rF5aQNL;L8_?4m|#s+s{^`Vs#aN_RViG}&Z7I=59vJ z(P8E+dHcK)k2DyO91keT19V1a?~v1O#I_AEz=T}tz_Mha!6;8mt8r{<4h%Q&b6OmsGqdCNUoi6QV--rNdiNL#5q)e<@PR6@W)16XXvomNgp)dmi|w$YiU2 z2Gxc%O`O)U(vtNIef?9kAr&s{1WjkY-O!Fgo-aUrt&$W@ud2lyt@&jN4v3GW+!-<8 zTZZUH>7-Dt8SX^r&d7)4SJ?yKkfw8K*q)hN`m838exo<9T4E*-tkDqY$pEoe9Z0004iknhix2yOrYl4SDQH~=YNr$n*D NXZr#G00004Sz0Wd#s~la literal 0 HcmV?d00001 diff --git a/tests/files/good-1-lzma2-1.lzma b/tests/files/good-1-lzma2-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..386a1c159473e1a8dd3cb04beafd74941995287d GIT binary patch literal 424 zcmV;Z0ayP2Oj=Dr0004LBHlIv03iVg0002k4vH<%0_2e@^`p>hS)*d z1vSlsrF;^l1>2ClFBH>pa+auyeS}tVjcOv8tDZfOfGx*f3=&e)KL-eV9He*i6Sx<^ z?va5xvxo5i0JCaRLG3QE@?@8!nx12!t{HRDzhLwWI`Eq@>)$`vuccLzMg>?R36H2?2n(a>KD@LR+sC7#z!udM zry}St=(Nyx7sZR@^1Ex+@0x)$P95CrHs(ZfzbUmm1uvH<%0_2e@^`p>hS)*d z1vSlsrF;^l1>2ClFBH>pa+auyeS}tVjcOv8tDZfOfGx*f3=&e)KL-eV9He*i6Sx<^ z?va5xvxo5i0JCaRLG3QE@?@8!nx12!t{HRF!iN;a*~>rB7^clCOI@hPUjb(*Of2Ng)l#05 z0)7x(^#ie(FDC7VdscrFd03A`5ZT!emgzgcwYb_V;nsoi$p8TRM@ZWM0bt1k0002P S{t-PsFb#_W000000a;q-q{Get literal 0 HcmV?d00001 diff --git a/tests/files/good-1-lzma2-3.lzma b/tests/files/good-1-lzma2-3.lzma new file mode 100644 index 0000000000000000000000000000000000000000..096f65c1abc1f73c902ff3519d03a83805c41e96 GIT binary patch literal 408 zcmV;J0cZaIOj=Dr0004LBHlIv03iVg0002k4fCOZ6IWCY;STP zb7^!SVQpn}EFfcVZgXX0bY*mPav))3X>e(CX=7<_XCMl|0h0kFT>vth32ao?E|qNg z_czrX;zk03b^A$njdN4B{cYRI+u1}5u`a$7wz{a|^X+w1+(97BzcyPSnOzSdp$4Ot zEw`7P@O%E^5{xZT@Y{BGeWZKSmFfEh=Jl74V(Hs3yR>jU7I*`9I*WSMxcEvZCH0P~ zBowA8C~Gfs;GuD}d!&F+K+5miw3Yq4!HkYzm)pyKA??5raT}jB^kOtXvnab% zHr;v-95y95SO)kNqs|z}{gxhSDFeg3C-diz+jF!lPdHd4zk?Jxesk{r&&s{?A!;aq z$c|ls$GjKNER(eemAVd+f@ZG&xB~$DM@ZWM0bI!g0000LIjK%QFb#_W000000a;pz Cv$ZV% literal 0 HcmV?d00001 diff --git a/tests/files/good-single-sparc-lzma.lzma b/tests/files/good-1-sparc-lzma2.lzma similarity index 95% rename from tests/files/good-single-sparc-lzma.lzma rename to tests/files/good-1-sparc-lzma2.lzma index c464fbb37ba1fc53f738b5f154f6399a5c219350..bfc7ac0b4a5aa9842e3dc39f47fe6c193202dcbb 100644 GIT binary patch delta 75 zcmcaE_(f3ezfY8}BLf4=vuj)2m>4-36d5@fcvihCe;_rPV^i!#@y#5{3=9m%Js98i f1tzazVBE|y#gu`;&xv!_#v|#!nSgQ(ERj(Fw{jQV delta 46 zcmew&cwJEZzfY8}BLg?9*#%c6hACS)9b)CLw(9Y3l-ta~d)$NZZC_yWs^b&6BclL- CUJ&pA diff --git a/tests/files/good-single-x86-lzma.lzma b/tests/files/good-1-x86-lzma2.lzma similarity index 77% rename from tests/files/good-single-x86-lzma.lzma rename to tests/files/good-1-x86-lzma2.lzma index 04b9b2d44ea3d29df20bcf28d2c81338cce60e9e..62fb01a29694ea17cea251eee48cc23e1f79ebf3 100644 GIT binary patch delta 73 zcmey$H-TU7zfY8}BLf4=vuj)2m>5|Y6d5@fF6}Hi{Xj^AJs@_Ycp$qXgZRfutZP@k d*vY_ni+QCA1H*UCa*K^e(tk5C00B#66abWv7+3%R delta 46 zcmV+}0MY-D5A_Zg|4dp!N{Hqb>P0I8$|KwTe6_aGRt9#IDk7x9t>t*Pj_<&Fzk ET5atR>Hq)$ diff --git a/tests/files/good-2-lzma2.lzma b/tests/files/good-2-lzma2.lzma new file mode 100644 index 0000000000000000000000000000000000000000..49165309bee155e248b2ec8a543828f53c12a4a6 GIT binary patch literal 92 zcmezG6Xom3z`&TPbkB^5L6MPzfq~%$zp^kR1FJ`BPEI}-kS}IB%@9?NEj+&{Cq)q` gdH3R}U5>8ik4I`wPQF5Teo;;e7e7N}6a&M50`dTzsuM2& diff --git a/tests/files/good-multi-none-1.lzma b/tests/files/good-multi-none-1.lzma deleted file mode 100644 index 53c6afaad4d6a75cb3f37e560326d49dd721e2b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 75 zcmezG6Xom3AnEI#xs8M2`wWIoRR#tIkJOx;d@cbo(`kkfK{cRYcz#h%iXxZ5-HWG! a7cj87D@L8?U=kH$V}D}BJV8)0G711z%@x1^ diff --git a/tests/files/good-multi-none-2.lzma b/tests/files/good-multi-none-2.lzma deleted file mode 100644 index bef06817b43d7d05ae7f41b24d559ff46134c8f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tOB~XlfygKTcVreG diff --git a/tests/files/good-multi-none-extra_3.lzma b/tests/files/good-multi-none-extra_3.lzma deleted file mode 100644 index 6e35306a327bde3048fac3994da1e77608dde657..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55 zcmezG6Xom3An@}4*7F<;stgPa9;rDw`CI}V3~E45cz#h%iXxZ50tT)=X4C;++047mUR diff --git a/tests/files/good-multi-none-header_1.lzma b/tests/files/good-multi-none-header_1.lzma deleted file mode 100644 index 169b5c90d983937b46bea83f2ed535e9e034d9d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 58 zcmezG6Xom3An@}4*7FM(7#TPiR2di;JW_LV^0@>!7}S8A@cg2j6h$t91q_`0EYiG8 MynJlzJOYtX0NZv9+5i9m diff --git a/tests/files/good-multi-none-header_2.lzma b/tests/files/good-multi-none-header_2.lzma deleted file mode 100644 index 9bec4ff5d8889177905831ca436db4e2b31a75b4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 66 zcmezG6Xom3An@}4*7FM(IQc`Rd6{_m*w{H3R2di;JW_LV^0@>!7}S8A@cg2j6h$rp NkTM>qG9H1*C;&5D4WIx3 diff --git a/tests/files/good-multi-none-header_3.lzma b/tests/files/good-multi-none-header_3.lzma deleted file mode 100644 index 45cceba29788e695b622239a99be966158be3179..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 59 zcmezG6Xom3An@}4*7FM(IQXP_nRxlw*f|(f85kHmQgd?hxdb>E)PS7u{Gyx`MJ|B_ N49v{j(rf~eQ2@`j4EX>6 diff --git a/tests/files/good-single-delta-lzma.tiff.lzma b/tests/files/good-single-delta-lzma.tiff.lzma deleted file mode 100644 index 94ff3a12d354559c2b5be61f097db8621dc26cca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51409 zcmV(jK=!}?Oj=Dr01Yscv#bgLAOb*LCk=d2hyWzurEb4(Zly)j0n5Cyv{sB`NaRg1 zFVvR`1eym}Bqe1D&aJsCmF<&5rrIN$l);~&P2Qw0_UGwqI@}ytabE<_@4 z5NkfK*v&DqdqAuVZ&vR61D5XjiGC=4_BL(YhgT=Q0?Z!q2+g*x!tCWs2YtBT#$Fg5 z;oG{TcLjd=>CVQp?BxN-t7GYkp$zE>)>{K)c?hebkj{jV%@)pkMrA=^uARoT`n9ZFQ#YxHs!wx_Z9G5E7uH`O@?Xp)$&kNm$ZnjzddT(Ce!2)lOdOwp`7P1oX2y1D8E3^Z zK_(pxml{c2R$Wy=$OlG~CAO2y% zicd!zjzk=MaLo|7N1xpyxg13%@rMTW$LArLQ^l4Q=XRofF#C1x++e1;A%?U8#R-{W z-3pVaF{oB#F|`VH$(Qrj*9FyV5GBY>V|TK)Zajf6^(f+t66?99rnX*SfCDdAVKfc; zK5yC-CJdWv5J#G00?*6%qac4KAMXMf$v*xn|?TcD#o+&K#Uy((S?;{kcojF zW19B2X$So$i={r|2F*!ZZ`m0!tU;zvNryH5U(F`sH^uaqvwYvuM&Sj14#v&&MKQ`A zS;3;o)t!T~&qiQb2$25BrmTJ80rGa_0z0nXI6}3f@D=3y{s5m`aWQ zk)D5c-e8V0{g-^EiCp+l`)r_gAFI+g zgws1qg=i1@_m|0P1}EwJFn6oYkYuIU4h{zO^dXOBhB~_Cw?)etTW5hI!#k=s^I<;y zEt*RfsCoyBaCI9~c4&`m-@8S|q?PBtsz7XTlD8Zq&0xlq*&owepsYlRwd9lKuQ@wV zMb_VoGsqrr1Sv@{Qh;f2v(j3Pp8PsS_`0hJ2e}s@W1ir~P zvuit`VI$ha-bQ*Pypt73#C&*Nc4*|()|RgK6z85i{F_z&vwV__l2Oo+&<@#kwEE`8 zPu-<$HXAWqQ3Q;$PiwIIQpys^{P$-lW}Xo#suM5*Q`$#o$BE}hZ(4C8M5eCpgLc&bILW2L6IHwEuiru9kPy&{ zX9f#2PCkyc--5DEv|tPuIhp#>3bscRmT34VYq2|iU3@?{1X9;zHBq`lJ_6A*k*IAZ z?lC_q%7$0L{UK#|I##W~hk#z*O^brj`@xB3d6cA5LG0P%Ar|ihNPr}#ywSYp{Fnjr z&uSh!#d{uETolOj<$({ZLvxBfIGPv9m? zMz>OG-Nq6eq=|+?m_k z4l6ascmV94w|v&8|9-H+$pc+$G)0x|OcS3iy7i(#UulSXr6$t7X{+2T(=f3@h&^Im z$NF#p^=A*v z-#rU^WTtDW@w?AOw(Aa6Y16q*B++Sgb#S(rS+%}*jnmILjsa#Wobc2Te)c2p$9k!OZ>JOd$sVG2|Q2PZf zY1K&`fqP^rpwOuUgA(V~t_J%<-Lh{zMqrj?_K?99WN%S8W~hmx!Xp1@W*y83#CXb6 z9&3%;l6;<_^ApE;tzZBVPPGU`l-;K2gJ|r`!D4s^LcksXI`()i z4?g|U1qSTJUf0Lce^!8K&{k8{KWs7!#*X`Hub=uI@QH|nt)JssO)Ogw?snH+GXY<- zJto$#N|&+}1vEX!PVdpw`oW6?ljb|lYZ#~x>Tg}Dzd|JQ zFXt5&8^kThfa2qHUbfptNpJ1?l#eJk*|qz@04$qI&ohNYAng0;!d{EcJBbi2KN5h3Nlz}Ky{y!gPfEKcL_0#kf#oy-f_dk$kKoj?~ z+LZcfXZbOSN}u9v))S~{B2XHHxDZriWI6>0ueQELXkMncpkKPxmsW{jn&|n@&Ddt_oRpuv) z$3NWjNXq-3^#Q_?fEnwE2ejK`9Q&jK^LU`OlMN<})UsJE3h$w>g%a4}l57kq(W({&Gb0lKUPT} zgDMt@Xd(+|y5zt-jQw1zCj3uu{q_ck&lZuf91ztIfVGWWRnt6}D}HH^3mxw&mvU6E zi`3RYuQJGOBvIO7$&u zPAk$4uh&jccXdGtoPE9nN(Pd<-V*)i?)hwj{(M!R@ZdjrScWFLYEHCR-+^S<+?@um z-}x0HZlytKjdi#s`fP888^^v))v`Xq{Z+E`0`n-r2&0o;7e7F|CUw6qWf;OB^JZ*w zV-Gas3voPbmeKcR#k7NmjGj_N@jVad|2QwQ8iar`E0^Iy1Z32Fi7uZJQJIhdim(4V z<0X<%j#ywYCAWXbbDhS`rCS!cg*B_dGvLBT!Q}JZJT}u*;~4W_CC=#1+Vs^MH~w$N zhusl>hbAsmTiHteQw`54Xt%l|K{9aOgp!kSRS*NRv{o8Rg38MApT`bg6l@AOmBY?l z#QFDC76|s`7Drvp6H|B%_5V#*t<*sCt2!f)@`L8CY|+^0WZKECm~@-qNX8fI=x4&1 zLjy-!kKc7FE`cxC-@-z7N&mqzK&6h^sCJT0`--4t-7FHNZnnIPi0VQiaqa%tfp4@; zN)b+1Az(L?;gn-}?;G#fwyco*wgaJ~St!uR%|6%Nsm}h1^G*SpFvLtIRLv{@NT!7z zj?8UhzC=6jayec3UE8V@wlJl4N=zqniuF6bOIFwhX6rDK;$dKRt_d&Y>m9ZiRrOu@ zgbfPUrGZv4vvJp(R@e<^g(E}S?K-N7P(cTA!pEY}v>Vk37ANVyVcyR`VD4(q^!Yy9 zjh)C#%jg$GHPp#|G>l~Zvd&N15f?oVuEJ+ui#awH;P5>X08NB=)>OCFhf#p=oS-LNs50sjV(RsDjPX)VaBRE%6VXjly+N_SHr_&$9nJ! z>t3>*Qa-*6GC^%BHNUz;R1x>P;5(ggA4XTNfpLqaXWvxbZ&y;*1~~`*JUR>p(a#3g42(ysnh%Zrgeo?MeYb(sP zf}*pWFNR}PgonGMMYEAl6a9%ZfHLW^wnlQ+MNbb{q8QE-O6II%Kfp9#9c$WlV&}Un zjS3l>`MNVWBQG;TLBAr zea2fN(1??<&F`I>!e6CIRTZh<+}0zD{QTcoT^G=(00q-=?!5jN+^&C(aBnk-Wt1=G zKseE9LMkh*5f7&ykh=U$2q!D~9|QE)Od$F>nawQF$M%8PhZM1% zAWLjuh)C=Jz-jneI-U#u6)9e}65i-TXJWQRX*gAXc77T23RR@&Ip1`=(JbJTFqq&L zLsWoI9?Tmy@Ka;n0eMpMV^5#q_K$hIFlk6adv$o_o>r_XR9~PIt}1Y#$4t>ksTh@* zWcD1#tpMF3^vb6@XDz>VX*OIlT^f$at`?PqK@TT<@Fm?@|8Gt%7i_h1gfuT}*z>GS zo=p6NGwu!94#%dbPCb!}kLxD-h9@o?9joq<4q5Lh#6wO}XKjAh3LM$Lr*0bmJ5;vL zz3LJobmd5J?}5<&#u}6zijwa4u1`9+Zs7fFx;@j)1pvulNA#>8`(nQf1exBQWx=aRqQ+D=>^(U6Uk1w~Fp zD~cT09*$N5EPRXtjszOh$ul2w@U{I9WOh{x@OMnjy3M7`h%#tNv zh3HIBFtIa059PqY^16`}p)A(3cUal-UvG{H&Zt3sY4T5}*c!X4*k$&g3;(T8J_{2} zI%&PWJ$*i+%0#0UD3jD@+}Cc>RhSzlk7H6iR=;!Ri25Rb;NY?blV(VbY*Pb8x7|0E zZ7QanG+}b~E@k9HdH0a@CwlXm;2~9FaU8{MS!@=>WE9&9$ToJsgdKQHoVs_&(^1PT z;W@kOOoxdnBxpqe7hL;0+GkgLJKtvj5W6g|wTzlNAQyQjYuK9cnd7D7O^BpniepnE z&9WH)Tj(r$@1nv$86{@oRtxiVAq)kYkwK$98jHH7(o9gPh@QKiDa-WAm!ybzS~8VM zf+p+IfSua3Fs7Q($q!}RtV+UD^Gl{mRo5rc0Hn`5Rso*`^xOZWogWgR{BE<4i0WW< z7N@O-&|LR}J*FHZ53S(LG%KL5z;dbDubcC_VT8c!Hp=$v$nc7ABiR0?~uF80T%)STmdj z^}IB;p%+BmoHm>niWL=xQb-?YI7QftadL59>O-j`$(Jltwh~D|THvY>DLIf|s@S5~ z$wlyfI9eT!a{?L-@kBe}h4~{G1owx8QmfkjPAfjbua)yxuIJEFrc=V+t-cTpZIPR+ zLOaiYdtT;ZB{Ph=P>6wnE|n@$&;YB942)C1OqdpB#%j5y1H_d$iK8aR`XI$5mN^c+ z>%E9|3Po87V6Xkb_%*oMP%B0|xsc6`Pm!-#cbh}sq!xs^Q5s5y8p6(fbT7O3%Zsjd zX2_6Lg3np^c?Y0`@q9N}`pjzfu8W~7eW}YsXmJ`^5kzD!_@0vf(y%|uQS2A*&iSJi@1?VnkR-f>s+p5-n)rE`mU_K40gHT6pY}4^etS zM*;n(c=B9quCj{bX+pNw5x#3L)im%ALMfs9@4U!$D9g;iF#pqgO9kbBOV1)^chjQu zZfpPM&@KJz?tm7A0o%BE=wAFrJ<^Z1DE13(#~ z$n=CZgsMfTMjhy2u-nhm#O0GZb>agYN68ya8aKo-wp35%{(AtXh;AL=Sq6bsYDvL{ z7~h%)ma`qQOIX;73TmqLmx98`?n0LB0(4_PSyw>urHGr9aL0<$-uR$BgV&we{3JO>7QUuEn-8XZv+ZNv%FGn~IS7@b`%ep?!E6n=INMLkdwm3%aKu ziAE{sx#s5LH`XG?%8DNFb7-z3>WE8n)RYt}Lk?reshM<&s!9(Br(rU{NY$<31-Ufe ztjpT!FTk*`$fpd}vcbczjWPE_YGowuB#g}A?m|fVSgS-ONyaBBc(xGq&Yr5ECKaA` zPXSB=PR_2Um%A`zabj^VG41u^vh;F*{!~FhZGPP>6QKwIHo~H0lJcN}a7a?Ty&83y zemRS}NPY2EE!gIEbQpQ9hgAJgO_tVk`#Bs4=(rIPPbv-S@;#(L2T;G@shtRtGRd+a z1TZ#$M||jYzeV4o!q^${OuENbBNwQ~SBA~^Xnd0X>J}O!kK!9=2F`CTw|p9>oy&hk z>M&N8DA{s?Vb$|5O+Gy;1>xyi!{zJ=$4^cVn8B5a<~$`6Ut8J+l4_8C@f#Xva_-w3 z4n9!6fp`z0M`^X9w0{W3vnHB@w*kU4g5yIjv=nLK$T}W0vZ|b@ zb(>E|bTu*#(?78Qba6A+H)mHN^wuwF+gCq8uP}U2WM_=AWPV7`D+1Q2>U*n9vIazD zjy2_L=*cR7`t-*?Mc6Jm4(+QEC=Q!w{2#Wtit6Pu>C#9sDf8&pxPw;jfI4fE zmGkWIzo03YP+F9Z&HH36p($RL>g<5BTQ1>NxeHdWu@8);QiX>*wJInAAkfGo%R=h! z=QwYrzzj3koW<9rU_Afz(mdO4d+2R4R5rZs{yw*ml?J5i5wjs;s?%;Dy=WR;^?gz% zecUX~*ttsx5QTJ|9m8E0F;jv``|1KwMvQOV%L1AsnWzxE-;!cIe6!{SE=B$Vj8i^d zSBeyH+L>t0q1}btvb-G%e<~~5+j$awTDCP>jhAqID;hV}3M-RhA}kBK5qvrstd5YfDAJQ^pce?EfCuM8Zi;LBNH1#0Ctahqn} zF1v$d8AR7E7`z_2;Cye8C$i1y$)-H(VTEk+X%g-1@A~qh%CkqDo;znv=pPwcA+nSl zDqhxqBvmD)CW=j8Z3>zD1q%}hi5N5QI2c8fkQJDQw3hs4hG?Kn8qHA!%4^D;&rUEc zzZw+xH7tb|U`G3xpzdAC+^fL_v*-6s7=m)g)UW9Ga7>PrLl}#?y7U?Gl#t8> zgvt2Ea#^ejNKj%@pvShrfoR=Y1}fC24qh4@%E~#ZNR9e!(qekQSy%%}L=%jEBBxe| z7!Q4F+evvIKGGtj@^0lLQ4xy3zIfPw#h6kIKUv2vIUtv*U@<185#>zKgb%DLc!ep* zcqTJhDKG|MP)HdwgFG6xzr;f5t0P-6e;qnPb-$H`YW(TTzj>5*Qq!cF%Tsi(Aj4Vv zP0&Mm{dm_FA~Qx0IT41jyBb+`04wr++Z%&+p_F{@k=Wj~4Je35wLYt2UD!2g)JRC% zA404Q`2AbALl0vUEWeGHmkuKuvCmIxj{zFyt%&7$evAwcvA22PSqZD?S^es)Y%K9K z+byge0@))zy1^6scCkfDSoNVdf*O*C%tATaP#~ym%2-XEE@YmKwFjd!F+prODrh5v zxeUzt4kQYWkT41^6P3T@1Xml6oHxJYJn=SI!F#4wAP(oj0dFad9}t6it%KWxuVr35Thf~B_ZESgc{Nzb7V9;Qxv{T9^bL0bl>zupaXh|W@s%Y0-<(Qu zdVM?~l$jpHIOgoNEFNrXr;QB&p0EZV96(^a4M z#5`dZ5j0HlL5N^YL2t$P))u-^tZ@rjke|}JBFeJp*Cpi{{o4_%6}vu0dy~ddqS|uc zNoq4VHvP;Fu!3vT>o@W=e|9k8pHIzPU6&+ZC_TajYx=oz^s1=u=DsnW7_;9pCWwec zmGsAyzi8+$PU+9mA8K@W8(p$u4aq}K9b57F4xb8z_EK;>?08R$@aU>4 z#1%*f<~y2h9NFl5iL4wy509)DDFGvTi!l4UA^Pt%_XiOc@9W}eY%wrbXlA#KrvdaG zO|p?_J1#{G7Fdo82aVq*YJhBD8s5;ufI!O8iCrZ_8-(McaJmg%mg*_ASZ`9p6|mUC z7;zJyRzN#X8UE!{dkNf+17dy?*v$GJWL9Jd00!T4;<;YAT57$+ zg|c>1o%)VRK|?B)ORSe!I2_afY7{0^n#{}J3Ay<=_;)xkaOdlatL{n7BnWp#G#O*H zvd9qVr7|?S39!sGSk($3{0;C`Lx;@&kUE#i1m7{*AZqq~^kbZqZ3mwpSIa#whcA=U z-;nH?e6Mq^IL#ijd^fb7PtFVMK);2;d>zN8S^(FD#M16y%T?U~aZj@xMl->`{r&Ia z)k@GixH2O}HsA4vL@#3SD&JEoVioU#w`jXHgsUOpZVI`Wo;Nn2j+sn+R4%&0wsz>TPub_ndRV(pycy880@DuWU=49Zj#YH zX==E6hl|iN#t^4;-kRp`BqLX93bDw1OUL1osowA?CrbCrSQu-0YMb_pSE)bz#1#jJ!qEC#d~0j8G1Mcc-%3U9;nm)m8+-q>FqZ+|NuVy$mj{WR9L z8`XpX_ZD%;?{r(y9fsdxI72_MSo0PzMcsK!+CD>PcTSerk#Wtx*NYa$cZHu4d&P5d zw@}Rd3dv41qqEXycDjuE;E|>ROq6GcM(K2SjWs{aXQ}%fJIf_>(kX05{JYJ&J5krU zy_Y>%=pmb9&G7ew522()3f3Y?1JJC%n>4DBMoB#mBmH=oUe}_Ay$M_oFNSLJ zX3y*|A27bC_x$(um;Ua?+AK0Or}$VC7Z1}^9nX?IVVbc`a5mQoQB9J@A6xX+!)7j2 z_Hg3MpvwsebG;*oq-{0=9~O?}7sa50nYjB9g;srQ{6;?33W~5iu@FjuX}P4bS>xpp zw-`Iz^`M!)dH{Q(1USMX%=0FOoQxY(fSH>4Z1S>UE)f?F(rfgC1{%-xgxd9HUt?;_ zA=SeTPoJjyWNsaWprw5-h^a}1h7v`F+p zL59zHjWWz~4ERtM<+t*dlEDN*88m&c4UvA#6L2^lh}I&~-o7%8ts8~iR7ko+IF8Vf zASM-eN}oiF4jF2wCVL{Z*_*Z@?b0mgt^^(QjfzF_9EY$s*?RO3@`0q6>F1xyp_5e| z=#=QM?6?42v#DNs=#x4 zVI3-MGt5Ghaj!->lV3n_mtgGR}<$}G=^w=J)&YG=oYgUfBpcz zTDw+=vM;=mY*kqfm*qrCnJCVH|9+Nv5O0BkX$w@mCF+nx5Bl7oS|k4=;!O4r;j`cv zB!W%dDdNucsyqO&XX=g)B9l3VsM>T!0_8u?ohW%+e}R0$cW&%N%gM^kiFW`Mv8G_K zynEWUbmb;WE%ZNAHVe|ckdP0NI4uZnxCrB=df;n+w1DS|R~;qw%=qggGRrhMW>d2T z6qXu<6Ax4{$?z$W9YNoXBEtd{PxMiipKwPo+&h7`kEXO z^NNCh5Trl0SPJs|ICnLF3F@HziG^J!>NOc0Ea4_ww;04@O^Qah%A7fdJ0F`}W%0nC z1WlZ!k;O*k_e6{^z*5j^1DGGb1RPRqN}gRWX|LDD0TEv`>}$Rz6+V}Mnfpf?g6DL5 zh`@~b&?Q&(V+Kuz5K6^MtsVQ3#F}FlhH~wgT_8ddGi5@r;By`86-k@j%LGm zOfs5T0HLXOLLoEARs_JI$6}Ap=b{10#ttPXIdA$AO6$YpAv~yRIOo`<`&dEv?HT48 zQL?$k^Of2)9Yt!(;+>%ixWPspk|VER#-`r=rJpTv+8UXFhzlE)1|)J)Xtss>B!I9_ z?ospNdCF(QYjdoa1YSdhJ-~MZBpy5kWh;V)BWvC~fr0LhFq)hx8@Du@aRYMZ( z3W;&3e*7>Kp_4j>(S~(e9E#9RB`RHVtLt~ZZwKcfeOkR<7kN$ZlXU=9$yDzq)EI8G0b{x_teSAp zLE+lx5aB9~h%J#Jsha_QF(HamKEp#>#F&u+P~Q4DlP86ND8Si|YnGb7ku`LGR4E(C zH|oJ5L^VyEtr~EczN?cMOa;7n)XFkxtva>^`rHl6p zH7P2|5mwcq8Rc7q|LTJk14j&3Qgxy77##hWeX?m!Rt*bTDZkWVo7hWmej-~zTFM@ z?+joO{+Rn~&E>K}iU0n5nK3=1frtGu4f{+CS#)wiWq|im(8SPKLX9pEA+f4>fS{dG zqfR2ZLGH1`a;PT61^f!LYN>}ZYQVk1&F7!@&ek*l&f#YiI5*d~tOIaZ=kVZP=@LGM z5QLuo$`X*Ym4-7_%29F%Dn<66pL|AdMqWzDPa=T9_a}NqZi&_8J!w|Nt`209F(FI@ z86LEdKl<~S=d|BlWD8mnB*@%6!}Q7KS8iVyA!s3U{-fSf z8I$r*cw%W*j>cp|b`(^nI_ytm-d*XyC_f~;%D_8*{X_8A)}&I@edIYvL<-?dVur#Y zYv*K$w&e&9$fidx?!0;%3K0gGkv)?@`v7gAGj&RV;;%{Hiocg^cXYyJ;KJ(t515;E z4DtJ~<@RjoZHkR7;Dphnik?cIk-O$@cIne62NmGP8jTtBR}ts3$=A|CuK_=V+QoVZ zMGdfV#2pLKN9Rp7KXevFq1Arsd-sjMOUzG!Wobv27RgM~6K97Qw%WdOqLmo5@gq_}hqS>fTK%kzt<1?}x@9gN z{qA#Mcp)xF$d3Gw7tVLh9%a!Np87s0i^NVC)W)(?*z}8_{$4}Ay<$51!a-$i#=W5f zCm|woq>N$|OTD61^NPeu9xGYCw>96rpe|4ufg0r-?bf!DLrmyM)hYUH^{tsV?fuJOTI$fw`9I;BxKB<4$yB}kNkuYlX;5^XJ?=&Nt zF#2F|b~N&#Jsz91Qd}Jx8`~ECalGf%xS+`o-vDeF>sK~yID;ArL}DVFgyi_oDM+n- z0S5xxNX99kDXys0tfXb1p#nDVbY2I+%Ppw&r^n|6lT!eyCMH>1Wp<8S(+@eascK(o z9nV1E%tiCb|8lmv1mr{;CKZs)ZTKO02JZHf701tXrhiP52m+^1pA8BnX{<5Ax*DNm zLs@al$?x)y7OJYIn)^JD-F=EC@@HJ|@&7hi>{TD>PPUu+D%(unyE%tI1*g)>sOdlviI}Jh@ejc^T2sOYzkaA!<}c_96!M* zIg;Ur>al(-Snp}D!^4ykb&RG!>$5d4Sumv1HPok2{)_{MHou~5nwaJ69NUUxD>149 z8TjhS>UG?YA)4Q5k7YQ@+c}Uw{$6Fh9N+!sAO0V4pb$iq6^{SH1js=`-4MYvfR3p% zC|CiKWL(R0A70?BlMY3>qiLfUV!U_!kePsW)*$i>T)}-i{V-f^wh%`jZX#G9% z^!{Jo&kcJEJPn^SB^e>XUUBcfT{eBG>scCVrB+B9N+hafIPibgUUEA7*c=aqF(<4_u9iZ+JT zN2?YdT0m?zk+;GF1;u`3-02LI~F0*%if~jN1p$nrBClrVW9Y)>`j@g`2%U1E}6O+4WBb_NkI|AY~haY?l}58)4?TDLW>M$81PBJZy7E1y`iLK1}J}w-VIx?y)q0 zt>WlrbAcSYTpgvW@@B)6P_%bNWk)Sd% zuETF&=Oq4xyyXs?)%{A1KksGT9IO1y_Ni)nw#P!HD+i)}rgiU%8&{8y%i=7*dNC39 z@d2lAGSX3SLCkZSbzIJPxv)jdBI#UHE|T%iA2`8HInc~-Bp;*2|EmLo^wUTpF>!e+ zn6q+}Mpp9W9-RISz23~ZZ*=oIt1xK?MKyZZO$;;(g_zBsAJL+qx?K8&aCk0@!n*@S zo^MHxBs!LW3HW!!gPrDqEVJ4qiv|n{0UDJW+DRmZ`~M+8eSew4+%vKEwNCx7?m+Y; z`XGnHb?mwbZH@fc#VD;yov!WUmZLlZTxOdesot=v2tN?uV~2hZs!If%7A!}C$}7a& z7WME+T^ikjrW4R^o7bJCP+A`7)-4M#B$rtlnZ4@;Sw|ok-N9VSUSAuxJH*FGK&1cG zeUrgPF@9A;b3S{A0Ea=!_!7Lt7xGBJ^7+|x#)p7Rh7d=qp(w3yOH4;;Xkn_OL8mbq97_DKd4sGA@+)@P`+81!^{U3 zZ>4ux=s;8J%KWC+AypSACb~a_Wl92nlcZD8JRpOO&fwsWY0Q+t0*4X})109iT z{e$k)T1kg*kLM3lkfU}{$7hZ0*ZBT1-O=IxOtzj6y5e0Q)H-iYk*MCt&18Qc;E46K z<~p3(CEE_qF}PnF%grv+-(=H3nj0br4trF915Q2a>{1UzBT9!^kv68YqfIRvyP zu|eI=y(8I!i{CE9Jx-j+fY*XX{lahUL!8Nkz6lTuS@cPfO?* zEU;`l5a_NCDBLNcnq}dGp&|J_Fetk{88oG-W`qN1IX;vg5qOxR`_&BpV|mR9*;_U< zfzX16>Zeg#=8gdqcjGFtFla_PwrtwN<)%4Y zsH5VSm?fxR%Z9{~tEb5tR?$%%$n;kkt)K&exhW0mUKtV2azWs7{4oo1;}SqTQ;Tf? z`N5;<##-vRc4vZrCYQKQYXzfNrZ!v;lq??~Qty!A3yLUIxSfU|TXME6w!xIdkb#qU zj6%B$3_s|t8f5!y$;J@<$$JN(rO2Eo@yXr}b@Q3VaK4y7Aa!o;tkD%SlGOs>XpMP8 zm@gEHu56U4F%2LczT6vkTq4g9!~{;O2v%ux*_>YD-%UrBXRcdRl1t7-;8V%c!WexK z>wTQYTl{g8ka>vWY0v$V5&bwBG0Xj39l#?ja}2s!TFEWDwa|J`20UyDAxd~^zQisZ zb48CPq~pN|GG|+~8Yg4cZJIJNX$fMO)e>T+T<1KP64^7GS5jCLYR`u>x>$gdVT!Xf_^ zP1LDT8t0{C8Eg(_(&X36Ec)YC0>U?o+}Il6b$eq-8C`cp!=VLvC%gOR;&UX4T4Isuxw(RN;N-AozJ!A=9$l3{9QkJ3SVl&BdZH*KDqNW> zk+jgBV;lIGAyubGLdvI~@RWT18=Q-?twUn1c6Bg05vW$aUGp)M?Fj6iW;)!_zqsNk z{%dhdL+@$J-oH{)syJTpE3)08ai% zw#rzV!vF?~qgbx1Dy8+iRSeng-CEJ0>&;D&L#=xi?7-2-(Om)Gu0%>rP95Lr;+0`v z6bQ$RfZOMeX8bMg5#D`HDv(qBx2Seh9m{rW5Yf!vsWL!p6?`gv7RCOuMDV16-UF`oDj5t_BAcIyGhXc1(asm@<6G zzafw}nJa88zd#|iB5(O2n%U_79*9xHYN|x8WUBk5-jYy zm9x=jAn4$RY?cyA=PV#UHIkJ>-w!DD#jyZcZ=7taU1rBp0pm|LE*b(_5KI>Kz`OAW zlVV5k?)-bxie-S*4Ni)56v-N9ptD&@7k-QjSYBoG9Tq@;a}^#*-M}ABD(&n^zPz@Q zf4Ov!l#T??PW)ph)v~NdiQ1!G90mW2qT$l{tN~m6_`+gXsEh;bHxTpni0bR zMQ!s2HuB)YsIjWWX{Azp>MlfAgn{iF8A`$(S>j-k)sriLUUMiqykJ{}c-T|*A=b!^ zaY)&Y*_ttBPl>~DIj~k;m1j59hR6h({ZC{F5~)Kl8eyihu8aiZ7BC2l9R+2M#uWc7 zVB4cyl5ZD?dzkwyLlJGLdNFsIl!op)VDCldbfk-_lxYIjvoK+fDJMd=FQ}I)yQk+< zUAE_k*bJE^6hS|oDV6baoaK?5>@~ny%KG=;<+C=|eJj?6lzH9Zjgp-NBHZo{EUBnH zDfB82aR~j+_w5}1q3W))d^JVl^40V^l5Pio_5kAI0Q16`N7Bp9t_7LbE#F|TE)(dg ztW^`Ptyui6cldp#$||!3>v2Wf;l>4@jfBr+0(K0_b$c55zS6gq)w*o(0W1-vCtjjz zQU@xj@pTdN?u7zYw+kDk3r$R@8`y-fr@#ABq8MuchZQQshx$35C(D98n?FE~i@(rutaZ_L!mJpMz~wYLD~Nb(TMTzd5z52Vuj78N`cu8?yx1x4;~ z-C(P7E;A0pmofN2*dUNI%($CiNF4M~o%lFxLWX6TCM0$i{NSSvOsoCVv8UYF0)yKc zftS#Y#2x;g8S3O9%yk(?`YX_(bH76By1L8`GVd*=v$3Li^8^k6wK#zj?R%8_Hf^uF zcj1Y=gt2U`%LLbP^fS(p{ns$CJYY%Ha!JW|0ChU z0+O^LSd2SFi-Kq^3{`Q6X6Y<&V*8CeaLHW6Gq*!%V$bp~5Eo(4zkMrzaCFR;YDw=U zFa_oeJj0^zj8+Z6YPTnIeDDoRdU4xBgz{BfMn}}UA}*n+McQ!gs63H;InlXxS#kkJ zw>1%e2|)Zs_^7Bir_i1`I_uDeU7`&2a;@0=_W3y@uNh3l&_Uwy(sKnQ!?y`R9lQ64 zbHJk<^r`;g@T294tD=U^QaAkc&m8rE_2`+;TCqu`iqt*L@b}2_L3??y2=Z~ymbzOQ zcl`M3TBRm_hiN4{|7?`&^e71j1-v)!v{`wiXyKheLXa)`bTB!U?oEwZ7%sDfkZW-i zjEXoA0Y!RUl4%h2}JQaW(=C2giTM^Xd^O~m>&FW;KWG!E-?yx%ds zZJ$ff#xeg-jWO>~fOdY$)axm(QoedQK%4mFaJ7}ORic$+ndaiBPM}X?-{QnC;>MQd zy&)-5RC6DkYC@@dw42emKTp0d=kDc7y)TxIFywFTd;uzZn8XLVFPr#`4|aRQ=mC>C zQn`sL2iVKk_n-U~lALKhJfvA~l6!+w(wGb}%TrEA(&`KdAPDP$Yj8b8O4o<>Z85B{ zOKAW_+*-O@x*g$xG$kIpv3lfSU2PfhKc2Tp*f(r$#P`klb7QbdqWwzL zdves7MeKp1Z{<4K&i~@x2Pj%Y4)7zsTBEF|mj?SdWDS^VzBrvxuS9Afccl)U%<=^G|d<{udSH1lQpO zB4>jK8wcHZ>omfDx8EMjh@k~CCS|Gp%(Q(O0Zmal=e4X09f<~U%XaUPh4>JZ5os&H z=uI1|OFoh?h?w0@5q8@9w+52TNG&;yNqIt!2sipM@73*jj2;a(f`C9r5#49J!AyjP zqB)kG+Pz^7>CRwG zv_M#ZzD3j-2RlX@tU^UklVlHkFA?q4``iVdb*`KIvni9?s^uaSamK%kPOLf?FkP!4 z2Dh;5j1nf)Nl?N?(n*XGd(1We6}TCcs1U0nIT%v91Yb%!n(77E89}XoO$A{2iXJST z)}$qr2a%qzn z@@zghnH|=sY)UEhjEyqv*r~B9t|1a&WPNU)e(@8_*c3XapB^;-m{~u?0PHG^cV(3Y z2+H3mlmG{j(uJDX)th+N|2vdh;PiW>n3S(^8J1NDVjjNHdWoJKud+pamT z8L7Ox3~%7_4tq)QoNG(+srhAIoxzN1H!#%;WM}n*zS3Qik&y8GMYof-gZ@Qz>Oi)a zbN%-5x58EzmOi$+WBQkJc>WE0xuEDdxxRg_#xq@ zjnwaq;j%14p%E8AcdHD9U|{qnxI2{=)DjRmN&<}d&T?@fVR^9SvuDv%FuzRAfr;Mi z)bnN#g>`=J8|((_&YM94HX9sZ87P%tnt2KDv_LpuoTVFnuO}+bD2xTxgnkL!6Ey(W3 zYqUKKSG4KHC4a$pkFy3_h2YX~qkSmOb-?z9cXTE;9mEu^>)c(tJYXnJ{-;RNhb-*_ z9@xT>nWUqpp2t%-?c2)dpF4VieQJY_NPnO)3!($sc4u{w!~`PlRer(hbA!O1`N4jPW5tdG$& z`C7aqIH7AK$byJi$M)z*A~+uXmK-W|H5_-eJ>%1;Jhlv4k)=wEM@gQ(#6 z;`5!Z0sq_YBVlpnoPTJ6z20XB>c2OyTP|SKBIU01=f2Ug)ku5^dLaj_ZK}_q{gXso zM-BeRnegsUkq)djCshJMHdfrJ0O^XorMZ(U?fNY6@YObb-&4doj+w)%ch`lBt+z+VGorqMh2Z`rwBkw z!Nl@kI8u^|{?cHE>%P zJSRyI6w;VLl)LW!yMCKkYec-#aSgw_7(F*Vc};cM(LTTJ!PAmqQG4wBmiO=>aja)O z>GS1}O*~d3d*5O3)-Nv_Nfm704=gV*EAV{lBG}e+yUNX5scX}WC6h~_#}cTx>=V2( z{k?`34YSEPxi9Atf^5gmAp2qT<9c?2H>TQA<{YqIVzgt-7$=j#z*s!DIr z-Jw~|nr3fRD&5s}^OaD19f#D-YAq&NUzS|b&gYfhf$7(f6ae<~a!g%3G{qXljMFNq z*FMb4;$6YSZ`2I7u$#iEpfbirz+c6J#j;T!IxDYL`+#~{QS%6ueBYJ*aC;M!!} z%WXLNveKwf4{;E37mjXH^ss^>+9NET91RM=d-J$Mg-}{E`b&d<(AmM(VkV>$b3hM_ z{8ap)@7;8EQVvmQ_Ve@D>+foN=oNUqjW%g%d)O9``MSy-k{~YRL`%{aXapo;HOea= zRpLIJ%!`6b4!KG(+!&_DY(??6EZD%Ak%^*|@=;NIn*~}~pKpBwE9IRfB;x{JS$(vL zms8M?jcdZSD|ede1qs|>ccseJsx5hRR2H96z#d|JZ9qu*>0>03a5$}9H4O)PXIm5* z;deOs)rqS((%u?X6k1*|pdA#Hy0MAM++kIJPXfQ7DC8?lMgROv-Gr<6ITYMIq;OBC zy&aC4&56fQr5kgYjWywR4dYDE>aNz9rz@)~T<-^&Mf}f{y_H#%wi<04`)9f=+qkbH zu(Z(;SexFTk|9*qLUma6TM=5}%4M45FffM5d$1^!5;lI~m}u*rUJVGm?2c#{u*;6A zD{YJC0XXyql~;qGj;%IWnq9vM9%SgZvbUgl?hSnE;y7cA3$nY) z<-oJML)*q-@(k;vjPreK35GD+bzeT+Hk4YYxSBh4gA(BEccbZJe&4YI!KPCt4^+v? z1^U*aQk2W@adS&vt9q|jCUe3T0Zad}fKq0bgaYHYFcNeKUS7AC)TAxcqVM0|iDq0% z-#WbZYR>ogl#$0g48{}usaTiH)5WdT1fqU-;{B}DDtu-oz>xGWcD-_r!!ih^2WRic zIw{s?2XuJU2)$~Y$Ibp6{V{ZR=Hic!Its4Ks8Nmp_0`F2$he}MyItCjjzxT_vKY}; z;mt%wYm$8VS<$bfKMK#I`jMOh%|H0ER>ls;@3bxW-FWN?AipPk)WQZqR*$jzuyJS3 z1cUC?d!Jy(x*(w9ch5$Ei=-Ip2U^gp@eiHQGM6`c`n^8~kKblpAet{Hb8*f=sv}U9@*EnpjW=xx0=YB! zsr%2;;OGV{yK)qngdhou^Lb=Qx_YFr?Y74AI=oiOfjQy`iyz@;09RReFkNwfuQ%=Z z5#6Sm=Y!>O@mc*pbDwW zx^E`iu=*u9{tGt>RFud3W6+Z<}*vFJtVgv0m_Z6b1glWiZ;k9Z% zeuK>FsZ`o~oIaz1kl95AVL^q2?=kbRIG7UC55OFNY<8Q1~7B| zc$5l;ARj!%P>L3>zGZ|KWH#keysoO%s|4bkamm+v!sanJ+!dY6E0w>ah}AOxzXRmR z_MN{^ZS23QOEHGMUyW8k)X~sd!m%qt=HgI{|2LXfFmClThz3OEr4YN?e;32qcAEQJ z--*ZHrfEd=0pv;weN>j zImfcU(J{!Z=&Zq11L>7v-dorhnXmFyz(AGqF@F8#V&_lx3kJEuw<9zk1q+`UFIRva z&Q-c4Xs9I5gJ80|qRfy?mcudC14(Q8^^%jP zGE%;i8`?M*rXaQrBfdETzD-1Bp_`~C@mF$6UoJ)qtN9leM~;cEL7>jwCYQZ2Z{60N zAz+>A6`}Gz+yQEyQ3Z8JLqbw2A7jrT52VoZ0jNL7=O>$nxf+oWNFH&B5#D~EU>pcr zl;WR0X~pdIpqMUL1Vk^_Fk@p5Na|F?d_{|=l^vN}=R)NVN|Jv&w}Uz)JY1mZpNi)i z;^hcd0J)eR0<#&F2vxbuPd&GLIod z{<)+USA2sy{3I(=3^M2k#~Yut{7xKk88o_3D_SaJe8}6m*YBFMVKf);8KdEIXIyGB z&TAI(?aQ4|Ywuz@)b9<)&#k_N`a*AWukrR6T-%zxO^3>It=P!_ED~ zg)C3IdNCRj!tUK3g*O&PY5vLcr$tNrmN#pGAlN6w2{sTLFwIUu|G56ED^eFqctmth z96F80!Cu*L@5j!Fm-dgn{3|zI>8+p8VU>%Q@(=UTe4Ub)i(g1!}E2i#<3FWEehU$ zV?`F;fv>&+ScTRsFa`yhuG9xvup9_H6wxgeH_v@B%W}r2-RjB|iBBdpmB$%uN~Svm zY~0_>rw9BGss7#D8$O)sBuH{uamc;8O7(evA^pcmhL-Cze<=4;U^ajS$#mu@z6kSs zFPQc=Re)mFuT+ebp-->Jt7Jlbfq0we*xvMBUMOvkJ)Sr>qo=bil+`5G%p91GCd(m~ zqrn{wf%;ud^3Oh5t;X&SNvhtew)zX~bB8@fWor25Ommi2VNb`w;z*c&*`<60=W-l& zO2Oy%%>qKt{vo&#Ax89)j1mr>Be>}B&>KKZa~PE*VT#y9js4?^EzzxXc%($aE>R`m ze5K30t}e8U@Q%*Nb)Z_mTRX{M8ZOhgczm?DQiY1LjLn&uI_enz>w`QP$$WOgScp8+ zw_Vp^2}t9%pPV4MvPvkBNN*@?n5stR!a|Dd_%^G%J3ilFKsX)h>q8eamt_xX_t(|Z zovMZP|0%a+*8cx9X`<*b5e^g|6$6F&LyPJy&HNc!LB(GkHRa5MLT6vZM*ho1{_J6W z;Pbe^SSG2Fw z;efWd{j{CrCVj|v%>OQSTI&P=(9GA)O`3Fs0{PxoCDU;+tNtd$_mR+O0M1}nTI8m$kJ5F{^PUd zO>-z;B^be{Yzhr|%!?Hm$UOT-2iZDvVM7ON2Ob_^f!9I>+Of(m;(h7-uS)I@qYMa4ooL%HKTQq2>iDTluw!yoM7bP%A8D%pWW}^ zWWHt~JTFRv8P#L{d+epf$p%9Ou|mu|ML{ujCSEQb_nA+Z5iYR9B!?Pt$8SNRv!W>5 z_nP9e$!|z11)B10q~NZ*ZUT!2^mHTGsM17+emQSUk%a+Knehap9tyxK)?+4O_YKW_ znn)U*UPbr!`%vet{x%98(urE{ivL#(KY^SP^8^g`Xc@-@!C9c$Ue$i(s|3<)C?l=u zeip`1OpnM?0WQGwXi%bh5Gt}Er3dih3c}fTu+T`!XnzctdT*0x8koXwMaEKWyM=@j z0)03X)+nl*KK{kuGs3v0iNJvLLs%*uiRl|=Zlh2hl%$^4ngJx>j1GnxqY!x*aPVj} zc%YS`nSJ-|8!DILf(Oe6Eh>njDXG@+cVrez)cz=c<2R;~47(dYx5+zzX%nDka(dIT zxN(Ua!!$~x&*j@W+;+1hxTjHgdpq(thesNiQky5=-(2j-`~1Vos7BIK1SsqM?qc

=(>TRip7 zUQ!ccMe3fRF@Yu^E-?|7*|j!!148_-vjn}EH`2rJy9zoTSF|+KWCDv^|Is$YrVbEZ zfOEmj6ej8awH{K2aKQ#_zt*eQqeuf^+iHK*6-(PUDY{u@Wq3WOr={$a3Oda|DvUHJ zfP5Q{MLFtT>NkG8_dkQ?RM>@TEGW zNrin8M&4P8ZA9aAp%#wiRXJ3Z)>%)%^wf-Dbbf?lgAIv~p|?GTt%32uQvIze-6mQ! zdXQ~+YtnH?iK(1}UdgEu%3gD>^vMG+mR^wSM3L7dps5N@&8~Pc#-%kv25{F*zBabt5r+$E9L5u*fi^EDW)GK9tGTQ_(7j+?;!d&e^v%e)$JiGnR-9&2AI*<3# zD*pAR-X9Wt$r-uEuHCGVHFinoLa^<`YxO`VwA}m4pkU<&B8hW|+mXk_mShgYSsJB5 z_tVh5oc=s0iGR%{B)VWjgb+X(<)0ISr*?efMbT&Z0Pu`>XHfdJDK1-YkMWYNGe zrMLSn3LCGF% z>eR#jbd&qTWqm9>tTPjl3n*Z2`b2e$yBh;A$a04^tXYM;vzAW*=F9gie&2HHxBk;cnmGEd)-3dS58ra&~aA+BWKxGh;9 z5*T>?uj=hrt}*rhpnAZ$k33Jzx^T7%DD1HzA#zAnpqp&4cKnb(h_rKT76H+<`mi>2 z@Wc|pR50$;_I3KzO4h1fZ{%x5$k0JyDu{clu@A2P;YjK)4BeF!Rv5R343pe zU30F?;%mD|{4YVga!*p-I&pF?gSs?q(NFO@5kT}YWLstz#7SA^T>$ayX7;#;4kNrj zI+t8W0`+jg)SHaIyRWHA1e_CewP1U(|1qjN-4a!BBPQa1P-JeBCp8cgWr5$VaNw^U zikAq)fK$OP6JCugZjlKkTb84}lnYITOsU8qQE5M$od zynj$X=$*CMt#Wn}klV7$e(MY{I*b7FX>Jt{NoA5g-n8X(uhY9CMRNo_-&IZ+Roc2Z z`HV!%E19_TERA?UQaUST^Ka<%$IZs_PdOhc`edr}{XY;#osk7{IcuPSzLa6j=%?pI zH`9u*-RI6S>zaOFJo!JKsOz$UU$rvV~6YiEYUTub1}u{jM5!mr(h-Ea*vDZx0t=U`b%a)*EJ(Z$>+8MliwW)xqgRr44Xe-(*Ym6Z-S z*Vh_P%}esss2h0DR6Q=0U-)`KGs)E<#|3Z@BLuONWG7*3o)d1kct_3O?bN4&V2$p~ zk<&FSHN*LxY0KrLrB~0OWZ(?y*E}uzZLr3bH*`Vf`pKIbO+%D*s@lut)!a8olH-B(ZmWm?s$7qm%6|`th$DcoSUcbPdZzYU z`JFO$Xi8Yrz3)u{tc-wO>y)p)35lZmR`65~L`}H7L~u0TGuEEfwUaG`D6)9BS}Ds} z$m1EP^!*dY+g1!;I!qy;`t@+0Zy7hyYdY0?9tKa#HD& zz1n?COvgNI9|`N##p&&BhS{HY`ThaI$(~!Vnr?WPYw^PnEN@+kT5`&3lW+=E*YE^T z%kdT86@q4XHRAwS|MeOertw-$|0lL)^MB?7p}Y?cv)^nVzP?~q;_BcN1no~%>=C+g z>|aw=U%hvVXxl{48?NlHVbIcOC#cf5>^e%YY`M-9h4N$L2mq`oQgTm8#+221V4*XE z&*2bEZwbt?8iUHA45+&ife>^7!8Uo87mv(&+3IQJd+@paI}FNaR`fmUN^?Cz6~Tzi zF3vmYP?eS(8CiVV^w5D1al?(bQlyrwU;;g~X`5k^(x|a-7yu?E)Ni{cg8Xx4jah;& zg)BhJ)5jv#j?AZ!dSLshP=&yv;d`M9iD3R{^EJtQU;xr2T(w}Ge^ZS^ZV@;n7zIz8 z!{l>xkRN=!QYn^c7VL!^$*Q(t$(IW&AJz!H9d)4Ye!wiHyAo|Gg)wVqFrE6^bf%*1cv1j z7iIf>ikY>exqJ_{CB_B2{z={cfw|ZUL7DgI0C*-n<|e7oBd*f|JvU`F-Q(s%P^bRC z4x!NTQ?Lf2QVS<-{{7xilb6G6h#TPp-)cI5i(h6~j(5%`V!&L-Q}n`^;0oSX!Ng#y zaJ{GV)dp7g0!F%}Kg1TWf;Y=%xt4?f|KpedLZsE0eN}=2m{#}G76mShzTmYotcU39 z$tr5+G5jI_htTC}b~*g0v@Hw?10s{e7r7=`EesP_&84ZxDDR|YalJTez7O()*`&Yj z!zwq>IURWzQ#5Bb4Hc~IN?tsWgQ>pR{&azYY)u@py^ueR)b~vs)ORc(K2N;0&qnDE)h{5)lG~yd*3-+Zu=ksFX5VD$F zsqDG4?6@4=VzA(ul+|v&&yWrQR(WL9y!8k%0t51=vfe=s^9j(R%>g7cnAy6i2`gY2kE#LQNr+IKl|O-J9r0G&%+zaf($joo zWV|U6*foi+xWt0QRsS78w&Yrh7)^hEM%GJYjiQyAFQv`}s8cQ zCjF1KH>E^%{fV5NP_E}~C2I>Bo&{OW90P0p`(I;yIwM?RS}0-dRAj{}7a*Wg`q0j8 z_IVk>WrdcK%}qeJWQYEA;u&K3i4SaWG%rdcTERRdxO4WrNM*7eIc<}GmT6$RAG=Kw z=@Mp_p7-G#<+bL*VjKFu;|y%`?aIi7hq+Q@W@!*|Ez)$~=yH|?svzrb*@7FmoOdU1AF=+x5=lDGYjzSC-{RH4xj5w#K1296} z%l`ABkC|!2BtKng#h|e6yq=Y(o{=)~m`Ot*m;vTtXjM>WF8E7nOC64%Xnk$Leh$~i z#eUHJPtxLWO9Zj2_7>cUcCTyV1b|{TwyB8pe zZ&Xu}!!GQrJw64b$=RyPZq?Ue6j_)SwN;anz_!4~0Vk$j9LM`T$FcdLM;E!3Hw6_F zn->jnP8PJhSJ3Mm67{qEb-05ev4$D(&-MazLmNH(4Tcw)#KmTG3ubICh1edXKp#Y> zL{?4DI2Y%(F&I6V3{{DzyU-9#o>tgSr}@y2b&ai;;db;PT? z$_=6=T2>_0*eodx8J1s6=5Q>SzeL#(2Lw>_8!a+}_QPH;BblhRh4AHNaC4`9J_I?MMU++cn#G{;Lw@ViOM8(9Ik{2c4G2MIcxjl&SHJmBt- zoU#)5_qHh6iyHkSChk<~nf(ymBM<*gHWGDU_!w=!v#?6pj8G1*#Au|O!2^ikMqnBW zh76|D<=)rBZMdm|VOz0VbQl$rY;_dd8`;g7>6jlQkvQf}R9RBG(<6v&9nO878x6@} zS^~#cSN0+Iy7;e=;(u(o?FgC^1haE6y&mf-o4Mzt%C5$GNH|0Q_yt)zsPu)=AveE* zTpGl2@G9=1WWUsi-&FdKsSF867E|gefiUIrkk;(obTwCX3wAYh;GmGP-;-9}`r1l` z4ON9gy;t>Ph<~q*%oO592T0i7gS7>thTQ%ujc@^RG-7Q<7tlF@u^oSWOv!f_Gx_0WTvPmEb;}DD0lPVjCM=?!aD}^mE@+%dFqQ(N_ErGlh zw##qRGd>>&>y_>8yIW7=tbxMR{X-Q&)u1zTS^nZiYYvg0SZUj`B!R;j#Aj$&0}xX> zy2cXlY2{LUxOE&lAqkJgGYvQ|XK65muTa;xOSSMMC+U-xe~l$vSY$!!$Ek&vK=5y! zZ)T07P!V}yyX=P$dDeO1ypA5 z(c2%J3c+Vv6erw2q=wBOG!Oz&O=A3&&p%I*t`^EKcBe>S{o2}&fIv4DkQjASgWfxfqAS|RJM9a3rw3?>xgO|BP~LSr50J|@apc63rPx>8)XFcU zeO+tdtm|7`X`BazM%DR>!K%C?Ej4(mo9TWM1{Q=H2Sv#jPGncCDaB-Bxb9Lo{8bpKX~B+su*}Tz@eSOKp~>ZZB-}e-Ix7jy1IUbJSIrgezB?I^YlMf2Hsf(ef%$ zPXRU02PyZ6W0GSyvx(+4DcZZ>Fpo@=<_hJB!uF({`Z+c+Q=6gnY$-&0C0zUrfd$mj z)qS-b7$kOljUUxs&Cxv6(ZT8BOKhMMl!80n4YK+NXm~)|8j(-@u(5DsP);ou|IZK^ zM{LMyiga%{Gl_t;B8w26-h%gm`-pDFa>?H@ZNDl}=BwiP~ggM=@(Q zCmj%~_m4%$mN)~i5hOn+hXQ3P9x?US!W!_LEFShf)Z7)IBu)t;4?_*+46mH{!Svr% zy0j1Ui3XnG+-yyQ%6;$=$ht{o*DA4@lktmMK8qB11HFquJV%H=RBh_69_2Yw8@$A%x2=AD$dI9`@uSRLQ1Mr3C^YE`ORZ~hHjQ0wo#8J;9hw6Ri4)|qGiixUPpJ=3;WuG{M-R^p!cF~ zK*`ckm}dro;wBGA><)ogGY7)y4U6CaTtt38Ai}x`CHeMoF7^YWBWBah&pc0Cgm&TzKwGoKm>h`i9HZJb!mFrI%NkG?_$dm6G-3i`o*R)Gd_7B6kS12?TjTC@0WOnOCFro&2)82p7Eyc zj0I{(?i|Cpr9Q6VBZQ>kzf*%*?b=uKW5x<=G}yWzF-)QKJ4x|Yy%atsQ$HtkRelUiij zGmcJ1*$cQv@St1y$r1x-)9asBhRuAYo+jl;RHit+6uuCbFNGs1MJqb>{XBUNnb2wt zFIkE_K_-Kz+4Gh+Vphs)CkRRa=u(gwLAjV5AuE4c-+7Ly_LSMmy!-LfR*((mhiT-S zLAOYkWGz=UV!(#8Hc+{je$~Ie6EU$gkNNOTpt9D7SY)srV;0Oar}*3~kVTgos<~xa zR#-$5#GIDJ9LMN6<5S)3a7__n9SM0TEN3}w;6`WB~mcYiJ(yt@`d}?%0NgBEzp0bD}AfEVbc~blr#%3%;hK&siq>W z>q+;u8YbTjP>_ClJDM2E@yYp&;&-B-&Qm7|LtGRh=olULI%vHh~?#qj3@RP2g-&PjE(3cu5D>o*%d9^&K`MEJKXxI&~_+!VDL^L zvtg^N8J-LIT7FB)zW;w@2a3Rs6#%6aBpsX$v8BkX zm~B1^x|HY&9E%p`N@0bk2#21$Yt>4dDT3u|z;Q7xYMgWQ)IvcD&`_0sS`zn)l&2#A zeutVWIDVA9idHFdBNW*yoqJ;z8hR84H@@9|C%*SO%u~CeynV~AT0fcKhI2fL3^F9} zM%GZvZ1a<0$qNEps0eVM>aJ5)5Myeckxhjr>10(8K<{6lg~pj3y1keLY5KTG)so8n zacl=s6{XQ+T_mpY1xOyug(r0(AvF-uo6yY?rBjpmCwuNUK%%0+Z@UM1P&`0Le0ucSsSZ~7=-c8K9LtX53z!B@5+ zP?VO}?=R(FD=3@TbSNuFDViN|+p`J=E0<_!@J&=q!BlInF<;s$_6fsme^U-TbyQ%n zBaG*d3*t31UtgG}4C0Ho6}tDiE4l{~bG?axo!t*^+J&`R7KsV7A>x;NnO(y)QMXbB z3uB<$>YQr;5rpk^9|i%I&g8Fjm~(QnX-CPJ&LrR+YVv;4+O&;5lSQkLfw6~h zcp1isJd6Z%ck5n0hFi)A77EaZY;wg8+Y@M}wX??fqd%xJm&Nv@cuo33YzwRT5j>6A zWK7x(E~2@&%H*XqMB1K-oG z%6k8ODXbz*%F;^UqaHQggnA{Q&S895&hx09Qo|ae`eJtEkkZ)CYz}~>NG?SCbFNnY$F)2Cj$`P3%zFwszu<0`>{d8m; zYd6GJ?O!0XYmP#nOuMvwyj0u?4~OM(U5;%W0$}F0?*W;wqC=Z}LsIFQ z$o9rboAL)EDHIdZM@>3ph(w^7R8!-HrD6hf^9n>h?$gw&m7BqZTw<`?Vk2_m%PMrZT!NA?rnXD32T6ELg#tgG5^apd;a~3g^=G{j{+8+ev4R_Z^wl)Nt&*Zi|!|AR-hHn^!aL@FE~AG74YByI(t!lsM$+ z)BWEZGyNcX(&g}N$jlA0CBo57?@btwMRnDK!6$&K>%`C3K%Or#yn9n9z>^2=U;$b@ z#@@lJs7T2jrX0Ge9hlf3GsdWb8;&5`l1!Pd$T9xHx19lL9Ea8P=4L)Rd$2qaZ8iV?*exPT)wY5TrKwTQSXU5~vVDYepLG z1+Yc2dxv2^!mBytIBqqf6;7Z zgXKD=b~pFT3(FENud@;RlpCLl%s|-O((5ZnwR7tc7ix=G_;{aWDLvBuTQ9Vf#4+$4 z*uQXuFU9I=8XD8YFiVOd=B;P$WMm^5j=@lm$AzL7!Y#Nrj%yw!M(;UyIbHx7BZ2}Z zMHT_h+;i~7aGL|GA&9^9{fC!(c65-_&TCNqXE_js`tbFz2f6%Lz+G*9RO}aUN-0;& zz%t_rY}Nr_D!eL@$7*aF5*18pN82M-4~mIYqUsgNab-uQ!2Z9B+R|& z2tJMmGr^qC4Wa4r$%Jt>xKH=m!o!Cn6zpFVO5)facy=5_W=jNp({@S*nzWcyw9PIA z+~aO#7~7yHH_`gRt@^8RhkwSoLN4Q^?Hbkv0VNGQ;No#Ve1LJ-d{VnE^G0X%u_-lIC8})T;My1RWUn+R+ z+EZC{GE!*xU|)7Qe`LRb-BETQ9SxS|BoY|~vNdaxDodZQ+9E$?=E5a{{p<*LdXn`% z0n6Rurvj{+hal^1tR;KiW~y3G4$LwBZ(mnrgx({G7m`wfS@ya?%kN}|IlXju<}_<7 z2vG4=#k@0G@opj3SF5wKwQqmR`|ME5k~dog)n!3wO;!BmTW;K8|Cq&&nek@^4fHUB zFSuy%|J2Dr73k`FbmankO^qexNS%*lKThyazMPyT=)A|yRIukzqaObS$~yEvcjJ5E zuQf-D*&$4rQb!zbjC3dW9S~j763WRUdC8TpyIXa1sS4!pR~v+x+THZT0SW0&Zytz3 zHj+weTkB6@&{ZB~lMawU^S|xlI~R6esX62F+mN&W-)lf75v(J26a}qd9}4u8v?k^v z--SA`=bXEZP=S-h<^_I4j&1M%G5&sY1>*d6^dcJoP6w(4$1gC_p!;6+it38fzFa&v z{8+yc8`&Bty|!cWo^l*c)g8}x+}e3YjnAkS2Yaxq6t^khiF_z$%fcp0SK;3^h z7(!J-JTlovQE0g-$~Ut8xJsciVK+rW!&or#izh0)An_&9V!~vVqJBs&8$%T2HO->= zHZvEfrAGB@WB2Z!B(G=x1>3b!@F2enH{j5xR^k190eDTO=*4MKB8{vq4`PpHzOn-* z<(j*ao-b&2BUQuYnS#w~-|=I=Kr|ULk_VwLzTX*u3jdQLdeV63$y|K~?*=c@RuZ%X z)Tf`^%|%sJE2cHm5s{ZRUaC_RzYRXcmo^_hajJ!>-5C0=l!S>M?_}s;Ods$VL6|Ow zL!@NaZV5`r9@%M5MeKf!hpKWqE@PJb9`TGk2^pZ5Ob%b4A^c|uYS(zFwpF#NAZF?U zUFw7?K}4W9XGP6}NKSHNK5&)MDQt>-FsMlbREX#sz*xUi8z2T7O#7MSv_zEtCeTbb zFyg6SA&mV@LepSl%MPs3xd{DIRJHWZyJV_u%vqYh8D>T2890g3PmNEe?Mr){b;m3s zTF!w{4`ihg`1=@L{+M}EM$#ljmGUyMHtcskJ8Kdi<;!OivCvosJOu5#+s*vAv+}${ zXH7CqCt?QA=j#!u5xvMur~j#@%$&Z$3YGb2#b0}jT}hIpLabe=Ll?f%79dny@MMK7 z$G6;M6-zb;awi{dCo{=p+29Cp_wSCs{&8Y_Q)eF2j|fQ2?A&P|Rp+UMG_>^e2s&EJZKPN}!wopcqXT@(wc2s^f1TnD#YRPcS zT&P|<3IR_&;RFG38kRLUtwLa+0af`HZE}eE(&TfBHPk$~8o$F)0pfXha_6l_1nIK2 zG`zF1J;%_Z4(5|Q134Uk(TBul5F9ds)5SWiEe7ooRb(HPXG<0*SG15nwh&&4&F2)K zWskHHa&_p}lec8CjMYkRA0WEk4B|UO(tBJrhqWPNo!02{wui~}4QERnZRM&ilt$*n zzvgar6}{4cDN0}`Am6IffG45=R}9hi{(j{=fJiC#q4ghhdOr8(-!!z6gKKtRPDuK$ znWlc*>5C9xQ!4IQwc()CN+MI2X5^=6MSV^U?ru;$LtewQA;t`0+U}G=G5iR& zz;tIp6%o_>!K2-I^*U8JfYQ7l=?9XKGY3+u(7xm2qC<;q*d|;;fe%Q|ae;yMfJ430 z^}{8GB3diHBeKV6ffN)Q>rOrAS*)TU`X@p7As|g4Ltovi*Nk zhnI<#RkfhMe4NG6C7RIfeOb0y8Oiq22mEtraYmu-UsOr8n=-oE+1)|1SB>JIk6C^)80ZAT{ZD?-4BL`ozhl^YNW7Z0 zMI;4&?g>MT`8mjt0@@O{zM2=HFxI`#+D1kXJ$YXT`FbE$kDZd?PP=E&-;iHoTsuQqr@s!BuMQW=EfR6;$L>@+ z22&AYh302@?HhVQmKZG`c2``uHiMlrLk4Gh8s`tyRqqDY_%0l0lgjq7hx>77Jt; zJ)%^6Q5hVwPF$x{=a)~ZpI^1|0B=jD$?ulHgt5n&B&*66x^1rOx7LdPXdDZlI2ANC zxKF89fOiE`*Z!d1vX(D-XsAo%Ewom=*O%Jmqh%C^8X zb$@X3WyPUxSto`7@I6!WmU{nuAm)sg5&t&tTr7FJq(wiGTDa?q z;ArccV`tq0#aZ%H`VBk34?A}`hl;`BP?ygix0kj?rNBT!+#%-OQzKwBAZP9ol%o~q z=Y`>cWJZYcuS7lGC-m%~1*;C`S8Leay>1=mM%!+PsH-9kjgBuJ{H>*UMX?mQ(xfUM z^%F{O32kEiW`fX-ADh4UF=~=7f`4om*J-2U{W9zslR)aUV#QNO8FM)@bFhGXn4`#3 zCQKJW{??9y`M_7iX%Z3cnxo7N+A2F)d_6uA{w-PiTfPtKqmL0v{U#XpkTn2c`g&$t zI>k?NgwoS#qY<)I;kLrwg{}Lc069R$zYes@7>^-`F)nz26TG7Rn`lyLHC7BuKi-Nvjc9yM2mtxnMW?%Wzy8i0`zQ+XdWEJuZ6M3vz8f5SCy7 z(zInFXa)CE;P$CW<>tY;Pn6a~ydW11Jk;e623YO)3Qo zua5l-lXttd`cHE73{|T$+BaVsGyayITYI_!g4RI=+ zaR4@kUz1iXk#xe1LEtBz^C?lph`stu$#s;J7OfzmIyiZK`d;=~AqMX&ht(m^jB1=+ zbx2)AS#B&u&XAYoD70Alg-6Ey75Z-!Yi5Am9Ve@T>x@!c;$md0@HPPy1tR`GC6{2h z2*Db7`mSybbr|~Tg+>e-ep1eiwv`6CC}b_9GSX%|=g1p~tL_LgE*1M+FkZe?vd?5= ziDBJT3N_}TF~=Pif>UvlX+<;Y^DPsqM6u9E{&Ywd^Q9)reYnwpXWH(+(t` zbZU?s1^XN55D~T7D0XI5nzEXv0^{+mqHF1dsix>uV%1u(0>WF>gUa}GuZdqiX+N%< zw=--HlQY>TMn5L?-vn0l4$`daN|*i8J}_|qgt7yAfAN;yP!9%uIGu9u<T!rb;ay&Z zLLZ`xd?t9Q-7(rXQtDsdPF*WSc6|Bj#X9B~*|i~Jy}W~rNlCg304r&N_|I>rWD_I$7d$8WuDC@$bYaTe=V2MBiw3l*`NVi~Un?^_Coaa0-%xO7=#FDX%>`mfrn?`MR z$Rr=Nl{kZ5{eBdE-m^@(Dw2y3PeW6`s{{AkycS+n!910H3;jT@TS;cTQjK95D3S0| z0{@DA5hSPGe#P<<(IiI($^kK`vlUj?j~?)l|J6rqGS%J?V_-#Mf7&Dpy+Opv(2b$O zg5|tyO|X8~RlE@GtBQ!z{`}j=fBWccmZitNrt1~I6U;;-d-L6zpS#j$gnq_eP_q+j zi1e>me{*wys&GLjZcPv(f}dlGwz5BlP zm^Z|JnMmzX&G?aapZs06Y}lE58;y}L8i0h+=am+bx#3ETx^yiwecXEzLQZ{HJ}*&Y z3>H2s6+hqrqJlK^&8MVp_J3Gf7$GsC<~uEJK(eh`b2@a^5iTtDDxN1@Y7V7MbbKf( zl%Q3trV=7!Hw*UXAJjpLw^$E!4lQpQ(p&%Z9>p~Yc;~4h>$YnWt5Qli(N8ECy7+2P zBvSf_!+yxzx1N-HMc&zQfSSw{>I|4?TsZI;EdhIS?ROI#1y_%gr{)Hf(j+sTQ#b3* z(e0-WRY`yWzqK`Vp75|c%eQJV&>mzVi(eJL{BZYCv67{sbI$KTld!7Y(sXz+m`lzX zlZ#_S3!=uVmAdeE!k$~VL>H-M0k&qYJQO!m%b93=9KOq0$CLkJw#RKw}3}}$94UMWtk4;4>Fgyh};QQOz9Lw2S0~EwN4TvM4mf1>o6bG_cZvqhTS8&lO zvRo%?snqrEl@g6H9IrHQ-sPJx#96qjQ6YhHvdSIgZ!kM_#}b~`cxAMuNf zUOz}7_j+V}!B6KV_)NP+v`BOYXaX7JY1XUY+|N6J0`_1faM2?~^)iM0)#QM9sz7uv z05mueb6Fv4X0-mGY2mN1_waG1RA5A7%3!|9{f6>{Y&<&ez%MVf@1vdBX8ZuddYf9^ z*M7EilDZ_&$3O6JFk`;bktf+8Xrp{Z=KrG?t}V~KJ&wi)_c;YuTzI9QfI&$_=o?Xz zEv^)uW#$Af;o4Lf*3uDy2qIo@#=1y%|8)8E*PuFZI^N&D;i##gfryYqAyfvpCa>5+ z@!jWKLisWHmeFnm6atJR)4A)7t#jz#PmnB%#aWW>hi&&!1FB*K6kQ5tepY8dA|FRK z;LFjE>_zAGSl`M+DY#;(8Nbw0KIH^i|L`P+e+*PI4EHvRM)yfx|GfsG^&q8;AKS~{ zoaWWPhXnCTc%u(G<-M}_5&uY}(ECBQ{U5OL{>${oHs9LW%#_O%2(r%*G&qOc7_?$# zO@5Zyvx>3i%vDDt(v(gBVcnwGKn36C7IJB#tr7v;X30H2B?tDyTpGG?Z*J(`Cd~dU zgzH4i5y49-N3RrEa67{ASR&dMU0Ks{ob~9lrv4YFt24I(*p-UB-$pAk%DwR^QY>vD z>)qc9NZHiL`)tA;Tf^&#YXJ`2Zd_H@Td4OEn1m($yycMEY74UOVWso$_(#1@WP^~F z%#Oz^5f$4ztuBUeV#PV}^y1SjmnNHEOf$!h?H%c``sG*p)TV(WCP1&qwDTXt*51p^tXLewM!fYdQEL6Q#Q*4;Uwj zaNjc~D~_W=XzYZXP#Sv4|7i6PNUpZ*qmRvECpFzxJ=vNjTxAkV0W>Cyv_sIc*@x zb{CaLl-lv7_Tn-N#`YGJS^KxPUyPY|^}GG9eM2wK{;D=gk&Ar950S1q*-C{MWh#}K z`K#wvNf_!xK0-AOMZ~XjvtyjfM6Sd*o~`im--dkR)Yj_7xu3GP4v5~ypsB;l^Kq+( zuam~y-uh!+p{KrEt!=-6Jvco`0Z?RehIS=<1kMrum{IPAcKnhDbQienY zcpBEbTt45V3;5vP=R_e?mj}G@Z%aR-)16PC>|P5k=x$VWmt8gb{^p>SbR7`O!*3kC zv_&}Mv^zFs1p1Nk(s4CN&fHo36!HMvhoK1-GsrV+#|4Y|OL>}+zDJnC%C;C53@h(! z%k?|;UhpNM`pt$BbF~W}e9^Vx(+tq)IXFd!xt{Fx8!cwQ>sjo0R4eSVt+}oLao-KD z_M@B|f) z)z`>vG$W8nWv>f0;VKKkg8B(Oo*jgUVeWsq6)onQfWcJI?b5~{`6wHu?N?<*h{?+(&XyeUd{qNIqkRVl-kiQHy~# z?70s(^4V>=-x1`s%A}@PJ!Q8pH32q;kTFvUJBl7)jC&B(P*01Ak{)pi(tW?QK2SB5~K-`x#f&lmD-rNQQV*5(96Za#;2`n zaXJzT!>o8pdH9B59DOI}EYAy|GliBYZ-f?NP}He1U>$XPzHi;|K_l3ce0}qMFy726 z;_M(+{@$JEi2|oE&r+$+g-=RHNCAtkW33$j$kI%GSZ`=5#Oyz*O_y+%M6mL(R$;xQ8V^ zdYLj&`yBsL5djahgz%_JeTSFn-;~9lmQnj?qPzDeacm|gx9X}j4l~SqdD!DMu4Q;R zl}MnF2zsB4?REM`d$K&;_g5^fUNZ2No961jwN_er7I(?g@bfU~%HVY5= zTUS>(!y<_YwHJlLOFqf~XWhj_uMbHLS4=W2j2hi&$=g8ah>as#ITWIc|E?Pn_Z@b0 z@29t`yBm)}1U<6{Z`?8!nXUf?!0uPMGlh9vS!JX_I@u6MA|z}2Dt*ecN&U zE+7l(Q&W2}naoVXB=YCA^K{0TFR8@e4XUSjEX8@d8fa zSCIeVxxIBwRu>kdtsU0!6)9ictczdT8Crs3b_hzR)Do{+;b>CHO3h-R^qAiXR&k~} zS2cEz{M5iMU;)dVIZifK=^Ub|t5bSzR5s;RHy;h=+h16Zhf+{Fh3ix<(zQuOKjttX z<>uHq5m3kLA#~a%prKRV6=QR4%VDzSOQDcaInji#!3rX9=}icKFE`0bCxO-8vZJ?O zN^Te39_SckjW#8B&aEL1K-UFCx{G*0fxEMYCw6RccjYXy>2+nSFNkX1+BKoF#LuCB zzc%E_1%CsZVe>F28sy2}?LIpU;IlP5M%$PrHgHhw74nx>xScrntPK$^2!d0Sm$F}X z5pY!8N$P!0DT!m2%jQLK_)lY1V-s>|8%Qs?h5s~)AQnaMlbb9!!tnSrda{aw z>wzT3m1A);txvj$|7ND}2mx#DLzn{r>36Bfq<;&jwW!q`44U`V(|nk9$L}LLUMAiK zJIO(NN^m@^`GFs$!o~533}|#Qv3zl|aWa!ed2NS=_MK@V%GLe@N^+A}n%%aQFN6v0 zMT2DmbnG0%&z10#i3vVmfhf<$8s3v5wq?*ds>@B1IAW|GqFS2d$6awr=Gc3Wik(y} z%_>wnKx8tyWMn-WEr==N(VOjke6u>=(=D<@(FTl8++H+@sIX{nCZvB`_vIq3X_$$G zy?wzdJT30b0B{H!q%HGA1umF@4iesm#9{E5X4lR*N~hACV>533k#5<5Syb!{a~j&7 z1tikPjl#9W_is}hzp9DjVGP@q(H58L(a;Mv!(BJbtH^~A!9*rY%fUHCOI6R|JF-v zdTR}DyXX)Y%AyA836kOVq6is%|G*%cMd+aQ?o5*yZET&Y`e%NW(Ubc+;GSK4>w9Ai z+Ldi>@RmN^4rX3?v-{CFE)bb8T+0KAmOH2ht4#Z5lgY{TqFTY z*bQ_{Zv9&Xt(>{;j*ofT8eens^#XfILG(7Xky}lMTQ9vst_F12QREb>mqRmvQuNJY zT)g``SMv-fn)^-lw*nwSi6F;CfiI)OZI^{th!8OeAf6WjvQ+LIkiX};ljz(e!V-GJ z*#FOSz&>D!GIv*bhZ9fi0Wm5k#GA`;HmNFus;GV09Uq@)Cew01wu|bB!8#o-dTq^F z{+(BP8g)Kbx7vi;$hjK2OEp37p$?*i5()SI50n|-b!Qe94Z|wKq232a(fV1Fkg1fv zk=kO`d5oyk!-`rQJEMT1kGhgKuxj0A&<*UX0xm_{F$zu`B zI;T6V(BM4E*&ns=mdf@@tX0?UR5-eM zLUP}aFj=2(K1GUSWQq*0?>j?-ENB>~w3?y#(M^cOn;(3Fd1EfX;Iyh?vnM1_(ySY( z!KAG4I&lT-|CMgS=Ztgin;Z-F5#nf^aS2QN&y9w-`8X_Pp}>|_aQ9i>F-cU0i7T(} zFQ6wVo&~M`Gs)|EVVRr8+$oo<;(jZSFPwgZ zwX9qWhKg7MSEv?Y4n1bN5p+SCQ7lMPT3Cr955q#__kFBMao(_;NpY;}%Q-nykNayb zlCqZgva1gt>W|O2lzrrxD#I5Y!b&kCz9RLR zG%)yG?)2xbV$;H>Lz)G7_ITLbHKnAmQDABM;WwvxRw6$v3tR0oX$5|xcccuq>EDVg7n z={OL`^+8vL>d7`PJzL|nSgYDRw8@*T>sSU1c998&(EmV}Wx(0!A3b8Ppb_@#D!}qd zi(`OkTx5<=+N>vgVbq-9cZ_f_?#Jt6z=cHm!*`rb06;FX1J;p|b3@&A0Zwr3I9%i7 z_)@y)+w(*TP;}sEZP_@Q!BFwb8UF%1F_9iYY@v(gIhMjHLi!`?^4GZ`a>_lIGqyqJ z5q#M@1EgVZp&~!nMV3CKLO+|l9oQe7g&_;ondf_&YI9We5F>WD;I@d2viI~mbyJS% zmdFNZ{~)zLsj%vIXtF{*mE+4?hjf3GnVMl+9h|Ka}5V$a>6YC8Xx zg1ov)Pvb?cSuRu=0xxy>?alUI(v#hXI?sFsY(b$&C%zV4^~F@b&>T&K*$DEai~W@R zkjGw&`&~07>R6UAlXG}{6wvc|xc!;~*y6s5ri;@A9fkGrW@aQU{);i%b0_>G+97%) z5x)rfj0K=p3egS0T?$Uw1lN!-8#H-K)Xieyy`nbFLsl$1&OV-=02-&qxo@o`52FA3 zE))<%k67rEid1#hE6OKZ7<@EKTK_prCL;ikXe<& z_SDfNu@NBZ0Bx|xDpD79F)<2IKn*y`G&tM)_1bf43WYv?RV%f?aU zs$M@A$vi6)Qn*hnD8%;ATVBv}i=uWwm6KsY#wR`;wi^ATw-gtUr_{7e z6Eo{N+A1W1p^(vI8c+_*`sNPy(;7OKobnSbvB2f*|5B-vh*9zYd!brCCY(Rb@U1Iy zwD9`m0#>A+aZ1cd&G+zD8={CB#N=)9v7==WcO<=Z{Utn7A!s*yGbCFY_)=~Fbj z{YkQ^-FL$EBhH|cB~h=kqP5vdvAyN3JW2y1^cv6{{L9}TQZ0dxDV?lV;%`lm?k=X) z2UYEglzg)CoJp!wysF%?gO#wP%SNlTpckxNllIuWvIcAk<86xK2s~K*fKq(v>{7QyP<3YKRttpWQMm>Pbi?fT`l>M8q(qZ2{pDUH zgCrMclCgp-ETWN*cd!@qQ0^6flB;p}|D>;L9~kMB*@?(Z6FWZQz^4OB?oCTPQVcp0JI_-&S4=inWq@DOV=^4FMobpB6d8ohq;by;E9anB5$J#TeZf@a^} zBxlzAztKW~YMDgLZFQvD&J1IvBzVZnp^+9o96I}rk%L4t()I-#O#HU%d_xUB&syhz zsz~|9CO;8e+UVCx=JD8fA+p1ExL{`A}aluf<>rNF?)r3wi-Mu+ox0a-o z^wb3;I?(Y$$z+v$Sq@hZ^;LDwws$r&+*CI1PnuJ8-!i{;ruTIH zR6?4iTj@-E0~J8nWwt%b6AEkqAZ_W-qlx9&Oz}l=+s2GWUaCtQVEo9y@i?S@`S?JR zPK#Q9iEf5G!d2G;@*in@KgY1zz3)c@a&GA>KzBAa>f(2eqn96{{A2Yl|wI@bop9hfn4m!kEAQrzAVUNRY~%@@BQ< z*pn*V>Kq`QAKA#e9XHHNbp;Dxy zU**=0ruO!>KCk#4hWhCEV7i;|Pe4D?Z>1nTazwgu?D79X38Y9@{nQ}};7C$n=j<6+q5KUO-HudX)^+c?)w@1LIb6r6 z8wh{Lodr&N^5gMR+vq2{4@C!$zX}^z!Z8v_aMPp+@eNll zc78d3_TtgJ0+}>T*Kg<$#EHv$^@~4feiTe%A#WNKmM+C#dGl!ts_OSO(DqWlAkGgieYT zjo`ww?-$9{ePE>@ebjaOB)63@weJHWQ&$${GhlKPgGWT{E$}Ist7Oe@g9;8S=ht&F zuL#m$4uJbl<+moDus3d9|D<|vjF5oM8YlJ@mQZBqAtaTF1Eck=Im z6c+|Fcxm8R%zRw~7!bC5`((z+=@?%nPbPJGQWm|vY1m?Bg&vlTxR*B4gHKdeMde49 zYepLv;=1TK2VD7#3LMMjzBvYW1%EvGH*-kky#63N7S$r@+(jzBx$$nl#Oww!k5@2p zv($0C%O)tcOg(&JmUG%h`RhjFXd$QkT=*Ne_d>u21-*w4(PRX>TNpD(ELHHMEIO;9 z89+<+Om3PG1ZK%*HCBlw<<7GR^SdQ4VB9I&pRHwH`KH^xafz-oq2mFa@lwLrsKxj& zXz+S)Yuc!pnu}(5Jd26Uui2{EDUg+#!Ik|llNEg+kwmA@*up~9Di;3{XL<^ACS^k2@P^BIK=Se9Q6}GSjsBfs zq@21{Iyt!aK_77!5u$+wg9dJ2Q3|&dzK*Q^^SHr7Le9OUY%*LwJ_jumPGzRMemU1( zHJJA`M)B8=mdE{kV2ek ze}n(tX%2T``aXSk!3P?MsU0OENycsVViUPBBQJ}V_9*Z1JD#GmN`7g#lE~VIF%f?S zbmEq<8>_XlX1KewFqr0S2zZYsceCRQX6;JBI6$`VWzvs=iY0XKwMc|kvlt>+n3y;e zenYckq2y&K3e%!7}Gg;+E})UT4yNK>n@A@NsFLpey?gBRkbrgDa+*rnF2g|2V+x!kL0f} zm+P+9;N23s7)eKJq%eBkKC0~B@AimAg;R)Dw7m#XgKQ%C-)$5}DraL+t%AONrozNS zg}`>}l_)7H4lVZMj`YTh@29SeG2|PzXC$_f47$ZHi!G1(<|KI)^K&Aoc-!iKLFbH;*yfid7nwyHSgXh zaLGh5!K1NGr?M<6PU`PW37G&!gRt@+qTaVTcz8Bf)&Vm`G)qcZ>qb$=dQ5mlmWx3v zh(XSplgr-C=S5qxL|NeUU;-$|T@hj`;*5N@Ltk;7F<3|SXX?1bMfL^)0-}2)Y3UmI zK2l(!=A0WWj6*8$3_Y(xYICQpYq~aajJfrF4M0bG%Ejn9^M2HLqSf-CTKR-e{jfY? zj^jM|0vSQ~{{PoK%NxCv;AJypPFroOf1-$sTF_+Yc`P#*p1%a$~6fggNCC=lCsCwL+o?Qn~ z-6JJ6Lbknd!g)d?pUfvkcqvim54I6(ShXNq3HgK*y*iveJ|=TO3;F`E*5q`Flx|X; z`W40KsHH8(b`*pGr5S%`lV&LGB_73Z_wh?z_U+}U z&B%9zNyosgfa&wgG^*TcWdudke-=N;x=6GdERf_rKywhK1>nTV3qOw&0k7D)=Y)9d z((R;%CEKPQE7q!vU!wa$keuT?Q?*_dOHs3WV&~wvk{55bt3>~G2cglSM!|>@w|su= zZKY)LBRVvR_K6JUn1Z7AK$iPjVqwX?+EM?iz&Oc<1fmqF-myi;Wl|K|TtG4LnxUH9 zSeD_JWjdw{k04VdfC7sf}TYe;r&G85OKx`*p_F?@G zs~*mPc2v8+zXEw2d~|cB2jDr^Tg5v4V0vYHI5Xd?YJSCo+m(b1LUSWJa`wZlwD#^6 z=in?w=Z1zra!T+4mbnLc|2alE3#k_3pw`;F2ML=4lrHrDVTcouK8>bo+XrXxwvd@& z%qBAEIDe2`$d8Ti)+|KdO^pcZ5+|N&b@!`hs37KkOC4EpUBA~86XC{H`OHYtk)U~@ z>c2gq?SON26y=_EUE?VB(Y;4h-B>#k0lJ@UjkQfj>5gpdrYR=spJOLc`8X%2&c@u@ zF$4n&hP8JVimtlYLL$s}v?ay(ekeRgX4$Cq*(2?rHX(qtHSt;}3jWk}3?O)$$!Zc? zGV0d4W7JEo@5*CHIQ3!SCVwAX+@!$jbOy*6NYrQJ_-1>}BbZ7F2i+EU+2XPzkig14 zmfA@lO5<{H#h6+)0{xiDKFn*bbs#BAT6CIhlCKx0y zYt@*lUKV>L>Hflyh`=T`=p2jrq9Tw@vTXs(r#5Lnh>o<#AB3T`HZNt4OK`a5fDm48p4Kz@`!7MfyfUS?5;t_z#Cy z+dRqt>1_Q7F-4tfR5J2&)iSWHNXkW1^9Y$wh)H6+Rl*35mXZ5ykrBCq&yE6e2*g~( zIE+IlU&Pt{8B&yx#X>n)mu?)L0#v<-aenfP*P}G=#n}>LiRkjD40?SeX7WBZAQ{7{ ztcWPuyUm^T7{GRIw<0$Ljhp5BCqAg`W!ZckRSU2N(;lPl@mg%;hJlB3Dic6FQ$i_e z-oEO8E#>0dd&Fbs>a3KT(^{Jk;jJ#6tKv{?=dcb7-SVe`OC)355(;z2ua=|^Kq5Q> zH7_C(94kNvO0*1%*Xu^eBgV09pxDEFU5ET+b9;$%vP7FIq;CIjtuJ`Q;fFri{Z2bd zvBP~+OG=BHtYD|X`zsr6##qC#LbLf=d5gqJ;%Q zc&q)YSAltoUvaVsdBrz1&1_2&cA}D;$}o|?gJ$7Y4Ex#pM{;HTB_4G&CDU3{T)Bcf zU%RapfF25@QiIKgKOKD91s=N$7S-HRq7@k?^s=7SHCIoy-)61|516705FZqtQOTMX3v_@gWWV0R*R|v}dZlhz zdH?2?=3@$s)s4dHuP+i75CP$9AK4^SkzMtboBS{_p4PXyU4KhT6}Y9$9X%*^M^%R7 zyJVi5LUr#~5@@0=9t!%8Vk(f%zt+9oFHF=n!ESpa{~snmISo2(8Vf6T-)MYh6SHKonMA$GiYTF7Of(%zH zO4kQJ*4Wq)d^o>JZ$s-h>Cd-cs%NJLQmrW5Ex6V0C5N>wH}(e%i1J$RT4N3+HC_{V znRNnPrX!4XAM(hD0Q~CTdwN$1D^7i&bp6(a2<)ET5kSAf{&YcVf(rfHsc8{q1KWek zgG=;u{zSB})h)_4-zwaT7kf7XIOYL;|J-OBzQn^*eF%@I6Ul_$29~kdYLRd3>UW}9 z{Z8hTS?l0S?1n~wGLw}=MODOC)cDLuPyneUkq7LP@>XaxI?EfgmKi`99ET(GVs~}X zVJjzPTjjIita+r3gFkSkzNts$7N1X0xs_jkvTp>+BCh4(w9VPs@TSHRPG887OTZun z?lUEQ(j`}&mcE~?kXl+yO{Y}#8pduWjit$t zk&zZBFnAXo&U_AbMIV@#f$igSu}R$JWEWNYNJ@|%Zp25f#3lqIP6=nD5MT($8u@YKulWq8;x7`78Os>dgqF!qGF=fGxnYi zPk>-03me^vO$wK;KGx^uWApzfZGnP!zK$dMp9H~z%80yauA*ucp;Fl_3B?PVG*tLW zyK&<+tL$KcXo-5qUm7ps=Z%3ALeS{p=5VCQF+7q)Gqi=i;czwC=<;gB<{#8zQ*{>6 z&W!hUvdea+?w|ZC&i#I^(pD`?dCf zz+#&WI`?IHc~AqIrtV9*jF#`QRA!lx-+Mv~D4b&=RL%SvZC@_%vSnBmGKXrQ9V=Ro=#;YMVDq zl(i$#tOtk5QOn^UX}pt1utgk62&p_5g-XX()6q;9jqJHeAk=(va224$bZZEJ-SM&` zooaK&V4x{3__V;De6P>^+T-8tf zNklqbk#l~Gt~C-eQ+Lm;BL}QQAzwwRp8muiGo#tlu@7|6`y6Ki6x*_#u@IEK4RA17 zAU9ZWil!0bc-Q7ekYE|2#Enw9can6b*Y)2iYMlS@TY9fB*)Q~9>5%f1L|a|li4nMR zz9ArQ{-=$?4Y_bfA!f#A$r4ifIRl+h+3(sXlO<~33bshHXN6LTLNNRhrZ6?35g4v3d_6nTWQ)X6vjEGPD zFRp~lRtFk{JEwxtkL~ILr3RnKjNA8LKWcB&9LPEmb2$tG$IUA&p|$I^x}+{xL|xvu zo^AFEL-k9MWMgQAc>kzY9x30{U+rtBD<(TST^p>;UXW1n)>+;93@ySWMlEbU`5lw# zxkfcmV&VR{3HgM$5$m6llyNEm?F1gf;DkJ>nPEeb*z=JKW0J6A|tRa1}_dAw!(1u`F%B)4UtM@?(ew{juNtI$$D$C>33OL4UQkb9C?axUheA_Hjrse6#&y=}8YI@y+O~AOE02&r0jCw^WNd42O zO~KT&L=->UiLsH_)##Ptp)kVU@$$<@s@_dsaFt)1);^9u0y9{=B!FHfbP1&>#%TAO zAMRkhL2=q&Cy6Hx2=Eb%f*1akkr@T*_ zmX~)@gQE2)j-}gF*&816q|dwg=zSa(%&Z?4GV6KfXRVLhSZt9G6J_qjbqND^v!;iN zLbWz`Yk4~Ljj9d24?~=2%V(l8c%4oCgXRU+IW2TuZ!x}xTdWfhm!QdO}>I{pkY~M|hQ=X%g zF8#V6U8vTYr1@q5s98nt$#JpD3<%G5PON>^Kax(&$+`N|&}wI$?}bR-Y{pEePvf2> zD4hYky|Z?F6!|XJBN+-MKx&N7!!)XAIlDO(v@+vw>sg)$YhcywA`MyiM+t}AZ-@B|KC4}l)j)m6sbqy=9`0WG? zYSx_g`B0K*tAb3&R1!BAw1X(k7WY}e$Uo+Y#%Eh-4eYb}{m4OE_#>T(21k|#SB?fI za%F_ahp_2he|J9sAwe!S$L-?zsq{&t?w9V&_wMx%3(x;Qjj z_Pv_Hm;igdVMvTrBUJ7E9IgF!V}XFi@yg9YpH+>A=?_;>Cy#UHOGENv^T z$~p%dg=XlQi%V+>hqr2gVZXTcwGjC+=-8AcO%)H#p7DjXd7pS5*^-D=N;xabbHfwZ zw|*bw+AIv53L0jR$dDk%gX0v|slH#CJgqs6$c8YxxAf-*w)gApJK#|qiC{7pU2pvn z5x&={jvBbJjGt#sNQ_Wc^wB{g8sYYSrSBrv( zZ^3i$;vtQk1o-mu8Cgu1U*wwB2r3Pzsm?K`<9%NWk8}71E?v66()HG+zxgelr#n-_ zXEE%d`^=yd=-vnpj!}{e*o&?MW-rH_jhE|dW92vDzRqyW(RyszV3o-QX+T;8q0_^X z0-Ivs0?{7=&8`kj-a}!|j=QzJDJQY-EN}xJD_HmOeZ3zaeoiR>gZbTgDsW?a7{446 z`9kJeg3`{oM8U9;rkjKW5PF>k8t;v+qZ6JG;Vsy>GVdTG4o?rG_=q(h5m0tXKxSRl z;b~AxG;$_Mlv_Q@s_d-UdJ1Hp=1T1o*dLd$U&(Uz(ag;Q38yo1g}4(C@&SSG&6nNWHW@`0}o`8T=2dF{VbVQm$jHOCkC@yDqX z?DeKPxB?FDUObgFSSJX?NHX_ddKJaXk}<3yfBCY@y1yt+Ko8FG)W$l_=5KFJ8DE!w&h2m=T z(2h3xc6OjU`sZ^>T5%y7`Y$I0nBFb{xER z?|{uDHxA!!fOG#LnFNKpyxZR%U{Z2zcWne6OPd#`uh@q2(@N;I`vK1q9iY4RpMy6u z=m)!u%EV2dd3&nV2~@y#?0I;&Hv?QQN6dBDSBJjUBYc};ctlyzj7~pai-hKb%~0i& zKGuGB24w*{Q?{cc6OeWU1?lYHd0jeplNvC~-WUq-_&xcHWEjt#6CN&v=P)X)Kh#c_ z$+)gelz5GY14Ilm1qrc0+YxTz#w_Y>=vGDZWm@_v*fz9T2zm0N3M9TVIDuh6t`xgd zC&g#1NLjz9JTznbKXrmCo$ofak3RPu1eG5#YWihgE{~R`_)RNPRnAM@BEX9cceqb^ zo}r5uFBBo7A{plaR>($EkQ6vy zAxhz~5vR7xWrHAbvd!p%F_L+ciF|k)ZFovV($=+zq|?d(!9{c0o-%RFn`O9Poao1V z;R7qx2{!&xx?Bewp1}?!1mbT~Eh5dpdqkbXSXdja8dH7C1PwEuC9)^pY6oRKO@RFk zQ}%tF{s)F8JL~)N&UOp~jGb?oqGzkglAR;2M2E(IH-uR@>&c|hdovPl>(u4aHFf*J zBwW?m+*ew=)df&|Sh#qEC_PM7l#gB2xlbma-6%=Q-oeGIGyk>O!8{)Q@Wmz?F*ogW z;r$t++Pc3Q(N>HOhuHzcb{di47mgyN-SE|x zYDtj;%L`mRQ)M2|e~*jxw;@e^XV)ZwtjGk>4l$aT+Ec(quY|~KS?FlE7PExu6WDU= zuqy_#Qu<$k1W{;;^wD5FP+r%b9Y+q&49uLxJirnIvODx|ft#tf+ajx@;&ttW|Bjn% z!7j&`g{f0oT6EAlew1h}BNlY}RUid3)`$%la()c*7aHKk`8V%J?9*FxmiggWW}CgA z-p_1BgiD1Xn3rLpx>W_VPqp1~=vYc?V@vQzY9RWoOmX8vyIEdFS))n7+{Nm~`s>+p zq5|V>z%!X|Jsa@GQNLDeVh`j6x268mGi+0!7{s0&@i3d=DV@tOU18yBam^7op$3q| zsA^bxCP?8vJ^({M1P;9Ug9lJ1oQ=x)cpA=nJoHMjkFg;}V1 z3DQj#^!@7)ymB!H1UG-b zpPgZu2_Sh8zd14kB`}ohI%3G?PaOEB9@tnUiJpz%OU5ilFh$!@@y=vDW*a4DJT0VB zTEh*uBPdAVg&4vic&!w1RBt{^Zu{os-`x(YUX4(?5>ZeQnr!m9n?n$1m0#Y%=Bfi1qXt-#4U& zRb!0k$J%5SA!#AlVnr-3%$CPbaKl8y4P5dNSOc zy4k5p0U+krQ5lj8^(-y~w)C4~@awPwAl#^N*0CErc@k|0&sbPJ{uvHVNS)bpoQN_t zgQqW(c?Z$xfE98d*vCw3S8_m5N4#>3epZR!>Ve8DbFXc zzTcB}6^H0l>@0f9#mC_+D6qjzx+*CTQLN0@{q5h90we)s2%GoiUT6AXiWdX6@)IPu zbLR1eeEVr4mS~yqrkc%^b0)?C3(+pD^PHPNbMf7ThnZFi--b^T61_r~TI0Zkz6f%X zqdTob=lAQnF&TQ|EX(za6MmMwg|x(Asom-bADTT8@nmO0Z#crZ{0|9rnLX(e+m(1g zG0(msZCN@+6od9u1k}vPbV`ocO(EC8zVmu`J!(NK=k;vwjrc1k$fLX+EGZ0$|1k6fiqFHpk zEl$^bP_yVMRSReX^UtZ3b&jAJ@LC7mzC&k$ldkf#G2S5iLpHk9q`6RL$xUeTWp1l(ey2Ces71dmf_sZM`I#Jur6)T^YG(VF*3g@K34tuXcuOO zRrb^@lcOY6nI(7$Xf|DO&kZQLB z>0nPZtrJJ=U?J`~!Rl6*3JVoiKfoGO2%AR29Q)*=nv$#fi;{6dE;!r4J9($%-||$u znNC51eW7qA#16wSnNLyJ?FfbLkmoTzp1VL(qTl(v6xexFD(IB=jh+P{#mnmR{ZdLH zz+KR0tJo+YT~)H|EHli=44%jC7A!?0jO0?D=5>HlH+E`^ock=EAJI@BG?8+YC0PI z0Ch66>+?5BH$N|Qgm>R9Ee`De_f&EY69f@}6NwVluJbRCvWc0{;~d*2Ul3mfgB%ky z-r}m>?c@*aj1Rl+EcPw2PsXvB&&qTNI^6Tj`fYp^f+5=21vgg`y9@h#NIBE}wnKee zORH<5N<2)-L-IoXFClWsXkHxCliSn92&67l_61z1s`8Km$Yco8x)CM!R@t*`ujBlF zZrl@W%8`}&#dYS757KRTw`gm*B@akM5Fb_}1wT2UCzyuc2i%Q_G--npy>)Tn8%F2- zKb4P#{9*^M8;flgevCVZ4Ya0rW8bP&t&>2g1Vl!iOF$Yhcg*Bnug@6Kr(FU$!xD^c`-&>|3`pUL1@L( zoDMtr<45myUtb3*(N|?156!{oLn0l?8c@XEq6NW>LI7SMju!<(mRM9#nW0oXh?hIJ zCuwe%5(;9*BEYt4j=^;rZoHeFh5R#wtLV$VU5VHHF%SBA=5LE%*R^?v2!9t)yD@qkC)Y_kb{@n)GayK)QoL?@Vo; zR9jk^E-LgKzXZ z6r@*}@jn>V%_`5YSDST^pxiKRx|gnp%SxZWce6PFP5&;atiYr|y~Vj6roJYVrYp7~ z>IVI~j@GN$DC4@;ur^QHyA8}!G={g7^8x6(dPW_7@g{40bcF1$2@uagnHyFtq4=56 z7E{|=kbq@2L(2IP)IjY2^!IU%#QgBZ+o*v&DWYV6Q~B?^!F_o~(!+3)h+6VmxWEyE G4Ov=Ml`%>H diff --git a/tests/files/good-single-lzma-empty.lzma b/tests/files/good-single-lzma-empty.lzma deleted file mode 100644 index b445780447132d8b016f99edbbe5311546efc69d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21 ccmezG6Xom3z|i}i>5?J?gF~!58$)Ci08d8-8UO$Q diff --git a/tests/files/good-single-lzma-flush_1.lzma b/tests/files/good-single-lzma-flush_1.lzma deleted file mode 100644 index 7de086faa4d7cee5635e4f7c7371c3e5a2b7a43e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48 zcmezG6Xom3z|i}i>5?J?k3+0HgNmf*jC@w7b^rhWf55@Oz@Q^LKW(j-K5t8dB12>p E0F$;5EdT%j diff --git a/tests/files/good-single-lzma-flush_2.lzma b/tests/files/good-single-lzma-flush_2.lzma deleted file mode 100644 index c444a2179a3a40e1121758bb74e088ea8f297926..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63 zcmV-F0KotMOj=Dr0052e0@4WpKwTdIBpFGVZv{fG|NsBs2mk;8EFGU_tx7bLuRs6% Vt6@L@gkRVn|EC_oVGJ+;Sz7h17@z5>8i12aQp6aYND1zG?A diff --git a/tests/files/good-single-none-empty_2.lzma b/tests/files/good-single-none-empty_2.lzma deleted file mode 100644 index 612bdac26f26cc0c5f82eb0c06860e63bb57684e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26 gcmezG6Xom3z^NI2Z?ysg!>9Q?#~FZtn=>*B0Cs-|kpKVy diff --git a/tests/files/good-single-none-empty_3.lzma b/tests/files/good-single-none-empty_3.lzma deleted file mode 100644 index 7e1d19b66e803a5bcaff3ab3f47b3c457258addd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19 acmezG6Xom3z|i}i>5>5h0|N^~WE22F5>8ik4I`wPQF5Teo;;e7e7N}6axbQwN?q3 diff --git a/tests/files/good-single-none.lzma b/tests/files/good-single-none.lzma deleted file mode 100644 index a6b24ae83c1fc4148188376afb446cb607626ae6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30 lcmezG6Xom3z|i}i>5>8ik4I`wPQF5Teo;;e7e7N}6ab=AEa{vFtk^le@ I3?l$pTGp%+#sB~S diff --git a/tests/files/good-single-subblock_implicit.lzma b/tests/files/good-single-subblock_implicit.lzma deleted file mode 100644 index b33c6028a62ed52502f3099383491c30f9c867be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35 qcmezG6Xom3z|i}i=@JKnHUk5LM`}(^zCw6@QBDe%0FMYmWE23x{0YDS diff --git a/tests/files/good-single-subblock_rle.lzma b/tests/files/good-single-subblock_rle.lzma deleted file mode 100644 index 32d49cc0e247f0ca718573f90cdd5a50f368e05d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 118 zcmezG6Xom3z-ush^BPVj#?B23w=yuOGcYiu<>zxH7N>9lnWjL@3}S=ClM;(C#DT&F yAT>#eRa^qsdMs}SU2i)7XIaAGp!%3A=3h8AA33ciy+zW*j_L38Z!YP)kx>9H!XtYC diff --git a/tests/files/malicious-multi-metadata-64PiB.lzma b/tests/files/malicious-multi-metadata-64PiB.lzma deleted file mode 100644 index a29f877651cf87d66aad3f40451c303ec11d2cc1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51 zcmezG6Xom3An@}4*7FJs3|$P2jOq*w49Y;evSyXUXNz4=}!futg2C+GP|Bf!^ z54-zx@7I{rD-S1jY$}4`bb>IA2e~nk#zP5aIvxs8e#j91H?L>l;%GN|ac^b{i@$>ktwSRsG zX_uUpi&k*@c{kYhhh_?=b;D_`ocmj?KOEVs7r%<%&O6O%f-v8IfyqA#D}M+1B`nGKH2m2m$@*pt(*|NlRAHh`glq48iRLu3>Hz(BXU<=PL%1Ke=VsKu(NR)w* VgLj_uyJ-`gX7VyJ00CoU6aW|N51Ie~ literal 0 HcmV?d00001 diff --git a/tests/files/unsupported-check.lzma b/tests/files/unsupported-check.lzma new file mode 100644 index 0000000000000000000000000000000000000000..6d8a295e405d67cecdbc2e281b2f8543c8d4dd74 GIT binary patch literal 68 zcmezG6Xom3z`%65`1*P#21P~=1_p*3{K~?N46GihIXU@UObl${`9(P?id+oNix-J9 XFmmwDbAC5XOL)Q*Mxa^-rpPD&T@?_y literal 0 HcmV?d00001 diff --git a/tests/files/unsupported-filter_flags-1.lzma b/tests/files/unsupported-filter_flags-1.lzma new file mode 100644 index 0000000000000000000000000000000000000000..c70571f99f977f3adcb590c00cf326ab5140cfc2 GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^5p`HN@(u?mhGO&82=H%paF)^@(=NILqC~`44FJ2_dz{tTn S&-vZ72~IP489|B}BclLC=n#+q literal 0 HcmV?d00001 diff --git a/tests/files/unsupported-filter_flags-2.lzma b/tests/files/unsupported-filter_flags-2.lzma new file mode 100644 index 0000000000000000000000000000000000000000..1fd68f9e07d0e6b3356ede503296cf0412953f85 GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^5ftm3?0|UeF)^ARX46GihIXU@UObl${`9(P?id+oNix-J9 XFmmwDbAC5%g40Z1Mxa^-#>glDke(71 literal 0 HcmV?d00001 diff --git a/tests/files/unsupported-filter_flags-3.lzma b/tests/files/unsupported-filter_flags-3.lzma new file mode 100644 index 0000000000000000000000000000000000000000..dcaf21f8073e729254430d0d3955e151b481d3eb GIT binary patch literal 68 zcmezG6Xom3z`&TPbkB^5QIU}Yh)ztDp3TU>>XDk0lh4J(z!sigl#`;!#o)YnkthQr V2k$)Rche>~&E#cd00PFyC;(155WWBa literal 0 HcmV?d00001 diff --git a/tests/test_block_header.c b/tests/test_block_header.c index a8ce09b6..28929dea 100644 --- a/tests/test_block_header.c +++ b/tests/test_block_header.c @@ -21,19 +21,19 @@ static uint8_t buf[LZMA_BLOCK_HEADER_SIZE_MAX]; -static lzma_options_block known_options; -static lzma_options_block decoded_options; +static lzma_block known_options; +static lzma_block decoded_options; -static lzma_options_filter filters_none[1] = { +static lzma_filter filters_none[1] = { { .id = LZMA_VLI_VALUE_UNKNOWN, }, }; -static lzma_options_filter filters_one[2] = { +static lzma_filter filters_one[2] = { { - .id = LZMA_FILTER_LZMA, + .id = LZMA_FILTER_LZMA2, .options = (void *)(&lzma_preset_lzma[0]), }, { .id = LZMA_VLI_VALUE_UNKNOWN, @@ -41,7 +41,7 @@ static lzma_options_filter filters_one[2] = { }; -static lzma_options_filter filters_four[5] = { +static lzma_filter filters_four[5] = { { .id = LZMA_FILTER_X86, .options = NULL, @@ -52,7 +52,7 @@ static lzma_options_filter filters_four[5] = { .id = LZMA_FILTER_X86, .options = NULL, }, { - .id = LZMA_FILTER_LZMA, + .id = LZMA_FILTER_LZMA2, .options = (void *)(&lzma_preset_lzma[0]), }, { .id = LZMA_VLI_VALUE_UNKNOWN, @@ -60,7 +60,7 @@ static lzma_options_filter filters_four[5] = { }; -static lzma_options_filter filters_five[6] = { +static lzma_filter filters_five[6] = { { .id = LZMA_FILTER_X86, .options = NULL, @@ -74,7 +74,7 @@ static lzma_options_filter filters_five[6] = { .id = LZMA_FILTER_X86, .options = NULL, }, { - .id = LZMA_FILTER_LZMA, + .id = LZMA_FILTER_LZMA2, .options = (void *)(&lzma_preset_lzma[0]), }, { .id = LZMA_VLI_VALUE_UNKNOWN, @@ -87,7 +87,7 @@ code(void) { expect(lzma_block_header_encode(&known_options, buf) == LZMA_OK); - lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; + lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; memcrap(filters, sizeof(filters)); memcrap(&decoded_options, sizeof(decoded_options)); @@ -114,7 +114,7 @@ code(void) static void test1(void) { - known_options = (lzma_options_block){ + known_options = (lzma_block){ .check = LZMA_CHECK_NONE, .compressed_size = LZMA_VLI_VALUE_UNKNOWN, .uncompressed_size = LZMA_VLI_VALUE_UNKNOWN, @@ -153,7 +153,7 @@ test1(void) static void test2(void) { - known_options = (lzma_options_block){ + known_options = (lzma_block){ .check = LZMA_CHECK_CRC32, .compressed_size = LZMA_VLI_VALUE_UNKNOWN, .uncompressed_size = LZMA_VLI_VALUE_UNKNOWN, @@ -179,7 +179,7 @@ test2(void) static void test3(void) { - known_options = (lzma_options_block){ + known_options = (lzma_block){ .check = LZMA_CHECK_CRC32, .compressed_size = LZMA_VLI_VALUE_UNKNOWN, .uncompressed_size = LZMA_VLI_VALUE_UNKNOWN, @@ -190,7 +190,7 @@ test3(void) known_options.header_size += 4; expect(lzma_block_header_encode(&known_options, buf) == LZMA_OK); - lzma_options_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; + lzma_filter filters[LZMA_BLOCK_FILTERS_MAX + 1]; decoded_options.header_size = known_options.header_size; decoded_options.check = known_options.check; decoded_options.filters = filters; diff --git a/tests/test_compress.sh b/tests/test_compress.sh index e322d385..49f3fd50 100755 --- a/tests/test_compress.sh +++ b/tests/test_compress.sh @@ -121,8 +121,8 @@ do --armthumb \ --sparc do - test_lzma $ARGS --lzma=dict=64KiB,fb=32,mode=fast - test_lzma --subblock $ARGS --lzma=dict=64KiB,fb=32,mode=fast + test_lzma $ARGS --lzma2=dict=64KiB,fb=32,mode=fast + test_lzma --subblock $ARGS --lzma2=dict=64KiB,fb=32,mode=fast done echo diff --git a/tests/test_filter_flags.c b/tests/test_filter_flags.c index dde01381..d4d309f4 100644 --- a/tests/test_filter_flags.c +++ b/tests/test_filter_flags.c @@ -21,8 +21,8 @@ static uint8_t buffer[4096]; -static lzma_options_filter known_flags; -static lzma_options_filter decoded_flags; +static lzma_filter known_flags; +static lzma_filter decoded_flags; static lzma_stream strm = LZMA_STREAM_INIT; @@ -39,8 +39,8 @@ encode(uint32_t known_size) return true; size_t out_pos = 0; - if (lzma_filter_flags_encode(buffer, &out_pos, known_size, - &known_flags) != LZMA_OK) + if (lzma_filter_flags_encode(&known_flags, + buffer, &out_pos, known_size) != LZMA_OK) return true; if (out_pos != known_size) @@ -78,32 +78,18 @@ decode(uint32_t known_size) } -#ifdef HAVE_FILTER_SUBBLOCK +#if defined(HAVE_ENCODER_SUBBLOCK) && defined(HAVE_DECODER_SUBBLOCK) static void test_subblock(void) { // Test 1 known_flags.id = LZMA_FILTER_SUBBLOCK; known_flags.options = NULL; - expect(!encode(2)); expect(!decode(2)); - expect(decoded_flags.options != NULL); - expect(((lzma_options_subblock *)(decoded_flags.options)) - ->allow_subfilters); + expect(decoded_flags.options == NULL); // Test 2 - known_flags.options = decoded_flags.options; - expect(!encode(2)); - expect(!decode(2)); - expect(decoded_flags.options != NULL); - expect(((lzma_options_subblock *)(decoded_flags.options)) - ->allow_subfilters); - - free(decoded_flags.options); - free(known_flags.options); - - // Test 3 buffer[0] = LZMA_FILTER_SUBBLOCK; buffer[1] = 1; buffer[2] = 0; @@ -112,7 +98,7 @@ test_subblock(void) #endif -#ifdef HAVE_FILTER_SIMPLE +#if defined(HAVE_ENCODER_X86) && defined(HAVE_DECODER_X86) static void test_simple(void) { @@ -147,7 +133,7 @@ test_simple(void) #endif -#ifdef HAVE_FILTER_DELTA +#if defined(HAVE_ENCODER_DELTA) && defined(HAVE_DECODER_DELTA) static void test_delta(void) { @@ -157,7 +143,10 @@ test_delta(void) expect(encode(99)); // Test 2 - lzma_options_delta options = { 0 }; + lzma_options_delta options = { + .type = LZMA_DELTA_TYPE_BYTE, + .distance = 0 + }; known_flags.options = &options; expect(encode(99)); @@ -185,7 +174,7 @@ test_delta(void) } #endif - +/* #ifdef HAVE_FILTER_LZMA static void validate_lzma(void) @@ -275,25 +264,25 @@ test_lzma(void) } } #endif - +*/ int main(void) { lzma_init(); -#ifdef HAVE_FILTER_SUBBLOCK +#if defined(HAVE_ENCODER_SUBBLOCK) && defined(HAVE_DECODER_SUBBLOCK) test_subblock(); #endif -#ifdef HAVE_FILTER_SIMPLE +#if defined(HAVE_ENCODER_X86) && defined(HAVE_DECODER_X86) test_simple(); #endif -#ifdef HAVE_FILTER_DELTA +#if defined(HAVE_ENCODER_DELTA) && defined(HAVE_DECODER_DELTA) test_delta(); #endif -#ifdef HAVE_FILTER_LZMA - test_lzma(); -#endif +// #ifdef HAVE_FILTER_LZMA +// test_lzma(); +// #endif lzma_end(&strm); diff --git a/tests/test_stream_flags.c b/tests/test_stream_flags.c index 0de87cd1..ead75501 100644 --- a/tests/test_stream_flags.c +++ b/tests/test_stream_flags.c @@ -95,7 +95,7 @@ test_encode_invalid(void) expect(lzma_stream_footer_encode(&known_flags, buffer) == LZMA_PROG_ERROR); - known_flags.check = (lzma_check_type)(-1); + known_flags.check = (lzma_check)(-1); expect(lzma_stream_header_encode(&known_flags, buffer) == LZMA_PROG_ERROR); @@ -171,7 +171,7 @@ main(void) // Valid headers known_flags.backward_size = 1024; - for (lzma_check_type check = LZMA_CHECK_NONE; + for (lzma_check check = LZMA_CHECK_NONE; check <= LZMA_CHECK_ID_MAX; ++check) { test_header(); test_footer(); diff --git a/tests/tests.h b/tests/tests.h index 89880552..3c59bb63 100644 --- a/tests/tests.h +++ b/tests/tests.h @@ -69,10 +69,18 @@ lzma_ret_sym(lzma_ret ret) str = "LZMA_HEADER_ERROR"; break; + case LZMA_NO_CHECK: + str = "LZMA_NO_CHECK"; + break; + case LZMA_UNSUPPORTED_CHECK: str = "LZMA_UNSUPPORTED_CHECK"; break; + case LZMA_SEE_CHECK: + str = "LZMA_SEE_CHECK"; + break; + case LZMA_FORMAT_ERROR: str = "LZMA_FORMAT_ERROR"; break;