1
0
Fork 0
mirror of https://git.tukaani.org/xz.git synced 2024-04-04 12:36:23 +02:00

xz: Add --memlimit-mt-decompress along with a default limit value.

--memlimit-mt-decompress allows specifying the limit for
multithreaded decompression. This matches memlimit_threading in
liblzma. This limit can only affect the number of threads being
used; it will never prevent xz from decompressing a file. The
old --memlimit-decompress option is still used at the same time.

If the value of --memlimit-decompress (the default value or
one specified by the user) is less than the value of
--memlimit-mt-decompress , then --memlimit-mt-decompress is
reduced to match --memlimit-decompress.

Man page wasn't updated yet.
This commit is contained in:
Lasse Collin 2022-04-11 22:20:49 +03:00
parent fe87b4cd53
commit cad299008c
5 changed files with 97 additions and 42 deletions

View file

@ -29,10 +29,11 @@ bool opt_ignore_check = false;
const char stdin_filename[] = "(stdin)";
/// Parse and set the memory usage limit for compression and/or decompression.
/// Parse and set the memory usage limit for compression, decompression,
/// and/or multithreaded decompression.
static void
parse_memlimit(const char *name, const char *name_percentage, char *str,
bool set_compress, bool set_decompress)
bool set_compress, bool set_decompress, bool set_mtdec)
{
bool is_percentage = false;
uint64_t value;
@ -49,8 +50,8 @@ parse_memlimit(const char *name, const char *name_percentage, char *str,
value = str_to_uint64(name, str, 0, UINT64_MAX);
}
hardware_memlimit_set(
value, set_compress, set_decompress, is_percentage);
hardware_memlimit_set(value, set_compress, set_decompress, set_mtdec,
is_percentage);
return;
}
@ -138,6 +139,7 @@ parse_real(args_info *args, int argc, char **argv)
OPT_BLOCK_LIST,
OPT_MEM_COMPRESS,
OPT_MEM_DECOMPRESS,
OPT_MEM_MT_DECOMPRESS,
OPT_NO_ADJUST,
OPT_INFO_MEMORY,
OPT_ROBOT,
@ -176,6 +178,7 @@ parse_real(args_info *args, int argc, char **argv)
{ "block-list", required_argument, NULL, OPT_BLOCK_LIST },
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
{ "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS },
{ "memlimit", required_argument, NULL, 'M' },
{ "memory", required_argument, NULL, 'M' }, // Old alias
{ "no-adjust", no_argument, NULL, OPT_NO_ADJUST },
@ -225,20 +228,27 @@ parse_real(args_info *args, int argc, char **argv)
case OPT_MEM_COMPRESS:
parse_memlimit("memlimit-compress",
"memlimit-compress%", optarg,
true, false);
true, false, false);
break;
// --memlimit-decompress
case OPT_MEM_DECOMPRESS:
parse_memlimit("memlimit-decompress",
"memlimit-decompress%", optarg,
false, true);
false, true, false);
break;
// --memlimit-mt-decompress
case OPT_MEM_MT_DECOMPRESS:
parse_memlimit("memlimit-mt-decompress",
"memlimit-mt-decompress%", optarg,
false, false, true);
break;
// --memlimit
case 'M':
parse_memlimit("memlimit", "memlimit%", optarg,
true, true);
true, true, true);
break;
// --suffix

View file

@ -524,32 +524,20 @@ coder_init(file_pair *pair)
mt_options.flags = flags;
mt_options.threads = hardware_threads_get();
// TODO: Support --memlimit-threading=LIMIT.
mt_options.memlimit_stop
= hardware_memlimit_get(MODE_DECOMPRESS);
// If single-threaded mode was requested, set the
// memlimit for threading to zero. This forces the
// decoder to use single-threaded mode which matches
// the behavior of lzma_stream_decoder().
//
// Otherwise use the limit for threaded decompression
// which has a sane default (users are still free to
// make it insanely high though).
mt_options.memlimit_threading
= mt_options.memlimit_stop;
if (mt_options.threads == 1) {
// Single-threaded mode was requested. Force
// the decoder to use minimal memory, matching
// the behavior of lzma_stream_decoder().
mt_options.memlimit_threading = 0;
} else if (mt_options.memlimit_threading
== UINT64_MAX) {
// TODO: Support --memlimit-threading=LIMIT.
//
// If lzma_physmem() fails, it returns 0 and
// we end up with a single thread.
//
// NOTE: It is assential that we never end up
// with an effectively infinite value in
// memlimit_threading!
mt_options.memlimit_threading
= lzma_physmem() / 4;
}
= mt_options.threads == 1
? 0 : hardware_memlimit_mtdec_get();
ret = lzma_stream_decoder_mt(&strm, &mt_options);
# else

View file

@ -18,10 +18,26 @@
static uint32_t threads_max = 1;
/// Memory usage limit for compression
static uint64_t memlimit_compress;
static uint64_t memlimit_compress = 0;
/// Memory usage limit for decompression
static uint64_t memlimit_decompress;
static uint64_t memlimit_decompress = 0;
/// Default memory usage for multithreaded modes:
///
/// - Default value for --memlimit-mt-decompress
///
/// This value is caluclated in hardware_init() and cannot be changed later.
static uint64_t memlimit_mt_default;
/// Memory usage limit for multithreaded decompression. This is a soft limit:
/// if reducing the number of threads to one isn't enough to keep memory
/// usage below this limit, then one thread is used and this limit is ignored.
/// memlimit_decompress is still obeyed.
///
/// This can be set with --memlimit-mt-decompress. The default value for
/// this is memlimit_mt_default.
static uint64_t memlimit_mtdec;
/// Total amount of physical RAM
static uint64_t total_ram;
@ -60,7 +76,8 @@ hardware_threads_get(void)
extern void
hardware_memlimit_set(uint64_t new_memlimit,
bool set_compress, bool set_decompress, bool is_percentage)
bool set_compress, bool set_decompress, bool set_mtdec,
bool is_percentage)
{
if (is_percentage) {
assert(new_memlimit > 0);
@ -110,6 +127,9 @@ hardware_memlimit_set(uint64_t new_memlimit,
if (set_decompress)
memlimit_decompress = new_memlimit;
if (set_mtdec)
memlimit_mtdec = new_memlimit;
return;
}
@ -132,6 +152,23 @@ hardware_memlimit_get(enum operation_mode mode)
}
extern uint64_t
hardware_memlimit_mtdec_get(void)
{
uint64_t m = memlimit_mtdec != 0
? memlimit_mtdec
: memlimit_mt_default;
// Cap the value to memlimit_decompress if it has been specified.
// This is nice for --info-memory. It wouldn't be needed for liblzma
// since it does this anyway.
if (memlimit_decompress != 0 && m > memlimit_decompress)
m = memlimit_decompress;
return m;
}
/// Helper for hardware_memlimit_show() to print one human-readable info line.
static void
memlimit_show(const char *str, size_t str_columns, uint64_t value)
@ -203,7 +240,20 @@ hardware_init(void)
if (total_ram == 0)
total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
// Set the defaults.
hardware_memlimit_set(0, true, true, false);
// FIXME? There may be better methods to determine the default value.
// One Linux-specific suggestion is to use MemAvailable from
// /proc/meminfo as the starting point.
memlimit_mt_default = total_ram / 4;
// A too high value may cause 32-bit xz to run out of address space.
// Use a conservative maximum value here. A few typical address space
// sizes with Linux:
// - x86-64 with 32-bit xz: 4 GiB
// - x86: 3 GiB
// - MIPS32: 2 GiB
const size_t mem_ceiling = SIZE_MAX / 3; // About 1365 GiB on 32-bit
if (memlimit_mt_default > mem_ceiling)
memlimit_mt_default = mem_ceiling;
return;
}

View file

@ -22,16 +22,21 @@ extern void hardware_threads_set(uint32_t threadlimit);
extern uint32_t hardware_threads_get(void);
/// Set the memory usage limit. There are separate limits for compression
/// and decompression (the latter includes also --list), one or both can
/// be set with a single call to this function. Zero indicates resetting
/// the limit back to the defaults. The limit can also be set as a percentage
/// of installed RAM; the percentage must be in the range [1, 100].
/// Set the memory usage limit. There are separate limits for compression,
/// decompression (also includes --list), and multithreaded decompression.
/// Any combination of these can be set with a single call to this function.
/// Zero indicates resetting the limit back to the defaults.
/// The limit can also be set as a percentage of installed RAM; the
/// percentage must be in the range [1, 100].
extern void hardware_memlimit_set(uint64_t new_memlimit,
bool set_compress, bool set_decompress, bool is_percentage);
bool set_compress, bool set_decompress, bool set_mtdec,
bool is_percentage);
/// Get the current memory usage limit for compression or decompression.
extern uint64_t hardware_memlimit_get(enum operation_mode mode);
/// Get the current memory usage limit for multithreaded decompression.
extern uint64_t hardware_memlimit_mtdec_get(void);
/// Display the amount of RAM and memory usage limits and exit.
extern void hardware_memlimit_show(void) lzma_attribute((__noreturn__));

View file

@ -1180,9 +1180,11 @@ message_help(bool long_help)
puts(_( // xgettext:no-c-format
" --memlimit-compress=LIMIT\n"
" --memlimit-decompress=LIMIT\n"
" --memlimit-mt-decompress=LIMIT\n"
" -M, --memlimit=LIMIT\n"
" set memory usage limit for compression, decompression,\n"
" or both; LIMIT is in bytes, % of RAM, or 0 for defaults"));
" threaded decompression, or all of these; LIMIT is in\n"
" bytes, % of RAM, or 0 for defaults"));
puts(_(
" --no-adjust if compression settings exceed the memory usage limit,\n"