mirror of
https://git.tukaani.org/xz.git
synced 2024-04-04 12:36:23 +02:00
xz: Add --memlimit-mt-decompress along with a default limit value.
--memlimit-mt-decompress allows specifying the limit for multithreaded decompression. This matches memlimit_threading in liblzma. This limit can only affect the number of threads being used; it will never prevent xz from decompressing a file. The old --memlimit-decompress option is still used at the same time. If the value of --memlimit-decompress (the default value or one specified by the user) is less than the value of --memlimit-mt-decompress , then --memlimit-mt-decompress is reduced to match --memlimit-decompress. Man page wasn't updated yet.
This commit is contained in:
parent
fe87b4cd53
commit
cad299008c
5 changed files with 97 additions and 42 deletions
|
@ -29,10 +29,11 @@ bool opt_ignore_check = false;
|
||||||
const char stdin_filename[] = "(stdin)";
|
const char stdin_filename[] = "(stdin)";
|
||||||
|
|
||||||
|
|
||||||
/// Parse and set the memory usage limit for compression and/or decompression.
|
/// Parse and set the memory usage limit for compression, decompression,
|
||||||
|
/// and/or multithreaded decompression.
|
||||||
static void
|
static void
|
||||||
parse_memlimit(const char *name, const char *name_percentage, char *str,
|
parse_memlimit(const char *name, const char *name_percentage, char *str,
|
||||||
bool set_compress, bool set_decompress)
|
bool set_compress, bool set_decompress, bool set_mtdec)
|
||||||
{
|
{
|
||||||
bool is_percentage = false;
|
bool is_percentage = false;
|
||||||
uint64_t value;
|
uint64_t value;
|
||||||
|
@ -49,8 +50,8 @@ parse_memlimit(const char *name, const char *name_percentage, char *str,
|
||||||
value = str_to_uint64(name, str, 0, UINT64_MAX);
|
value = str_to_uint64(name, str, 0, UINT64_MAX);
|
||||||
}
|
}
|
||||||
|
|
||||||
hardware_memlimit_set(
|
hardware_memlimit_set(value, set_compress, set_decompress, set_mtdec,
|
||||||
value, set_compress, set_decompress, is_percentage);
|
is_percentage);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -138,6 +139,7 @@ parse_real(args_info *args, int argc, char **argv)
|
||||||
OPT_BLOCK_LIST,
|
OPT_BLOCK_LIST,
|
||||||
OPT_MEM_COMPRESS,
|
OPT_MEM_COMPRESS,
|
||||||
OPT_MEM_DECOMPRESS,
|
OPT_MEM_DECOMPRESS,
|
||||||
|
OPT_MEM_MT_DECOMPRESS,
|
||||||
OPT_NO_ADJUST,
|
OPT_NO_ADJUST,
|
||||||
OPT_INFO_MEMORY,
|
OPT_INFO_MEMORY,
|
||||||
OPT_ROBOT,
|
OPT_ROBOT,
|
||||||
|
@ -176,6 +178,7 @@ parse_real(args_info *args, int argc, char **argv)
|
||||||
{ "block-list", required_argument, NULL, OPT_BLOCK_LIST },
|
{ "block-list", required_argument, NULL, OPT_BLOCK_LIST },
|
||||||
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
|
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
|
||||||
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
|
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
|
||||||
|
{ "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS },
|
||||||
{ "memlimit", required_argument, NULL, 'M' },
|
{ "memlimit", required_argument, NULL, 'M' },
|
||||||
{ "memory", required_argument, NULL, 'M' }, // Old alias
|
{ "memory", required_argument, NULL, 'M' }, // Old alias
|
||||||
{ "no-adjust", no_argument, NULL, OPT_NO_ADJUST },
|
{ "no-adjust", no_argument, NULL, OPT_NO_ADJUST },
|
||||||
|
@ -225,20 +228,27 @@ parse_real(args_info *args, int argc, char **argv)
|
||||||
case OPT_MEM_COMPRESS:
|
case OPT_MEM_COMPRESS:
|
||||||
parse_memlimit("memlimit-compress",
|
parse_memlimit("memlimit-compress",
|
||||||
"memlimit-compress%", optarg,
|
"memlimit-compress%", optarg,
|
||||||
true, false);
|
true, false, false);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
// --memlimit-decompress
|
// --memlimit-decompress
|
||||||
case OPT_MEM_DECOMPRESS:
|
case OPT_MEM_DECOMPRESS:
|
||||||
parse_memlimit("memlimit-decompress",
|
parse_memlimit("memlimit-decompress",
|
||||||
"memlimit-decompress%", optarg,
|
"memlimit-decompress%", optarg,
|
||||||
false, true);
|
false, true, false);
|
||||||
|
break;
|
||||||
|
|
||||||
|
// --memlimit-mt-decompress
|
||||||
|
case OPT_MEM_MT_DECOMPRESS:
|
||||||
|
parse_memlimit("memlimit-mt-decompress",
|
||||||
|
"memlimit-mt-decompress%", optarg,
|
||||||
|
false, false, true);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
// --memlimit
|
// --memlimit
|
||||||
case 'M':
|
case 'M':
|
||||||
parse_memlimit("memlimit", "memlimit%", optarg,
|
parse_memlimit("memlimit", "memlimit%", optarg,
|
||||||
true, true);
|
true, true, true);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
// --suffix
|
// --suffix
|
||||||
|
|
|
@ -524,32 +524,20 @@ coder_init(file_pair *pair)
|
||||||
mt_options.flags = flags;
|
mt_options.flags = flags;
|
||||||
|
|
||||||
mt_options.threads = hardware_threads_get();
|
mt_options.threads = hardware_threads_get();
|
||||||
|
|
||||||
// TODO: Support --memlimit-threading=LIMIT.
|
|
||||||
mt_options.memlimit_stop
|
mt_options.memlimit_stop
|
||||||
= hardware_memlimit_get(MODE_DECOMPRESS);
|
= hardware_memlimit_get(MODE_DECOMPRESS);
|
||||||
|
|
||||||
|
// If single-threaded mode was requested, set the
|
||||||
|
// memlimit for threading to zero. This forces the
|
||||||
|
// decoder to use single-threaded mode which matches
|
||||||
|
// the behavior of lzma_stream_decoder().
|
||||||
|
//
|
||||||
|
// Otherwise use the limit for threaded decompression
|
||||||
|
// which has a sane default (users are still free to
|
||||||
|
// make it insanely high though).
|
||||||
mt_options.memlimit_threading
|
mt_options.memlimit_threading
|
||||||
= mt_options.memlimit_stop;
|
= mt_options.threads == 1
|
||||||
|
? 0 : hardware_memlimit_mtdec_get();
|
||||||
if (mt_options.threads == 1) {
|
|
||||||
// Single-threaded mode was requested. Force
|
|
||||||
// the decoder to use minimal memory, matching
|
|
||||||
// the behavior of lzma_stream_decoder().
|
|
||||||
mt_options.memlimit_threading = 0;
|
|
||||||
|
|
||||||
} else if (mt_options.memlimit_threading
|
|
||||||
== UINT64_MAX) {
|
|
||||||
// TODO: Support --memlimit-threading=LIMIT.
|
|
||||||
//
|
|
||||||
// If lzma_physmem() fails, it returns 0 and
|
|
||||||
// we end up with a single thread.
|
|
||||||
//
|
|
||||||
// NOTE: It is assential that we never end up
|
|
||||||
// with an effectively infinite value in
|
|
||||||
// memlimit_threading!
|
|
||||||
mt_options.memlimit_threading
|
|
||||||
= lzma_physmem() / 4;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = lzma_stream_decoder_mt(&strm, &mt_options);
|
ret = lzma_stream_decoder_mt(&strm, &mt_options);
|
||||||
# else
|
# else
|
||||||
|
|
|
@ -18,10 +18,26 @@
|
||||||
static uint32_t threads_max = 1;
|
static uint32_t threads_max = 1;
|
||||||
|
|
||||||
/// Memory usage limit for compression
|
/// Memory usage limit for compression
|
||||||
static uint64_t memlimit_compress;
|
static uint64_t memlimit_compress = 0;
|
||||||
|
|
||||||
/// Memory usage limit for decompression
|
/// Memory usage limit for decompression
|
||||||
static uint64_t memlimit_decompress;
|
static uint64_t memlimit_decompress = 0;
|
||||||
|
|
||||||
|
/// Default memory usage for multithreaded modes:
|
||||||
|
///
|
||||||
|
/// - Default value for --memlimit-mt-decompress
|
||||||
|
///
|
||||||
|
/// This value is caluclated in hardware_init() and cannot be changed later.
|
||||||
|
static uint64_t memlimit_mt_default;
|
||||||
|
|
||||||
|
/// Memory usage limit for multithreaded decompression. This is a soft limit:
|
||||||
|
/// if reducing the number of threads to one isn't enough to keep memory
|
||||||
|
/// usage below this limit, then one thread is used and this limit is ignored.
|
||||||
|
/// memlimit_decompress is still obeyed.
|
||||||
|
///
|
||||||
|
/// This can be set with --memlimit-mt-decompress. The default value for
|
||||||
|
/// this is memlimit_mt_default.
|
||||||
|
static uint64_t memlimit_mtdec;
|
||||||
|
|
||||||
/// Total amount of physical RAM
|
/// Total amount of physical RAM
|
||||||
static uint64_t total_ram;
|
static uint64_t total_ram;
|
||||||
|
@ -60,7 +76,8 @@ hardware_threads_get(void)
|
||||||
|
|
||||||
extern void
|
extern void
|
||||||
hardware_memlimit_set(uint64_t new_memlimit,
|
hardware_memlimit_set(uint64_t new_memlimit,
|
||||||
bool set_compress, bool set_decompress, bool is_percentage)
|
bool set_compress, bool set_decompress, bool set_mtdec,
|
||||||
|
bool is_percentage)
|
||||||
{
|
{
|
||||||
if (is_percentage) {
|
if (is_percentage) {
|
||||||
assert(new_memlimit > 0);
|
assert(new_memlimit > 0);
|
||||||
|
@ -110,6 +127,9 @@ hardware_memlimit_set(uint64_t new_memlimit,
|
||||||
if (set_decompress)
|
if (set_decompress)
|
||||||
memlimit_decompress = new_memlimit;
|
memlimit_decompress = new_memlimit;
|
||||||
|
|
||||||
|
if (set_mtdec)
|
||||||
|
memlimit_mtdec = new_memlimit;
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -132,6 +152,23 @@ hardware_memlimit_get(enum operation_mode mode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
extern uint64_t
|
||||||
|
hardware_memlimit_mtdec_get(void)
|
||||||
|
{
|
||||||
|
uint64_t m = memlimit_mtdec != 0
|
||||||
|
? memlimit_mtdec
|
||||||
|
: memlimit_mt_default;
|
||||||
|
|
||||||
|
// Cap the value to memlimit_decompress if it has been specified.
|
||||||
|
// This is nice for --info-memory. It wouldn't be needed for liblzma
|
||||||
|
// since it does this anyway.
|
||||||
|
if (memlimit_decompress != 0 && m > memlimit_decompress)
|
||||||
|
m = memlimit_decompress;
|
||||||
|
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Helper for hardware_memlimit_show() to print one human-readable info line.
|
/// Helper for hardware_memlimit_show() to print one human-readable info line.
|
||||||
static void
|
static void
|
||||||
memlimit_show(const char *str, size_t str_columns, uint64_t value)
|
memlimit_show(const char *str, size_t str_columns, uint64_t value)
|
||||||
|
@ -203,7 +240,20 @@ hardware_init(void)
|
||||||
if (total_ram == 0)
|
if (total_ram == 0)
|
||||||
total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
|
total_ram = (uint64_t)(ASSUME_RAM) * 1024 * 1024;
|
||||||
|
|
||||||
// Set the defaults.
|
// FIXME? There may be better methods to determine the default value.
|
||||||
hardware_memlimit_set(0, true, true, false);
|
// One Linux-specific suggestion is to use MemAvailable from
|
||||||
|
// /proc/meminfo as the starting point.
|
||||||
|
memlimit_mt_default = total_ram / 4;
|
||||||
|
|
||||||
|
// A too high value may cause 32-bit xz to run out of address space.
|
||||||
|
// Use a conservative maximum value here. A few typical address space
|
||||||
|
// sizes with Linux:
|
||||||
|
// - x86-64 with 32-bit xz: 4 GiB
|
||||||
|
// - x86: 3 GiB
|
||||||
|
// - MIPS32: 2 GiB
|
||||||
|
const size_t mem_ceiling = SIZE_MAX / 3; // About 1365 GiB on 32-bit
|
||||||
|
if (memlimit_mt_default > mem_ceiling)
|
||||||
|
memlimit_mt_default = mem_ceiling;
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,16 +22,21 @@ extern void hardware_threads_set(uint32_t threadlimit);
|
||||||
extern uint32_t hardware_threads_get(void);
|
extern uint32_t hardware_threads_get(void);
|
||||||
|
|
||||||
|
|
||||||
/// Set the memory usage limit. There are separate limits for compression
|
/// Set the memory usage limit. There are separate limits for compression,
|
||||||
/// and decompression (the latter includes also --list), one or both can
|
/// decompression (also includes --list), and multithreaded decompression.
|
||||||
/// be set with a single call to this function. Zero indicates resetting
|
/// Any combination of these can be set with a single call to this function.
|
||||||
/// the limit back to the defaults. The limit can also be set as a percentage
|
/// Zero indicates resetting the limit back to the defaults.
|
||||||
/// of installed RAM; the percentage must be in the range [1, 100].
|
/// The limit can also be set as a percentage of installed RAM; the
|
||||||
|
/// percentage must be in the range [1, 100].
|
||||||
extern void hardware_memlimit_set(uint64_t new_memlimit,
|
extern void hardware_memlimit_set(uint64_t new_memlimit,
|
||||||
bool set_compress, bool set_decompress, bool is_percentage);
|
bool set_compress, bool set_decompress, bool set_mtdec,
|
||||||
|
bool is_percentage);
|
||||||
|
|
||||||
/// Get the current memory usage limit for compression or decompression.
|
/// Get the current memory usage limit for compression or decompression.
|
||||||
extern uint64_t hardware_memlimit_get(enum operation_mode mode);
|
extern uint64_t hardware_memlimit_get(enum operation_mode mode);
|
||||||
|
|
||||||
|
/// Get the current memory usage limit for multithreaded decompression.
|
||||||
|
extern uint64_t hardware_memlimit_mtdec_get(void);
|
||||||
|
|
||||||
/// Display the amount of RAM and memory usage limits and exit.
|
/// Display the amount of RAM and memory usage limits and exit.
|
||||||
extern void hardware_memlimit_show(void) lzma_attribute((__noreturn__));
|
extern void hardware_memlimit_show(void) lzma_attribute((__noreturn__));
|
||||||
|
|
|
@ -1180,9 +1180,11 @@ message_help(bool long_help)
|
||||||
puts(_( // xgettext:no-c-format
|
puts(_( // xgettext:no-c-format
|
||||||
" --memlimit-compress=LIMIT\n"
|
" --memlimit-compress=LIMIT\n"
|
||||||
" --memlimit-decompress=LIMIT\n"
|
" --memlimit-decompress=LIMIT\n"
|
||||||
|
" --memlimit-mt-decompress=LIMIT\n"
|
||||||
" -M, --memlimit=LIMIT\n"
|
" -M, --memlimit=LIMIT\n"
|
||||||
" set memory usage limit for compression, decompression,\n"
|
" set memory usage limit for compression, decompression,\n"
|
||||||
" or both; LIMIT is in bytes, % of RAM, or 0 for defaults"));
|
" threaded decompression, or all of these; LIMIT is in\n"
|
||||||
|
" bytes, % of RAM, or 0 for defaults"));
|
||||||
|
|
||||||
puts(_(
|
puts(_(
|
||||||
" --no-adjust if compression settings exceed the memory usage limit,\n"
|
" --no-adjust if compression settings exceed the memory usage limit,\n"
|
||||||
|
|
Loading…
Reference in a new issue