TezRuntimeConfiguration


Property Name Default Value Description Type Is Private? Is Unstable? Is Evolving?
tez.runtime.cleanup.files.on.interrupt false Used only for internal testing. Strictly not recommended to be used elsewhere. This parameter could be changed/dropped later. boolean true false true
tez.runtime.combiner.class null Specifies a combiner class (primarily for Shuffle) string false false false
tez.runtime.combine.min.spills 3 integer false false false
tez.runtime.compress null boolean false false false
tez.runtime.compress.codec null string false false false
tez.runtime.convert.user-payload.to.history-text false Value: Boolean Whether to publish configuration information to History logger. Default false. string false false false
tez.runtime.empty.partitions.info-via-events.enabled true boolean false false false
tez.runtime.enable.final-merge.in.output true Expert level setting. Enable final merge in ordered (defaultsorter/pipelinedsorter) outputs. Speculative execution needs to be turned off when disabling this parameter. //TODO: TEZ-2132 boolean false false false
tez.runtime.group.comparator.class null string false false false
tez.runtime.ifile.readahead true Configuration key to enable/disable IFile readahead. boolean false false false
tez.runtime.ifile.readahead.bytes 4194304 Configuration key to set the IFile readahead length in bytes. integer false false false
tez.runtime.index.cache.memory.limit.bytes 1048576 integer false false false
tez.runtime.task.input.post-merge.buffer.percent null float false false false
tez.runtime.internal.sorter.class null string false false false
tez.runtime.io.sort.factor 100 integer false false false
tez.runtime.io.sort.mb 100 integer false false false
tez.runtime.key.class null string false false false
tez.runtime.key.comparator.class null string false false false
tez.runtime.key.secondary.comparator.class null string false false false
tez.runtime.optimize.local.fetch true If the shuffle input is on the local host bypass the http fetch and access the files directly boolean false false false
tez.runtime.optimize.shared.fetch false Share data fetched between tasks running on the same host if applicable boolean false false false
tez.runtime.partitioner.class null Specifies a partitioner class, which is used in Tez Runtime components like OnFileSortedOutput string false false false
tez.runtime.pipelined-shuffle.enabled false Expert level setting. Enable pipelined shuffle in ordered outputs and in unordered partitioned outputs. In ordered cases, it works with PipelinedSorter. set tez.runtime.sort.threads to greater than 1 to enable pipelinedsorter. Ensure to set tez.runtime.enable.final-merge.in.output=false. Speculative execution needs to be turned off when using this parameter. //TODO: TEZ-2132 boolean false false false
tez.runtime.pipelined.sorter.lazy-allocate.memory false Setting this to true would enable sorter to auto-allocate memory on need basis in progressive fashion. Setting to false would allocate all available memory during initialization of sorter. In such cases,@link{#TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB} would be honored and memory specified in @link{#TEZ_RUNTIME_IO_SORT_MB} would be initialized upfront. boolean false false false
tez.runtime.pipelined.sorter.min-block.size.in.mb 2000 Tries to allocate @link{#TEZ_RUNTIME_IO_SORT_MB} in chunks specified in this parameter. integer false false false
tez.runtime.pipelined.sorter.sort.threads 2 integer false false false
tez.runtime.merge.progress.records 10000 integer true false true
tez.runtime.report.partition.stats null Report partition statistics (e.g better scheduling in ShuffleVertexManager). TEZ-2496 This can be enabled/disabled at vertex level. {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration.ReportPartitionStats} defines the list of values that can be specified. TODO TEZ-3303 Given ShuffleVertexManager doesn't consume precise stats yet. So do not set the value to "precise" yet when ShuffleVertexManager is used. string false false false
tez.runtime.shuffle.acceptable.host-fetch.failure.fraction 0.2 float true false true
tez.runtime.shuffle.batch.wait -1 Expert level setting. How long should @link{ShuffleManager} wait for batching before sending the events in milliseconds. Set to -1 to not wait. integer false false false
tez.runtime.shuffle.buffersize 8192 integer false false false
tez.runtime.shuffle.connect.timeout null integer false false false
tez.runtime.shuffle.memory-to-memory.enable false boolean false false false
tez.runtime.shuffle.ssl.enable false boolean false false false
tez.runtime.shuffle.failed.check.since-last.completion true boolean true false true
tez.runtime.shuffle.fetcher.use-shared-pool false boolean true false true
tez.runtime.shuffle.fetch.buffer.percent 0.9 float false false false
tez.runtime.shuffle.fetch.failures.limit 5 integer false false false
tez.runtime.shuffle.fetch.max.task.output.at.once 20 integer false false false
tez.runtime.shuffle.fetch.verify-disk-checksum true Controls verification of data checksums when fetching data directly to disk. Enabling verification allows the fetcher to detect corrupted data and report the failure against the upstream task before the data reaches the Processor and causes the fetching task to fail. boolean false false false
tez.runtime.shuffle.host.penalty.time.limit 600000 Specifies in milliseconds the maximum delay a penalized host can have before being retried, defaults to 10 minutes. integer false false false
tez.runtime.shuffle.keep-alive.enabled false boolean false false false
tez.runtime.shuffle.keep-alive.max.connections 20 integer false false false
tez.runtime.shuffle.max.allowed.failed.fetch.fraction 0.5 float true false true
tez.runtime.shuffle.max.stall.time.fraction 0.5 float true false true
tez.runtime.shuffle.memory.limit.percent 0.25 float false false false
tez.runtime.shuffle.memory-to-memory.segments null integer false false false
tez.runtime.shuffle.merge.percent 0.9 float false false false
tez.runtime.shuffle.min.failures.per.host 4 integer true false true
tez.runtime.shuffle.min.required.progress.fraction 0.5 float true false true
tez.runtime.shuffle.notify.readerror true boolean false false false
tez.runtime.shuffle.parallel.copies 20 integer false false false
tez.runtime.shuffle.read.timeout 180000 integer false false false
tez.runtime.shuffle.src-attempt.abort.limit -1 integer true false true
tez.runtime.shuffle.use.async.http false boolean false false false
tez.runtime.sorter.class null String value. Which sorter implementation to use. Valid values: - LEGACY - PIPELINED ( default ) {@link org.apache.tez.runtime.library.conf.OrderedPartitionedKVOutputConfig.SorterImpl} string false false false
tez.runtime.sort.spill.percent 0.8 float false false false
tez.runtime.unordered.output.buffer.size-mb 100 Size of the buffer to use if not writing directly to disk. integer false false false
tez.runtime.unordered.output.max-per-buffer.size-bytes null Maximum size for individual buffers used in the UnsortedPartitionedOutput. This is only meant to be used by unit tests for now. integer true false false
tez.runtime.unordered-partitioned-kvwriter.buffer-merge-percent 0 Integer value. Percentage of buffer to be filled before we spill to disk. Default value is 0, which will spill for every buffer. int false false false
tez.runtime.value.class null string false false false