Run Apache Spark on Google Colab¶
This is a compact guide on how to set up Apache Spark on Google Colab.
A more detailed walkthrough of how to setup Spark on a single machine in standalone mode is presented in in Hadoop: Setting up Spark Standalone on Google Colab.
Setup Spark¶
Setup Apache Spark in 1️⃣ 2️⃣ 3️⃣ 4️⃣ steps (step 0️⃣ is the Java installation, which is skipped because Java is available in Google Colab).
The following code should also run on any Ubuntu machine or Docker container except for the Web servers links.
In [1]:
import requests
import subprocess
import os
import re
import socket
import shutil
import time
import sys
def run(cmd):
# run a shell command
try:
# Run the command and capture stdout and stderr
subprocess_output = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
# Access stdout (stderr redirected to stdout)
stdout_result = subprocess_output.stdout.strip().splitlines()[-1]
# Process the results as needed
print(f'✅ {stdout_result}')
return stdout_result
except subprocess.CalledProcessError as e:
# Handle the error if the command returns a non-zero exit code
print(f"Command failed with return code {e.returncode}")
print("stdout:", e.stdout)
def is_java_installed():
return shutil.which("java")
def install_java():
# Uncomment and modify the desired version
# java_version= 'openjdk-11-jre-headless'
# java_version= 'default-jre'
# java_version= 'openjdk-17-jre-headless'
# java_version= 'openjdk-18-jre-headless'
java_version= 'openjdk-19-jre-headless'
os.environ['JAVA_HOME'] = ' /usr/lib/jvm/java-19-openjdk-amd64'
print(f"Java not found. Installing {java_version} ... (this might take a while)")
try:
cmd = f"apt install -y {java_version}"
subprocess_output = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
stdout_result = subprocess_output.stdout
# Process the results as needed
print(f'✅ Done installing Java {java_version}')
except subprocess.CalledProcessError as e:
# Handle the error if the command returns a non-zero exit code
print(f"Command failed with return code {e.returncode}")
print("stdout:", e.stdout)
print("\n0️⃣ Install Java if not available")
if is_java_installed():
print("✅ Java is already installed.")
else:
install_java()
print("\n1️⃣ Download and install Hadoop and Spark")
# URL for downloading Hadoop and Spark
SPARK_VERSION = "3.5.3"
HADOOP_SPARK_URL = "https://dlcdn.apache.org/spark/spark-" + SPARK_VERSION + \
"/spark-" + SPARK_VERSION + "-bin-hadoop3.tgz"
r = requests.head(HADOOP_SPARK_URL)
if r.status_code >= 200 and r.status_code < 400:
print(f'✅ {HADOOP_SPARK_URL} was found')
else:
SPARK_CDN = "https://dlcdn.apache.org/spark/"
print(f'⚠️ {HADOOP_SPARK_URL} was NOT found. \nCheck for available Spark versions in {SPARK_CDN}')
# set some environment variables
os.environ['SPARK_HOME'] = os.path.join(os.getcwd(), os.path.splitext(os.path.basename(HADOOP_SPARK_URL))[0])
os.environ['PATH'] = ':'.join([os.path.join(os.environ['SPARK_HOME'], 'bin'), os.environ['PATH']])
os.environ['PATH'] = ':'.join([os.path.join(os.environ['SPARK_HOME'], 'sbin'), os.environ['PATH']])
# download Spark
# using --no-clobber option will prevent wget from downloading file if already present
# shell command: wget --no-clobber $HADOOP_SPARK_URL
cmd = f"wget --no-clobber {HADOOP_SPARK_URL}"
run(cmd)
# uncompress
try:
# Run the command and capture stdout and stderr
cmd = "([ -d $(basename {0}|sed 's/\.[^.]*$//') ] && echo -n 'Folder already exists') || (tar xzf $(basename {0}) && echo 'Uncompressed Spark distribution')"
subprocess_output = subprocess.run(cmd.format(HADOOP_SPARK_URL), shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
# Access stdout (stderr redirected to stdout)
stdout_result = subprocess_output.stdout
# Process the results as needed
print(f'✅ {stdout_result}')
except subprocess.CalledProcessError as e:
# Handle the error if the command returns a non-zero exit code
print(f"Command failed with return code {e.returncode}")
print("stdout:", e.stdout)
print("\n2️⃣ Start Spark engine")
# start master
# shell command: $SPARK_HOME/sbin/start-master.sh
cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'stop-master.sh')
run(cmd)
cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'start-master.sh')
out = run(cmd)
# start one worker (first stop it in case it's already running)
# shell command: $SPARK_HOME/sbin/start-worker.sh spark://${HOSTNAME}:7077
cmd = [os.path.join(os.environ['SPARK_HOME'], 'sbin', 'stop-worker.sh')]
run(cmd)
cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'start-worker.sh') + ' ' + 'spark://'+socket.gethostname()+':7077'
run(cmd)
print("\n3️⃣ Start Master Web UI")
# get master UI's port number
# the subprocess that's starting the master with start-master.sh
# might still not be ready with assigning the port number at this point
# therefore we check the logfile a few times (attempts=5) to see if the port
# has been assigned. This might take 1-2 seconds.
master_log = out.partition("logging to")[2].strip()
print("Search for port number in log file {}".format(master_log))
attempts = 10
search_pattern = "Successfully started service 'MasterUI' on port (\d+)"
found = False
for i in range(attempts):
if not found:
with open(master_log) as log:
found = re.search(search_pattern, log.read())
if found:
webUIport = found.group(1)
print(f"✅ Master UI is available at localhost:{webUIport} (attempt nr. {i})")
break
else:
time.sleep(2) # need to try until port information is found in the logfile
i+=1
if not found:
print("Could not find port for Master Web UI\n")
IN_COLAB = 'google.colab' in sys.modules
if IN_COLAB:
# serve the Web UI on Colab
print("Click on the link below to open the Spark Web UI 🚀")
from google.colab import output
output.serve_kernel_port_as_window(webUIport)
print("\n4️⃣ Start history server")
# start history server
# shell command: mkdir -p /tmp/spark-events
# shell command: $SPARK_HOME/sbin/start-history-server.sh
spark_events_dir = os.path.join('/tmp', 'spark-events')
if not os.path.exists(spark_events_dir):
os.mkdir(spark_events_dir)
cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'stop-history-server.sh')
run(cmd)
cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'start-history-server.sh')
run(cmd)
if IN_COLAB:
# serve the History Server
print("Click on the link below to open the Spark History Server Web UI 🚀")
output.serve_kernel_port_as_window(18080)
0️⃣ Install Java if not available
✅ Java is already installed.
1️⃣ Download and install Hadoop and Spark
✅ https://dlcdn.apache.org/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz was found
✅ 2024-10-07 11:49:46 (54.8 MB/s) - ‘spark-3.5.3-bin-hadoop3.tgz’ saved [400864419/400864419]
✅ Uncompressed Spark distribution
2️⃣ Start Spark engine
✅ no org.apache.spark.deploy.master.Master to stop
✅ starting org.apache.spark.deploy.master.Master, logging to /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.master.Master-1-651dec53631c.out
✅ no org.apache.spark.deploy.worker.Worker to stop
✅ starting org.apache.spark.deploy.worker.Worker, logging to /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.worker.Worker-1-651dec53631c.out
3️⃣ Start Master Web UI
Search for port number in log file /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.master.Master-1-651dec53631c.out
✅ Master UI is available at localhost:8081 (attempt nr. 3)
Click on the link below to open the Spark Web UI 🚀
Warning: This function may stop working due to changes in browser security.
Try `serve_kernel_port_as_iframe` instead.
4️⃣ Start history server
✅ no org.apache.spark.deploy.history.HistoryServer to stop
✅ starting org.apache.spark.deploy.history.HistoryServer, logging to /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.history.HistoryServer-1-651dec53631c.out
Click on the link below to open the Spark History Server Web UI 🚀
Warning: This function may stop working due to changes in browser security.
Try `serve_kernel_port_as_iframe` instead.
Run a couple of examples¶
We are going to run pre-built Java examples that come with the Spark distribution.
Run the Java application SparkPi to estimate $\pi$¶
Run the pre-built example org.apache.spark.examples.SparkPi that comes with the Spark distribution using bash to submit the job.
$100$ is the number of iterations.
In [2]:
%%bash
EXAMPLES_JAR=$(find $SPARK_HOME/examples/jars/ -name "spark-examples*")
$SPARK_HOME/bin/spark-submit \
--class org.apache.spark.examples.SparkPi \
--master spark://${HOSTNAME}:7077 \
--conf spark.eventLog.enabled=true \
$EXAMPLES_JAR \
100 \
2>/tmp/SparkPi_bash.log
Pi is roughly 3.1415227141522712
Java Word Count¶
The source code for this example is available on GitHub at: https://github.com/apache/spark/blob/master/examples/src/main/java/org/apache/spark/examples/JavaWordCount.java
In [3]:
%%bash
echo "Downloading file ..."
URL="https://www.gutenberg.org/cache/epub/71036/pg71036.txt"
([ -f datafile.txt ] && echo "File already exists") || curl -o datafile.txt $URL
Downloading file ...
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 47566 100 47566 0 0 244k 0 --:--:-- --:--:-- --:--:-- 245k
In [4]:
%%bash
EXAMPLES_JAR=$(find $SPARK_HOME/examples/jars/ -name "spark-examples*")
$SPARK_HOME/bin/spark-submit \
--class org.apache.spark.examples.JavaWordCount \
--master spark://${HOSTNAME}:7077 \
--conf spark.eventLog.enabled=true \
$EXAMPLES_JAR \
datafile.txt \
2>/tmp/JavaWordCount.log \
1>/tmp/JavaWordCount.out
head /tmp/JavaWordCount.out
young: 5 Unless: 1 seriousness,: 1 errors,: 1 repent.: 1 (or: 3 Printers.: 1 full: 10 previous: 1 secure: 1
To download a larger amount of data from the Gutenberg collection, see the tutorial Explore and download books from the Gutenberg Books collection from the BDb repository.
Where to find things¶
Spark Web UI¶
The Spark Web UI is available at:
In [5]:
if IN_COLAB:
from google.colab.output import eval_js
print(eval_js( "google.colab.kernel.proxyPort(" + str(webUIport) + ")" ))
https://7ct245v95zw-496ff2e9c6d22116-8081-colab.googleusercontent.com/
In the free tier of Google Colab this functionality might not be available (see https://research.google.com/colaboratory/faq.html#limitations-and-restrictions). As an alternative, you can use ngrok after signing up for a free account.
Use ngrok to access the Web UI¶
Check the NGROK box below if you want to use ngrok (by default this is set to False).
In [6]:
# you should set this to True
NGROK = False #@param {type:"boolean"}
We are going to use the Python ngrok client pyngrok (see the Colab example).
In [7]:
if NGROK:
!pip install pyngrok
from pyngrok import ngrok, conf
import getpass
print("Enter your authtoken, which can be copied from https://dashboard.ngrok.com/get-started/your-authtoken")
authtoken = getpass.getpass()
conf.get_default().auth_token = authtoken
Note: It might be necessary to close other open sessions by stopping ngrok agents in https://dashboard.ngrok.com/tunnels/agents (the ngrok free tier has a limit of 1 simultaneous ngrok agent sessions).
After entering the ngrok authorization token, you can open a connection.
In [8]:
if NGROK:
# close all existing connections (https://pyngrok.readthedocs.io/en/latest/#get-active-tunnels)
tunnels = ngrok.get_tunnels()
if tunnels:
map(lambda t: ngrok.disconnect(t.public_url), tunnels)
# Open a ngrok tunnel to the HTTP server
public_url = ngrok.connect(webUIport).public_url
print(f'Click on {public_url} to open the Spark Master Web UI')
You can safely ignore the warning since we are not disclosing any confidential information and proceed with clicking on the "Visit site" button.
Spark History Server¶
The Spark History Server is available at:
In [9]:
if IN_COLAB:
from google.colab.output import eval_js
print(eval_js( "google.colab.kernel.proxyPort(" + str(18080) + ")" ))
https://5tsehcxns9p-496ff2e9c6d22116-18080-colab.googleusercontent.com/
With ngrok:
In [10]:
if NGROK:
# Open a ngrok tunnel to the HTTP server
public_url = ngrok.connect(18080).public_url
print(f'Click on {public_url} to open the Spark Master Web UI')
Logs for the Spark Master¶
In [11]:
!head -20 $SPARK_HOME/logs/*Master*.out
Spark Command: /usr/lib/jvm/java-11-openjdk-amd64/bin/java -cp /content/spark-3.5.3-bin-hadoop3/conf/:/content/spark-3.5.3-bin-hadoop3/jars/* -Xmx1g org.apache.spark.deploy.master.Master --host 651dec53631c --port 7077 --webui-port 8080 ======================================== Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties 24/10/07 11:50:02 INFO Master: Started daemon with process name: 732@651dec53631c 24/10/07 11:50:02 INFO SignalUtils: Registering signal handler for TERM 24/10/07 11:50:02 INFO SignalUtils: Registering signal handler for HUP 24/10/07 11:50:02 INFO SignalUtils: Registering signal handler for INT 24/10/07 11:50:04 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 24/10/07 11:50:04 INFO SecurityManager: Changing view acls to: root 24/10/07 11:50:04 INFO SecurityManager: Changing modify acls to: root 24/10/07 11:50:04 INFO SecurityManager: Changing view acls groups to: 24/10/07 11:50:04 INFO SecurityManager: Changing modify acls groups to: 24/10/07 11:50:04 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: root; groups with view permissions: EMPTY; users with modify permissions: root; groups with modify permissions: EMPTY 24/10/07 11:50:06 INFO Utils: Successfully started service 'sparkMaster' on port 7077. 24/10/07 11:50:06 INFO Master: Starting Spark master at spark://651dec53631c:7077 24/10/07 11:50:06 INFO Master: Running Spark version 3.5.3 24/10/07 11:50:07 INFO JettyUtils: Start Jetty 0.0.0.0:8080 for MasterUI 24/10/07 11:50:07 WARN Utils: Service 'MasterUI' could not bind on port 8080. Attempting port 8081. 24/10/07 11:50:07 INFO Utils: Successfully started service 'MasterUI' on port 8081. 24/10/07 11:50:08 INFO MasterWebUI: Bound MasterWebUI to 0.0.0.0, and started at http://651dec53631c:8081
Logs for the Spark Worker¶
In [12]:
!head -20 $SPARK_HOME/logs/*Worker*.out
Spark Command: /usr/lib/jvm/java-11-openjdk-amd64/bin/java -cp /content/spark-3.5.3-bin-hadoop3/conf/:/content/spark-3.5.3-bin-hadoop3/jars/* -Xmx1g org.apache.spark.deploy.worker.Worker --webui-port 8081 spark://651dec53631c:7077 ======================================== Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties 24/10/07 11:50:08 INFO Worker: Started daemon with process name: 807@651dec53631c 24/10/07 11:50:08 INFO SignalUtils: Registering signal handler for TERM 24/10/07 11:50:08 INFO SignalUtils: Registering signal handler for HUP 24/10/07 11:50:08 INFO SignalUtils: Registering signal handler for INT 24/10/07 11:50:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 24/10/07 11:50:09 INFO SecurityManager: Changing view acls to: root 24/10/07 11:50:09 INFO SecurityManager: Changing modify acls to: root 24/10/07 11:50:09 INFO SecurityManager: Changing view acls groups to: 24/10/07 11:50:09 INFO SecurityManager: Changing modify acls groups to: 24/10/07 11:50:09 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: root; groups with view permissions: EMPTY; users with modify permissions: root; groups with modify permissions: EMPTY 24/10/07 11:50:10 INFO Utils: Successfully started service 'sparkWorker' on port 41689. 24/10/07 11:50:10 INFO Worker: Worker decommissioning not enabled. 24/10/07 11:50:11 INFO Worker: Starting Spark worker 172.28.0.12:41689 with 2 cores, 11.7 GiB RAM 24/10/07 11:50:11 INFO Worker: Running Spark version 3.5.3 24/10/07 11:50:11 INFO Worker: Spark home: /content/spark-3.5.3-bin-hadoop3 24/10/07 11:50:11 INFO ResourceUtils: ============================================================== 24/10/07 11:50:11 INFO ResourceUtils: No custom resources configured for spark.worker.
Spark events (used by History Server)¶
In [13]:
!head -20 /tmp/spark-events/*
==> /tmp/spark-events/app-20241007115024-0000 <==
{"Event":"SparkListenerLogStart","Spark Version":"3.5.3"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"651dec53631c","Port":33019},"Maximum Memory":455501414,"Timestamp":1728301824810,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-11-openjdk-amd64","Java Version":"11.0.24 (Ubuntu)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"651dec53631c","spark.eventLog.enabled":"true","spark.driver.port":"37939","spark.jars":"file:/content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar","spark.app.name":"Spark Pi","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.app.submitTime":"1728301820515","spark.app.startTime":"1728301820670","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.initial.jar.urls":"spark://651dec53631c:37939/jars/spark-examples_2.12-3.5.3.jar","spark.submit.deployMode":"client","spark.master":"spark://651dec53631c:7077","spark.app.id":"app-20241007115024-0000"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"2","java.specification.version":"11","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Ubuntu","java.vm.specification.version":"11","user.home":"/root","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-11-openjdk-amd64/lib","user.dir":"/content","java.library.path":"/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"11.0.24+8-post-Ubuntu-1ubuntu322.04","jetty.git.hash":"cef3fbd6d736a21e7d541a5db490381d95a2047d","java.runtime.version":"11.0.24+8-post-Ubuntu-1ubuntu322.04","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2024-07-16","file.separator":"/","java.class.version":"55.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Etc/UTC","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"32-bit","os.version":"6.1.85+","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"https://ubuntu.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Ubuntu","jdk.debug":"release","java.vendor.url.bug":"https://bugs.launchpad.net/ubuntu/+source/openjdk-lts","user.name":"root","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://651dec53631c:7077 --conf spark.eventLog.enabled=true --class org.apache.spark.examples.SparkPi /content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar 100","java.home":"/usr/lib/jvm/java-11-openjdk-amd64","java.version":"11.0.24","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/chill_2.12-0.10.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-serde-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/paranamer-2.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-collections4-4.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-cli-1.5.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/joda-time-2.12.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-jvm-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-repl_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-math3-3.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-all-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-graphx_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-exec-2.3.9-core.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arpack_combined_all-0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kryo-shaded-4.0.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arpack-3.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/rocksdbjni-8.3.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/lapack-3.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/okio-1.17.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-client-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/ST4-4.0.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-sql_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/pickle-1.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jline-2.14.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/leveldbjni-all-1.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-unsafe_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zjsonpatch-0.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/guava-14.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-launcher_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hk2-locator-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-lang-2.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-logging-1.1.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/shims-0.9.45.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datasketches-java-3.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-core-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/libfb303-0.9.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/annotations-17.0.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-kvstore_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/libthrift-0.12.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-metastore-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/curator-framework-2.13.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-core_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-codec-1.16.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/tink-1.9.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-compress-1.23.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-json-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-compiler-2.12.18.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-common-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/derby-10.14.2.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/super-csv-2.2.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-io-2.16.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/conf/":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/transaction-api-1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-text-1.10.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/xz-1.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/ivy-2.5.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.inject-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jdo-api-3.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/blas-3.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/httpclient-4.5.14.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datasketches-memory-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-engine-2.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-common-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/aircompressor-0.27.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-tags_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/snappy-java-1.1.10.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/opencsv-2.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/orc-core-1.9.4-shaded-protobuf.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-beeline-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/audience-annotations-0.5.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-jmx-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-sql-api_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-compiler-3.1.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jpam-1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/stream-2.9.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-mesos_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-common-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/JLargeArrays-1.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/javassist-3.29.2-GA.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/avro-mapred-1.11.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/minlog-1.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jta-1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/chill-java-0.10.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-format-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/compress-lzf-1.1.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/objenesis-3.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-server-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/orc-shims-1.9.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-2.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/avro-1.11.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-cli-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-encoding-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/activation-1.1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-column-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hk2-api-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json-1.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jsr305-3.0.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/JTransforms-3.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/stax-api-1.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-databind-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-graphite-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/gson-2.2.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/antlr-runtime-3.5.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/breeze_2.12-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-lang3-3.12.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-network-common_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-common-utils_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-llap-common-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-jackson-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-reflect-2.12.18.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-dbcp-1.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/logging-interceptor-3.12.12.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-core-4.1.17.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/univocity-parsers-2.9.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-api-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-hk2-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-sketch_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-hive_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-storage-api-2.8.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-annotations-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-common-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-kubernetes_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/httpcore-4.4.16.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/oro-2.0.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/orc-mapreduce-1.9.4-shaded-protobuf.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/curator-client-2.13.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hk2-utils-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/okhttp-3.12.12.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-core-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/threeten-extra-1.7.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-yarn_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-crypto-1.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-common-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/curator-recipes-2.13.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-jdbc-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/HikariCP-2.5.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-vector-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-pool-1.5.4.jar":"System Classpath","spark://651dec53631c:37939/jars/spark-examples_2.12-3.5.3.jar":"Added By User","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-catalyst_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jodd-core-3.5.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-collections-3.2.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-3.6.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/javolution-5.5.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/slf4j-api-2.0.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/py4j-0.10.9.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib-local_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/lz4-java-1.8.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-network-shuffle_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/avro-ipc-1.11.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-streaming_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/algebra_2.12-2.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/janino-3.1.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-library-2.12.18.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"Spark Pi","App ID":"app-20241007115024-0000","Timestamp":1728301820670,"User":"root"}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1728301829820,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at SparkPi.scala:38","Number of Tasks":100,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"map at SparkPi.scala:34","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":100,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at SparkPi.scala:34","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":100,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.rdd.RDD.reduce(RDD.scala:1121)\norg.apache.spark.examples.SparkPi$.main(SparkPi.scala:38)\norg.apache.spark.examples.SparkPi.main(SparkPi.scala)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"reduce\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at SparkPi.scala:38","Number of Tasks":100,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"map\"}","Callsite":"map at SparkPi.scala:34","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":100,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parallelize at SparkPi.scala:34","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":100,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.rdd.RDD.reduce(RDD.scala:1121)\norg.apache.spark.examples.SparkPi$.main(SparkPi.scala:38)\norg.apache.spark.examples.SparkPi.main(SparkPi.scala)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1728301830104,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"reduce\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerExecutorAdded","Timestamp":1728301838229,"Executor ID":"0","Executor Info":{"Host":"172.28.0.12","Total Cores":2,"Log Urls":{"stdout":"http://172.28.0.12:8082/logPage/?appId=app-20241007115024-0000&executorId=0&logType=stdout","stderr":"http://172.28.0.12:8082/logPage/?appId=app-20241007115024-0000&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1728301838229}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.28.0.12","Port":39293},"Maximum Memory":455501414,"Timestamp":1728301838480,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1728301838872,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1728301838975,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1728301840899,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1728301840933,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Partition ID":1,"Launch Time":1728301838975,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1728301840952,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":1424,"Value":1424,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":126236942,"Value":126236942,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":290,"Value":290,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":72587498,"Value":72587498,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":1012,"Value":1012,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":1424,"Executor Deserialize CPU Time":126236942,"Executor Run Time":290,"Executor CPU Time":72587498,"Peak Execution Memory":0,"Result Size":1012,"JVM GC Time":0,"Result Serialization Time":16,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1728301838872,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1728301840971,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":1431,"Value":2855,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":385065942,"Value":511302884,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":286,"Value":576,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":73833811,"Value":146421309,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":1012,"Value":2024,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":9,"Value":25,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":1431,"Executor Deserialize CPU Time":385065942,"Executor Run Time":286,"Executor CPU Time":73833811,"Peak Execution Memory":0,"Result Size":1012,"JVM GC Time":0,"Result Serialization Time":9,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Partition ID":4,"Launch Time":1728301841176,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Partition ID":2,"Launch Time":1728301840899,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1728301841222,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":60,"Value":2915,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6552543,"Value":517855427,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":126,"Value":702,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":26924342,"Value":173345651,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":1012,"Value":3036,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":6,"Value":31,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":60,"Executor Deserialize CPU Time":6552543,"Executor Run Time":126,"Executor CPU Time":26924342,"Peak Execution Memory":0,"Result Size":1012,"JVM GC Time":0,"Result Serialization Time":6,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Partition ID":5,"Launch Time":1728301841355,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Partition ID":3,"Launch Time":1728301840933,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1728301841356,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"internal.metrics.executorDeserializeTime","Update":45,"Value":2960,"Internal":true,"Count Failed Values":true},{"ID":1,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5884696,"Value":523740123,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorRunTime","Update":281,"Value":983,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorCpuTime","Update":27269942,"Value":200615593,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.resultSize","Update":1012,"Value":4048,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.resultSerializationTime","Update":17,"Value":48,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":45,"Executor Deserialize CPU Time":5884696,"Executor Run Time":281,"Executor CPU Time":27269942,"Peak Execution Memory":0,"Result Size":1012,"JVM GC Time":0,"Result Serialization Time":17,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Partition ID":6,"Launch Time":1728301841462,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
==> /tmp/spark-events/app-20241007115053-0001 <==
{"Event":"SparkListenerLogStart","Spark Version":"3.5.3"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"651dec53631c","Port":41653},"Maximum Memory":455501414,"Timestamp":1728301853573,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-11-openjdk-amd64","Java Version":"11.0.24 (Ubuntu)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"651dec53631c","spark.eventLog.enabled":"true","spark.driver.port":"33373","spark.jars":"file:/content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar","spark.app.name":"JavaWordCount","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"","spark.app.submitTime":"1728301851272","spark.app.startTime":"1728301851395","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.initial.jar.urls":"spark://651dec53631c:33373/jars/spark-examples_2.12-3.5.3.jar","spark.submit.deployMode":"client","spark.master":"spark://651dec53631c:7077","spark.app.id":"app-20241007115053-0001"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"2","java.specification.version":"11","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Ubuntu","java.vm.specification.version":"11","user.home":"/root","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-11-openjdk-amd64/lib","user.dir":"/content","java.library.path":"/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"11.0.24+8-post-Ubuntu-1ubuntu322.04","jetty.git.hash":"cef3fbd6d736a21e7d541a5db490381d95a2047d","java.runtime.version":"11.0.24+8-post-Ubuntu-1ubuntu322.04","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2024-07-16","file.separator":"/","java.class.version":"55.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Etc/UTC","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"32-bit","os.version":"6.1.85+","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"https://ubuntu.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Ubuntu","jdk.debug":"release","java.vendor.url.bug":"https://bugs.launchpad.net/ubuntu/+source/openjdk-lts","user.name":"root","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://651dec53631c:7077 --conf spark.eventLog.enabled=true --class org.apache.spark.examples.JavaWordCount /content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar datafile.txt","java.home":"/usr/lib/jvm/java-11-openjdk-amd64","java.version":"11.0.24","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/chill_2.12-0.10.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-serde-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/paranamer-2.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-collections4-4.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-cli-1.5.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/joda-time-2.12.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-jvm-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-repl_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-math3-3.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-all-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-graphx_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-exec-2.3.9-core.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arpack_combined_all-0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kryo-shaded-4.0.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arpack-3.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/rocksdbjni-8.3.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/lapack-3.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/okio-1.17.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-client-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/ST4-4.0.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-sql_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/pickle-1.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jline-2.14.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/leveldbjni-all-1.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-unsafe_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zjsonpatch-0.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/guava-14.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-launcher_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hk2-locator-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-lang-2.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-logging-1.1.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/shims-0.9.45.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datasketches-java-3.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-core-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/libfb303-0.9.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/annotations-17.0.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-kvstore_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/libthrift-0.12.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-metastore-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/curator-framework-2.13.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-core_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-codec-1.16.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/tink-1.9.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-compress-1.23.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-json-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-compiler-2.12.18.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-common-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/derby-10.14.2.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/super-csv-2.2.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-io-2.16.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/conf/":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/transaction-api-1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-text-1.10.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/xz-1.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/ivy-2.5.1.jar":"System Classpath","spark://651dec53631c:33373/jars/spark-examples_2.12-3.5.3.jar":"Added By User","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.inject-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jdo-api-3.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/blas-3.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/httpclient-4.5.14.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datasketches-memory-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-engine-2.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-common-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/aircompressor-0.27.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-tags_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/snappy-java-1.1.10.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/opencsv-2.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/orc-core-1.9.4-shaded-protobuf.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-beeline-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/audience-annotations-0.5.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-jmx-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-sql-api_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-compiler-3.1.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jpam-1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/stream-2.9.6.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-mesos_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-common-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/JLargeArrays-1.5.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/javassist-3.29.2-GA.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/avro-mapred-1.11.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/minlog-1.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jta-1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/chill-java-0.10.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-format-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/compress-lzf-1.1.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/objenesis-3.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-server-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/orc-shims-1.9.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-2.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/avro-1.11.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-cli-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-encoding-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/activation-1.1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-column-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hk2-api-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json-1.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jsr305-3.0.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/JTransforms-3.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/stax-api-1.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-databind-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-graphite-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/gson-2.2.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/antlr-runtime-3.5.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/breeze_2.12-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-lang3-3.12.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-network-common_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-common-utils_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-llap-common-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-jackson-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-reflect-2.12.18.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-dbcp-1.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/logging-interceptor-3.12.12.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-core-4.1.17.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/univocity-parsers-2.9.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/log4j-api-2.20.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jersey-hk2-2.40.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-sketch_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-hive_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-storage-api-2.8.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-annotations-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-common-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-kubernetes_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/httpcore-4.4.16.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/oro-2.0.8.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/orc-mapreduce-1.9.4-shaded-protobuf.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/curator-client-2.13.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hk2-utils-2.6.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/okhttp-3.12.12.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/metrics-core-4.2.19.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/threeten-extra-1.7.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-yarn_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-crypto-1.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-common-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/curator-recipes-2.13.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-jdbc-2.3.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/HikariCP-2.5.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-vector-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-pool-1.5.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-catalyst_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jodd-core-3.5.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/commons-collections-3.2.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-3.6.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/javolution-5.5.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/slf4j-api-2.0.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/py4j-0.10.9.7.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib-local_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/lz4-java-1.8.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-network-shuffle_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/avro-ipc-1.11.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/spark-streaming_2.12-3.5.3.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/algebra_2.12-2.0.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/janino-3.1.9.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/scala-library-2.12.18.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/content/spark-3.5.3-bin-hadoop3/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"JavaWordCount","App ID":"app-20241007115053-0001","Timestamp":1728301851395,"User":"root"}
{"Event":"SparkListenerExecutorAdded","Timestamp":1728301863574,"Executor ID":"0","Executor Info":{"Host":"172.28.0.12","Total Cores":2,"Log Urls":{"stdout":"http://172.28.0.12:8082/logPage/?appId=app-20241007115053-0001&executorId=0&logType=stdout","stderr":"http://172.28.0.12:8082/logPage/?appId=app-20241007115053-0001&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1728301863574}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.28.0.12","Port":35809},"Maximum Memory":455501414,"Timestamp":1728301863844,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1728301867496,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"mapToPair at JavaWordCount.java:49","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"map\"}","Callsite":"mapToPair at JavaWordCount.java:49","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"1\",\"name\":\"Scan text \"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"mapPartitions\"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"SQLExecutionRDD","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"flatMap\"}","Callsite":"flatMap at JavaWordCount.java:47","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"Scan text \"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45)\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:49)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"collect at JavaWordCount.java:53","Number of Tasks":1,"RDD Info":[{"RDD ID":7,"Name":"ShuffledRDD","Scope":"{\"id\":\"5\",\"name\":\"reduceByKey\"}","Callsite":"reduceByKey at JavaWordCount.java:51","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:53)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0,1],"Properties":{"spark.rdd.scope":"{\"id\":\"6\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"mapToPair at JavaWordCount.java:49","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"map\"}","Callsite":"mapToPair at JavaWordCount.java:49","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"1\",\"name\":\"Scan text \"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"mapPartitions\"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"SQLExecutionRDD","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"flatMap\"}","Callsite":"flatMap at JavaWordCount.java:47","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"Scan text \"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45)\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:49)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1728301867588,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"6\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1728301867803,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1728301867803,"Executor ID":"0","Host":"172.28.0.12","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1728301873732,"Failed":false,"Killed":false,"Accumulables":[{"ID":0,"Name":"number of output rows","Update":"921","Value":"921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4,"Name":"internal.metrics.executorDeserializeTime","Update":1159,"Value":1159,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.executorDeserializeCpuTime","Update":864336864,"Value":864336864,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.executorRunTime","Update":4604,"Value":4604,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorCpuTime","Update":2593529632,"Value":2593529632,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.resultSize","Update":1842,"Value":1842,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.jvmGCTime","Update":99,"Value":99,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":13,"Name":"internal.metrics.peakExecutionMemory","Update":219951,"Value":219951,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":21712,"Value":21712,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":2357,"Value":2357,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.shuffle.write.writeTime","Update":1716215,"Value":1716215,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.input.bytesRead","Update":47566,"Value":47566,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.input.recordsRead","Update":921,"Value":921,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":1159,"Executor Deserialize CPU Time":864336864,"Executor Run Time":4604,"Executor CPU Time":2593529632,"Peak Execution Memory":219951,"Result Size":1842,"JVM GC Time":99,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":21712,"Shuffle Write Time":1716215,"Shuffle Records Written":2357},"Input Metrics":{"Bytes Read":47566,"Records Read":921},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"mapToPair at JavaWordCount.java:49","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"map\"}","Callsite":"mapToPair at JavaWordCount.java:49","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"FileScanRDD","Scope":"{\"id\":\"1\",\"name\":\"Scan text \"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"2\",\"name\":\"mapPartitions\"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"DeserializeToObject\"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"SQLExecutionRDD","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"flatMap\"}","Callsite":"flatMap at JavaWordCount.java:47","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"Scan text \"}","Callsite":"javaRDD at JavaWordCount.java:45","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45)\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:49)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1728301867588,"Completion Time":1728301873773,"Accumulables":[{"ID":0,"Name":"number of output rows","Value":"921","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4,"Name":"internal.metrics.executorDeserializeTime","Value":1159,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.executorDeserializeCpuTime","Value":864336864,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.executorRunTime","Value":4604,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.executorCpuTime","Value":2593529632,"Internal":true,"Count Failed Values":true},{"ID":8,"Name":"internal.metrics.resultSize","Value":1842,"Internal":true,"Count Failed Values":true},{"ID":9,"Name":"internal.metrics.jvmGCTime","Value":99,"Internal":true,"Count Failed Values":true},{"ID":10,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":11,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":13,"Name":"internal.metrics.peakExecutionMemory","Value":219951,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":21712,"Internal":true,"Count Failed Values":true},{"ID":33,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":2357,"Internal":true,"Count Failed Values":true},{"ID":34,"Name":"internal.metrics.shuffle.write.writeTime","Value":1716215,"Internal":true,"Count Failed Values":true},{"ID":35,"Name":"internal.metrics.input.bytesRead","Value":47566,"Internal":true,"Count Failed Values":true},{"ID":36,"Name":"internal.metrics.input.recordsRead","Value":921,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"collect at JavaWordCount.java:53","Number of Tasks":1,"RDD Info":[{"RDD ID":7,"Name":"ShuffledRDD","Scope":"{\"id\":\"5\",\"name\":\"reduceByKey\"}","Callsite":"reduceByKey at JavaWordCount.java:51","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:53)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1728301873823,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"6\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1728301873862,"Executor ID":"0","Host":"172.28.0.12","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1728301873862,"Executor ID":"0","Host":"172.28.0.12","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1728301874427,"Failed":false,"Killed":false,"Accumulables":[{"ID":39,"Name":"internal.metrics.executorDeserializeTime","Update":181,"Value":181,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeCpuTime","Update":79298899,"Value":79298899,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorRunTime","Update":287,"Value":287,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorCpuTime","Update":174608711,"Value":174608711,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.resultSize","Update":52992,"Value":52992,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.jvmGCTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.resultSerializationTime","Update":9,"Value":9,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.memoryBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.diskBytesSpilled","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.peakExecutionMemory","Update":153591,"Value":153591,"Internal":true,"Count Failed Values":true},{"ID":50,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":51,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":21712,"Value":21712,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.shuffle.read.recordsRead","Update":2357,"Value":2357,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":58,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":59,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":60,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":181,"Executor Deserialize CPU Time":79298899,"Executor Run Time":287,"Executor CPU Time":174608711,"Peak Execution Memory":153591,"Result Size":52992,"JVM GC Time":24,"Result Serialization Time":9,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":21712,"Total Records Read":2357,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"collect at JavaWordCount.java:53","Number of Tasks":1,"RDD Info":[{"RDD ID":7,"Name":"ShuffledRDD","Scope":"{\"id\":\"5\",\"name\":\"reduceByKey\"}","Callsite":"reduceByKey at JavaWordCount.java:51","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[0],"Details":"org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:53)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)","Submission Time":1728301873823,"Completion Time":1728301874430,"Accumulables":[{"ID":39,"Name":"internal.metrics.executorDeserializeTime","Value":181,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.executorDeserializeCpuTime","Value":79298899,"Internal":true,"Count Failed Values":true},{"ID":41,"Name":"internal.metrics.executorRunTime","Value":287,"Internal":true,"Count Failed Values":true},{"ID":42,"Name":"internal.metrics.executorCpuTime","Value":174608711,"Internal":true,"Count Failed Values":true},{"ID":43,"Name":"internal.metrics.resultSize","Value":52992,"Internal":true,"Count Failed Values":true},{"ID":44,"Name":"internal.metrics.jvmGCTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":45,"Name":"internal.metrics.resultSerializationTime","Value":9,"Internal":true,"Count Failed Values":true},{"ID":46,"Name":"internal.metrics.memoryBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":47,"Name":"internal.metrics.diskBytesSpilled","Value":0,"Internal":true,"Count Failed Values":true},{"ID":48,"Name":"internal.metrics.peakExecutionMemory","Value":153591,"Internal":true,"Count Failed Values":true},{"ID":50,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":51,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":52,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":53,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":54,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":21712,"Internal":true,"Count Failed Values":true},{"ID":55,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":56,"Name":"internal.metrics.shuffle.read.recordsRead","Value":2357,"Internal":true,"Count Failed Values":true},{"ID":57,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":58,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":59,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":60,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":61,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":62,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":63,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":64,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":65,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":66,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1728301874450,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerApplicationEnd","Timestamp":1728301874564}
Spark configuration¶
To customize Spark use the configuration templates in $SPARK_HOME/conf (remove the template extension).
In [14]:
!ls -al $SPARK_HOME/conf
total 44 drwxr-xr-x 2 1000 1000 4096 Sep 9 05:33 . drwxr-xr-x 15 1000 1000 4096 Oct 7 11:50 .. -rw-r--r-- 1 1000 1000 1105 Sep 9 05:33 fairscheduler.xml.template -rw-r--r-- 1 1000 1000 3350 Sep 9 05:33 log4j2.properties.template -rw-r--r-- 1 1000 1000 9141 Sep 9 05:33 metrics.properties.template -rw-r--r-- 1 1000 1000 1292 Sep 9 05:33 spark-defaults.conf.template -rwxr-xr-x 1 1000 1000 4694 Sep 9 05:33 spark-env.sh.template -rw-r--r-- 1 1000 1000 865 Sep 9 05:33 workers.template
Shutdown¶
Stop all services.
In [15]:
%%bash
$SPARK_HOME/sbin/stop-history-server.sh
$SPARK_HOME/sbin/stop-worker.sh
$SPARK_HOME/sbin/stop-master.sh
stopping org.apache.spark.deploy.history.HistoryServer stopping org.apache.spark.deploy.worker.Worker stopping org.apache.spark.deploy.master.Master
Terminate the ngrok processes.
In [16]:
if NGROK:
ngrok.kill()