# @title
from IPython.core.display import HTML
HTML("""
<div style="background-color:rgb(16, 163, 127,.2);border:2px solid rgb(16, 163, 127,.3);padding:3px;">
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 320"  style="width:32px;height:32px;">
 <g fill="currentColor">
  <path d="m297.06 130.97c7.26-21.79 4.76-45.66-6.85-65.48-17.46-30.4-52.56-46.04-86.84-38.68-15.25-17.18-37.16-26.95-60.13-26.81-35.04-.08-66.13 22.48-76.91 55.82-22.51 4.61-41.94 18.7-53.31 38.67-17.59 30.32-13.58 68.54 9.92 94.54-7.26 21.79-4.76 45.66 6.85 65.48 17.46 30.4 52.56 46.04 86.84 38.68 15.24 17.18 37.16 26.95 60.13 26.8 35.06.09 66.16-22.49 76.94-55.86 22.51-4.61 41.94-18.7 53.31-38.67 17.57-30.32 13.55-68.51-9.94-94.51zm-120.28 168.11c-14.03.02-27.62-4.89-38.39-13.88.49-.26 1.34-.73 1.89-1.07l63.72-36.8c3.26-1.85 5.26-5.32 5.24-9.07v-89.83l26.93 15.55c.29.14.48.42.52.74v74.39c-.04 33.08-26.83 59.9-59.91 59.97zm-128.84-55.03c-7.03-12.14-9.56-26.37-7.15-40.18.47.28 1.3.79 1.89 1.13l63.72 36.8c3.23 1.89 7.23 1.89 10.47 0l77.79-44.92v31.1c.02.32-.13.63-.38.83l-64.41 37.19c-28.69 16.52-65.33 6.7-81.92-21.95zm-16.77-139.09c7-12.16 18.05-21.46 31.21-26.29 0 .55-.03 1.52-.03 2.2v73.61c-.02 3.74 1.98 7.21 5.23 9.06l77.79 44.91-26.93 15.55c-.27.18-.61.21-.91.08l-64.42-37.22c-28.63-16.58-38.45-53.21-21.95-81.89zm221.26 51.49-77.79-44.92 26.93-15.54c.27-.18.61-.21.91-.08l64.42 37.19c28.68 16.57 38.51 53.26 21.94 81.94-7.01 12.14-18.05 21.44-31.2 26.28v-75.81c.03-3.74-1.96-7.2-5.2-9.06zm26.8-40.34c-.47-.29-1.3-.79-1.89-1.13l-63.72-36.8c-3.23-1.89-7.23-1.89-10.47 0l-77.79 44.92v-31.1c-.02-.32.13-.63.38-.83l64.41-37.16c28.69-16.55 65.37-6.7 81.91 22 6.99 12.12 9.52 26.31 7.15 40.1zm-168.51 55.43-26.94-15.55c-.29-.14-.48-.42-.52-.74v-74.39c.02-33.12 26.89-59.96 60.01-59.94 14.01 0 27.57 4.92 38.34 13.88-.49.26-1.33.73-1.89 1.07l-63.72 36.8c-3.26 1.85-5.26 5.31-5.24 9.06l-.04 89.79zm14.63-31.54 34.65-20.01 34.65 20v40.01l-34.65 20-34.65-20z"></path>
  </svg>
  <b>Note:</b>
While the MiniCluster is useful for many development and testing scenarios, it's important to note that it does not fully replicate the complexities and challenges of a true multi-node Hadoop cluster. For production-scale testing or performance evaluations, a larger, more representative cluster setup is recommended.
</div>
""")

import urllib.request
import os
import shutil
import tarfile
import logging
import subprocess
import time
import sys
############
# COSTANTS #
############

# URL for downloading Hadoop (archive site https://archive.apache.org/dist/)
HADOOP_URL = "https://archive.apache.org/dist/hadoop/core/hadoop-3.4.0/hadoop-3.4.0.tar.gz"

# logging level (should be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL)
LOGGING_LEVEL = "INFO" #@param ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]

# setup logging
for handler in logging.root.handlers[:]:
    logging.root.removeHandler(handler)

logging_level = getattr(logging, LOGGING_LEVEL.upper(), 10)

logging.basicConfig(level=logging_level, \
                    format='%(asctime)s - %(levelname)s: %(message)s', \
                    datefmt='%d-%b-%y %I:%M:%S %p')

logger = logging.getLogger('my_logger')

JAVA_PATH = '/usr/lib/jvm/java-11-openjdk-amd64'

# true if running on Google Colab
IN_COLAB = 'google.colab' in sys.modules
if IN_COLAB:
 from google.colab import output

# setup logging
for handler in logging.root.handlers[:]:
    logging.root.removeHandler(handler)

logging_level = getattr(logging, LOGGING_LEVEL.upper(), 10)

logging.basicConfig(level=logging_level, \
                    format='%(asctime)s - %(levelname)s: %(message)s', \
                    datefmt='%d-%b-%y %I:%M:%S %p')

logger = logging.getLogger('my_logger')

# set variable JAVA_HOME (install Java if necessary)
def is_java_installed():
    os.environ['JAVA_HOME'] = os.path.realpath(shutil.which("java")).split('/bin')[0]
    return os.environ['JAVA_HOME']

def install_java():
    # Uncomment and modify the desired version
    # java_version= 'openjdk-11-jre-headless'
    # java_version= 'default-jre'
    # java_version= 'openjdk-17-jre-headless'
    # java_version= 'openjdk-18-jre-headless'
    java_version= 'openjdk-19-jre-headless'

    print(f"Java not found. Installing {java_version} ... (this might take a while)")
    try:
        cmd = f"apt install -y {java_version}"
        subprocess_output = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
        stdout_result = subprocess_output.stdout
        # Process the results as needed
        logger.info("Done installing Java {}".format(java_version))
        os.environ['JAVA_HOME'] = os.path.realpath(shutil.which("java")).split('/bin')[0]
        logger.info("JAVA_HOME is {}".format(os.environ['JAVA_HOME']))
    except subprocess.CalledProcessError as e:
        # Handle the error if the command returns a non-zero exit code
        logger.warn("Command failed with return code {}".format(e.returncode))
        logger.warn("stdout: {}".format(e.stdout))

# Install Java if not available
if is_java_installed():
    logger.info("Java is already installed: {}".format(os.environ['JAVA_HOME']))
else:
    logger.info("Installing Java")
    install_java()


# download Hadoop
file_name = os.path.basename(HADOOP_URL)
if os.path.isfile(file_name):
   logger.info("{} already exists, not downloading".format(file_name))
else:
  logger.info("Downloading {}".format(file_name))
  urllib.request.urlretrieve(HADOOP_URL, file_name)

# uncompress archive
dir_name = file_name[:-7]
if os.path.exists(dir_name):
  logger.info("{} is already uncompressed".format(file_name))
else:
  logger.info("Uncompressing {}".format(file_name))
  tar = tarfile.open(file_name)
  tar.extractall()
  tar.close()

# environment variables
os.environ['HADOOP_HOME'] = os.path.join(os.getcwd(), dir_name)
logger.info("HADOOP_HOME is {}".format(os.environ['HADOOP_HOME']))
os.environ['PATH'] = ':'.join([os.path.join(os.environ['HADOOP_HOME'], 'bin'), os.environ['PATH']])
logger.info("PATH is {}".format(os.environ['PATH']))


!mapred -h

!mapred envvars

os.environ['HADOOP_TOOLS_LIB_JARS_DIR'] = os.path.join(os.environ['HADOOP_HOME'], 'share/hadoop/tools/lib/') #IMPORTANT

!find hadoop-3.4.0 -name "mockito*"

!wget --no-clobber https://repo1.maven.org/maven2/org/mockito/mockito-core/2.28.2/mockito-core-2.28.2.jar

shutil.copy('mockito-core-2.28.2.jar', os.path.join(os.environ['HADOOP_HOME'],'share/hadoop/mapreduce/'))

os.listdir(os.path.join(os.environ['HADOOP_HOME'],'share/hadoop/mapreduce/'))

!mkdir -p ./target/test/data/dfs/{name-0-1,name-0-2}

!ls ./target/test/data/dfs/

!mapred minicluster -help

# check if the file is there
!find $HADOOP_HOME -name "core-site.xml"

# view the contents of the file
!cat $(find $HADOOP_HOME -name "core-site.xml")

with open(os.environ['HADOOP_HOME']+'/etc/hadoop/core-site.xml', 'w') as file:
  file.write("<configuration>\n</configuration>")

!cat $(find $HADOOP_HOME -name "core-site.xml")

#!mapred minicluster -format

!lsof -n -i -P +c0 -sTCP:LISTEN

import subprocess
with open('out.txt', "w") as stdout_file, open('err.txt', "w") as stderr_file:
    process = subprocess.Popen(
        ["mapred", "minicluster", "-format"],
        stdout=stdout_file,
        stderr=stderr_file
    )

if not IN_COLAB:
  time.sleep(30)
else:
  time.sleep(10)

!lsof -n -i -P +c0 -sTCP:LISTEN

!wget http://localhost:8088

if IN_COLAB:
    # serve the Web UI on Colab
    print("Click on the link below to open the Resource Manager Web UI 🚀")
    output.serve_kernel_port_as_window(8088, path='/node')

!wget http://localhost:19888

if IN_COLAB:
    # serve the Web UI on Colab
    print("Click on the link below to open the MapReduce JobHistory Server Web UI 🚀")
    output.serve_kernel_port_as_window(19888, path='/node')

# you should set this to True
NGROK = False #@param {type:"boolean"}

if NGROK:
  !pip install pyngrok
  from pyngrok import ngrok, conf
  import getpass

  print("Enter your authtoken, which can be copied from https://dashboard.ngrok.com/get-started/your-authtoken")
  conf.get_default().auth_token = getpass.getpass()

if NGROK:
  # Open a ngrok tunnel to the HTTP server
  public_url = ngrok.connect(19888).public_url

if NGROK:
  print(f'Click on {public_url} to open the MapReduce JobHistory Server Web UI')

process.kill()

!lsof -n -i -P +c0 -sTCP:LISTEN

!pkill -f java

import subprocess
with open('out.txt', "w") as stdout_file, open('err.txt', "w") as stderr_file:
    process = subprocess.Popen(
        ["mapred", "minicluster", "-format", "-jhsport", "8900", "-nnhttpport", "8901", "-nnport", "8902", "-rmport", "8903"],
        stdout=stdout_file,
        stderr=stderr_file
    )

if not IN_COLAB:
  time.sleep(30)
else:
  time.sleep(10)

!lsof -n -i -P +c0 -sTCP:LISTEN | grep "^COMMAND\|java"

!tail err.txt

!wget http://localhost:8901

if IN_COLAB and not NGROK:
    # serve the Web UI on Colab
    print("Click on the link below to open the NameNode Web UI 🚀")
    output.serve_kernel_port_as_window(8901, path='/index.html')
else:
  if NGROK:
    # disconnect previous tunnels (note: you can have max 3 tunnels open!)
    # see: https://pyngrok.readthedocs.io/en/latest/index.html#get-active-tunnels
    tunnels = ngrok.get_tunnels()
    for t in tunnels:
      ngrok.disconnect(t.public_url)
    # Open a ngrok tunnel to the HTTP server on port 8901
    public_url = ngrok.connect(8901).public_url
    print(f'Click on {public_url} to open the NameNode Web UI 🚀')

%%bash
# create a folder my_dir
hdfs dfs -mkdir hdfs://localhost:8902/my_dir

!hdfs dfs -ls hdfs://localhost:8902/my_dir

!ls -lh sample_data

!du -h sample_data

!hdfs dfs -put sample_data hdfs://localhost:8902/my_dir/

!hdfs dfs -ls hdfs://localhost:8902/my_dir

!hdfs dfs -du -h hdfs://localhost:8902/my_dir

!hdfs dfs -ls -R -h hdfs://localhost:8902/my_dir

!hdfs dfs -rm -r hdfs://localhost:8902/my_dir

!hdfs dfs -fs hdfs://localhost:8902/ -ls /

!hdfs dfs -ls hdfs://localhost:8902/

!hdfs dfs -D fs.defaultFS=hdfs://localhost:8902/ -ls /

with open(os.environ['HADOOP_HOME']+'/etc/hadoop/core-site.xml', 'w') as file:
  file.write("""<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost:8902/</value>
  </property>
</configuration>""")

!cat $HADOOP_HOME/etc/hadoop/core-site.xml

!hdfs dfs -ls /

with open(os.environ['HADOOP_HOME']+'/etc/hadoop/core-site.xml', 'w') as file:
  file.write("""
<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>file:///</value>
  </property>
</configuration>""")

!hdfs dfs -ls /

!hdfs dfsadmin -h

!hdfs dfsadmin -fs hdfs://localhost:8902/ -report

!find . -name "*examples*.jar"

!lsof -n -i -P +c0 -sTCP:LISTEN -ac java

# @title
from IPython.core.display import HTML
HTML("""
<div style="background-color:rgb(16, 163, 127,.2);border:2px solid rgb(16, 163, 127,.3);padding:3px;">
<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 320"  style="width:32px;height:32px;">
 <g fill="currentColor">
  <path d="m297.06 130.97c7.26-21.79 4.76-45.66-6.85-65.48-17.46-30.4-52.56-46.04-86.84-38.68-15.25-17.18-37.16-26.95-60.13-26.81-35.04-.08-66.13 22.48-76.91 55.82-22.51 4.61-41.94 18.7-53.31 38.67-17.59 30.32-13.58 68.54 9.92 94.54-7.26 21.79-4.76 45.66 6.85 65.48 17.46 30.4 52.56 46.04 86.84 38.68 15.24 17.18 37.16 26.95 60.13 26.8 35.06.09 66.16-22.49 76.94-55.86 22.51-4.61 41.94-18.7 53.31-38.67 17.57-30.32 13.55-68.51-9.94-94.51zm-120.28 168.11c-14.03.02-27.62-4.89-38.39-13.88.49-.26 1.34-.73 1.89-1.07l63.72-36.8c3.26-1.85 5.26-5.32 5.24-9.07v-89.83l26.93 15.55c.29.14.48.42.52.74v74.39c-.04 33.08-26.83 59.9-59.91 59.97zm-128.84-55.03c-7.03-12.14-9.56-26.37-7.15-40.18.47.28 1.3.79 1.89 1.13l63.72 36.8c3.23 1.89 7.23 1.89 10.47 0l77.79-44.92v31.1c.02.32-.13.63-.38.83l-64.41 37.19c-28.69 16.52-65.33 6.7-81.92-21.95zm-16.77-139.09c7-12.16 18.05-21.46 31.21-26.29 0 .55-.03 1.52-.03 2.2v73.61c-.02 3.74 1.98 7.21 5.23 9.06l77.79 44.91-26.93 15.55c-.27.18-.61.21-.91.08l-64.42-37.22c-28.63-16.58-38.45-53.21-21.95-81.89zm221.26 51.49-77.79-44.92 26.93-15.54c.27-.18.61-.21.91-.08l64.42 37.19c28.68 16.57 38.51 53.26 21.94 81.94-7.01 12.14-18.05 21.44-31.2 26.28v-75.81c.03-3.74-1.96-7.2-5.2-9.06zm26.8-40.34c-.47-.29-1.3-.79-1.89-1.13l-63.72-36.8c-3.23-1.89-7.23-1.89-10.47 0l-77.79 44.92v-31.1c-.02-.32.13-.63.38-.83l64.41-37.16c28.69-16.55 65.37-6.7 81.91 22 6.99 12.12 9.52 26.31 7.15 40.1zm-168.51 55.43-26.94-15.55c-.29-.14-.48-.42-.52-.74v-74.39c.02-33.12 26.89-59.96 60.01-59.94 14.01 0 27.57 4.92 38.34 13.88-.49.26-1.33.73-1.89 1.07l-63.72 36.8c-3.26 1.85-5.26 5.31-5.24 9.06l-.04 89.79zm14.63-31.54 34.65-20.01 34.65 20v40.01l-34.65 20-34.65-20z"></path>
  </svg>


<p>
In the realm of <span style="color: #00f;">networks</span>, where processes twine,<br>
A <code>command</code> unfolds, a <em>symphony</em> of lines.<br>
"<strong>Lsof</strong>," it whispers, with a mystic hum,<br>
A dance of flags, each one has its own drum.<br>
<p>
"<code>-n -i -P</code>," the conductor commands,<br>
Navigate swiftly, across distant lands.<br>
"<code>+c0</code>" echoes softly, a chorus of glee,<br>
Embrace all processes, as far as eyes can see.<br>
<p>
"<code>-sTCP:LISTEN</code>," a stanza profound,<br>
Seeking the echoes of ports, a network's sound.<br>
Processes in repose, in a state so keen,<br>
A tapestry of <span style="font-style: italic;">LISTEN</span>, a poetic scene.<br>
</div>
""")

!yarn jar ./hadoop-3.4.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.4.0.jar

!yarn jar ./hadoop-3.4.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.4.0.jar pi

!yarn jar ./hadoop-3.4.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.4.0.jar pi \
       5 1000

!yarn application -D yarn.resourcemanager.address=localhost:8903 -list -appStates ALL

file_mapred_site = os.path.join(os.environ['HADOOP_HOME'],'etc/hadoop/mapred-site.xml')
file_core_site = os.path.join(os.environ['HADOOP_HOME'],'etc/hadoop/core-site.xml')
file_yarn_site = os.path.join(os.environ['HADOOP_HOME'],'etc/hadoop/yarn-site.xml')

%%bash
cat > $HADOOP_HOME/'etc/hadoop/mapred-site.xml' << 🐸
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*</value>
    </property>
</configuration>
🐸

!cat $HADOOP_HOME/'etc/hadoop/mapred-site.xml'

with open(file_core_site, 'w') as file:
  file.write("""
<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost:8902/</value>
  </property>
</configuration>""")

with open(file_yarn_site, 'w') as file:
 file.write("""
<configuration>
<property>
    <name>yarn.resourcemanager.address</name>
    <value>localhost:8903</value>
</property>
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>
</configuration>""")


process.kill()
!pkill -f java # kill java processes

with open('out.txt', "w") as stdout_file, open('err.txt', "w") as stderr_file:
    process = subprocess.Popen(
        ["mapred", "minicluster", "-format", "-jhsport", "8900", "-nnhttpport", "8901", "-nnport", "8902", "-rmport", "8903"],
        stdout=stdout_file,
        stderr=stderr_file
    )

for att in range(10):
    with open('err.txt') as myfile:
         if 'Started MiniMRCluster' in myfile.read():
             print('MiniCluster is up and running')
             break
         else:
             time.sleep(2)


!lsof -n -i -P +c0 -sTCP:LISTEN -ac java

!yarn jar ./hadoop-3.4.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.4.0.jar pi \
       5 1000

!yarn application -list -appStates ALL

with open('job_out.txt', "w") as stdout_file, open('job_err.txt', "w") as stderr_file:
    process = subprocess.Popen(
        ["yarn", "jar", "./hadoop-3.4.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.4.0.jar", "pi",
         "-D", "localhost:8903",
         "50", "1000000"],
        stdout=stdout_file,
        stderr=stderr_file
    )

time.sleep(10)

!yarn application -list -appStates ALL

!yarn application -list -appStates ALL

!yarn application -list -appStates FINISHED 2>/dev/null|grep SUCCEEDED|tail -1| cut -f1

%%bash
app_id=$(yarn application -list -appStates FINISHED 2>/dev/null|grep SUCCEEDED|tail -1| cut -f1)
yarn logs -applicationId $app_id