thanoojgithub/Hadoop3.2.2_Start-up_commands.txt

## Hadoop3.2.2_Start-up_commands.txt
Hadoop 3.2.2
Start-up commands:
--------------------------------
1. Stop the dfs and yarn first.
2. Remove the datanode and namenode directories as specified in the core-site.xml file.
3. Re-create the directories.
4. hdfs namenode -format
5. Then re-start the dfs and the yarn as follows.
start-dfs.sh
start-yarn.sh
6. mapred --daemon start historyserver


- Check SSH servicee is up and running :
huser@thanoojWin10Home:~/tmpdata$ sudo service ssh status
[sudo] password for huser:
 * sshd is running
huser@thanoojWin10Home:~/tmpdata$

- If not restart
thanooj@thanoojWin10Home:~$ sudo service ssh restart
[sudo] password for thanooj:
 * Restarting OpenBSD Secure Shell server sshd                                                                                               [ OK ]
thanooj@thanoojWin10Home:~$ su - huser
Password:
huser@thanoojWin10Home:~$


huser@thanoojWin10Home:~$ ls -ltr
-rw-rw-r--  1 huser huser 395448622 Jan 14  2021 hadoop-3.2.2.tar.gz
drwxr-xr-x 10 huser huser      4096 Mar 20 15:34 hadoop-3.2.2
drwxrwxr-x  4 huser huser      4096 Mar 21 09:47 dfsdata
drwxrwxr-x  4 huser huser      4096 Mar 21 09:48 tmpdata
huser@thanoojWin10Home:~/dfsdata$ ls -ltr
drwxrwxr-x 2 huser huser 4096 Mar 21 09:47 namenode
drwx------ 3 huser huser 4096 Mar 21 09:56 datanode
huser@thanoojWin10Home:~/tmpdata$ ls -ltr
drwxrwxr-x 4 huser huser 4096 Mar 21 09:55 dfs
drwxr-xr-x 5 huser huser 4096 Mar 21 09:56 nm-local-dir
huser@thanoojWin10Home:~/tmpdata$

ref:
https://hadoop.apache.org/docs/r3.1.1/hadoop-project-dist/hadoop-common/ClusterSetup.html


huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ mkdir wordcountSample
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ cd wordcountSample/
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi WordCount.java
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat WordCount.java
import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

  public static class TokenizerMapper
       extends Mapper<Object, Text, Text, IntWritable>{

    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();

    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
      StringTokenizer itr = new StringTokenizer(value.toString());
      while (itr.hasMoreTokens()) {
        word.set(itr.nextToken());
        context.write(word, one);
      }
    }
  }

  public static class IntSumReducer
       extends Reducer<Text,IntWritable,Text,IntWritable> {
    private IntWritable result = new IntWritable();

    public void reduce(Text key, Iterable<IntWritable> values,
                       Context context
                       ) throws IOException, InterruptedException {
      int sum = 0;
      for (IntWritable val : values) {
        sum += val.get();
      }
      result.set(sum);
      context.write(key, result);
    }
  }

  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "word count");
    job.setJarByClass(WordCount.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

Hello World Bye World
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);

Hello Hadoop Goodbye Hadoop
  }
}
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
total 4
-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop com.sun.tools.javac.Main WordCount.java
/home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2366: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_USER: invalid variable name
/home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2461: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_OPTS: invalid variable name
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ jar cf wc.jar WordCount*.class
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
total 20
-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21  WordCount.java
-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22  WordCount.class
-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22  wc.jar
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /
Found 1 items
drwxrwx---   - huser supergroup          0 2022-03-21 10:07 /tmp
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/input
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/output
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
Found 2 items
drwxr-xr-x   - huser supergroup          0 2022-03-21 10:24 /huser/data/wc/one/input
drwxr-xr-x   - huser supergroup          0 2022-03-21 10:24 /huser/data/wc/one/output
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file01.txt
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file01.txt
Hello World Bye World
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file02.txt
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file02.txt
Hello Hadoop Goodbye Hadoop
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
total 28
-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21  WordCount.java
-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22  WordCount.class
-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22  wc.jar
-rw-rw-r-- 1 huser huser   22 Mar 21 10:26  file01.txt
-rw-rw-r-- 1 huser huser   28 Mar 21 10:26  file02.txt

huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcoun
tSample/file01.txt /huser/data/wc/one/input/
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample/file02.txt /huser/data/wc/one/input/
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/input
Found 2 items
-rw-r--r--   1 huser supergroup         22 2022-03-21 10:29 /huser/data/wc/one/input/file01.txt
-rw-r--r--   1 huser supergroup         28 2022-03-21 10:29 /huser/data/wc/one/input/file02.txt
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$


/home/huser/hadoop-3.2.2

huser@thanoojWin10Home:~/hadoop-3.2.2/etc/hadoop$ cat mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
  <name>mapreduce.framework.name</name>
  <value>yarn</value>
</property>
<property>
  <name>yarn.app.mapreduce.am.env</name>
  <value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
</property>
<property>
  <name>mapreduce.map.env</name>
  <value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
</property>
<property>
  <name>mapreduce.reduce.env</name>
  <value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
</property>
</configuration>


huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -rm -r -f /huser/data/wc/one/output
Deleted /huser/data/wc/one/output
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
Found 1 items
drwxr-xr-x   - huser supergroup          0 2022-03-21 10:29 /huser/data/wc/one/input
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$


huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
total 28
-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21  WordCount.java
-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22  WordCount.class
-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22  wc.jar
-rw-rw-r-- 1 huser huser   22 Mar 21 10:26  file01.txt
-rw-rw-r-- 1 huser huser   28 Mar 21 10:26  file02.txt
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop jar wc.jar WordCount /huser/data/wc/one/input /huser/data/wc/one/output
2022-03-21 10:41:32,060 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
2022-03-21 10:41:32,655 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
2022-03-21 10:41:32,692 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/huser/.staging/job_1647836809874_0002
2022-03-21 10:41:33,064 INFO input.FileInputFormat: Total input files to process : 2
2022-03-21 10:41:33,190 INFO mapreduce.JobSubmitter: number of splits:2
2022-03-21 10:41:33,634 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1647836809874_0002
2022-03-21 10:41:33,637 INFO mapreduce.JobSubmitter: Executing with tokens: []
2022-03-21 10:41:33,933 INFO conf.Configuration: resource-types.xml not found
2022-03-21 10:41:33,934 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.
2022-03-21 10:41:34,052 INFO impl.YarnClientImpl: Submitted application application_1647836809874_0002
2022-03-21 10:41:34,116 INFO mapreduce.Job: The url to track the job: http://thanoojWin10Home.localdomain:8088/proxy/application_1647836809874_0002/
2022-03-21 10:41:34,118 INFO mapreduce.Job: Running job: job_1647836809874_0002
2022-03-21 10:41:44,438 INFO mapreduce.Job: Job job_1647836809874_0002 running in uber mode : false
2022-03-21 10:41:44,444 INFO mapreduce.Job:  map 0% reduce 0%
2022-03-21 10:41:53,673 INFO mapreduce.Job:  map 100% reduce 0%
2022-03-21 10:42:01,783 INFO mapreduce.Job:  map 100% reduce 100%
2022-03-21 10:42:01,805 INFO mapreduce.Job: Job job_1647836809874_0002 completed successfully
2022-03-21 10:42:01,997 INFO mapreduce.Job: Counters: 54
        File System Counters
                FILE: Number of bytes read=79
                FILE: Number of bytes written=704452
                FILE: Number of read operations=0
                FILE: Number of large read operations=0
                FILE: Number of write operations=0
                HDFS: Number of bytes read=292
                HDFS: Number of bytes written=41
                HDFS: Number of read operations=11
                HDFS: Number of large read operations=0
                HDFS: Number of write operations=2
                HDFS: Number of bytes read erasure-coded=0
        Job Counters
                Launched map tasks=2
                Launched reduce tasks=1
                Data-local map tasks=2
                Total time spent by all maps in occupied slots (ms)=13631
                Total time spent by all reduces in occupied slots (ms)=4545
                Total time spent by all map tasks (ms)=13631
                Total time spent by all reduce tasks (ms)=4545
                Total vcore-milliseconds taken by all map tasks=13631
                Total vcore-milliseconds taken by all reduce tasks=4545
                Total megabyte-milliseconds taken by all map tasks=13958144
                Total megabyte-milliseconds taken by all reduce tasks=4654080
        Map-Reduce Framework
                Map input records=2
                Map output records=8
                Map output bytes=82
                Map output materialized bytes=85
                Input split bytes=242
                Combine input records=8
                Combine output records=6
                Reduce input groups=5
                Reduce shuffle bytes=85
                Reduce input records=6
                Reduce output records=5
                Spilled Records=12
                Shuffled Maps =2
                Failed Shuffles=0
                Merged Map outputs=2
                GC time elapsed (ms)=171
                CPU time spent (ms)=2510
                Physical memory (bytes) snapshot=785104896
                Virtual memory (bytes) snapshot=8188284928
                Total committed heap usage (bytes)=625999872
                Peak Map Physical memory (bytes)=291561472
                Peak Map Virtual memory (bytes)=2730262528
                Peak Reduce Physical memory (bytes)=213037056
                Peak Reduce Virtual memory (bytes)=2727878656
        Shuffle Errors
                BAD_ID=0
                CONNECTION=0
                IO_ERROR=0
                WRONG_LENGTH=0
                WRONG_MAP=0
                WRONG_REDUCE=0
        File Input Format Counters
                Bytes Read=50
        File Output Format Counters
                Bytes Written=41
huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$
	Hadoop 3.2.2
	Start-up commands:
	--------------------------------
	1. Stop the dfs and yarn first.
	2. Remove the datanode and namenode directories as specified in the core-site.xml file.
	3. Re-create the directories.
	4. hdfs namenode -format
	5. Then re-start the dfs and the yarn as follows.
	start-dfs.sh
	start-yarn.sh
	6. mapred --daemon start historyserver




	- Check SSH servicee is up and running :
	huser@thanoojWin10Home:~/tmpdata$ sudo service ssh status
	[sudo] password for huser:
	* sshd is running
	huser@thanoojWin10Home:~/tmpdata$

	- If not restart
	thanooj@thanoojWin10Home:~$ sudo service ssh restart
	[sudo] password for thanooj:
	* Restarting OpenBSD Secure Shell server sshd [ OK ]
	thanooj@thanoojWin10Home:~$ su - huser
	Password:
	huser@thanoojWin10Home:~$


	huser@thanoojWin10Home:~$ ls -ltr
	-rw-rw-r-- 1 huser huser 395448622 Jan 14 2021 hadoop-3.2.2.tar.gz
	drwxr-xr-x 10 huser huser 4096 Mar 20 15:34 hadoop-3.2.2
	drwxrwxr-x 4 huser huser 4096 Mar 21 09:47 dfsdata
	drwxrwxr-x 4 huser huser 4096 Mar 21 09:48 tmpdata
	huser@thanoojWin10Home:~/dfsdata$ ls -ltr
	drwxrwxr-x 2 huser huser 4096 Mar 21 09:47 namenode
	drwx------ 3 huser huser 4096 Mar 21 09:56 datanode
	huser@thanoojWin10Home:~/tmpdata$ ls -ltr
	drwxrwxr-x 4 huser huser 4096 Mar 21 09:55 dfs
	drwxr-xr-x 5 huser huser 4096 Mar 21 09:56 nm-local-dir
	huser@thanoojWin10Home:~/tmpdata$

	ref:
	https://hadoop.apache.org/docs/r3.1.1/hadoop-project-dist/hadoop-common/ClusterSetup.html











	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ mkdir wordcountSample
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ cd wordcountSample/
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi WordCount.java
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat WordCount.java
	import java.io.IOException;
	import java.util.StringTokenizer;

	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.io.IntWritable;
	import org.apache.hadoop.io.Text;
	import org.apache.hadoop.mapreduce.Job;
	import org.apache.hadoop.mapreduce.Mapper;
	import org.apache.hadoop.mapreduce.Reducer;
	import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
	import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

	public class WordCount {

	public static class TokenizerMapper
	extends Mapper<Object, Text, Text, IntWritable>{

	private final static IntWritable one = new IntWritable(1);
	private Text word = new Text();

	public void map(Object key, Text value, Context context
	) throws IOException, InterruptedException {
	StringTokenizer itr = new StringTokenizer(value.toString());
	while (itr.hasMoreTokens()) {
	word.set(itr.nextToken());
	context.write(word, one);
	}
	}
	}

	public static class IntSumReducer
	extends Reducer<Text,IntWritable,Text,IntWritable> {
	private IntWritable result = new IntWritable();

	public void reduce(Text key, Iterable<IntWritable> values,
	Context context
	) throws IOException, InterruptedException {
	int sum = 0;
	for (IntWritable val : values) {
	sum += val.get();
	}
	result.set(sum);
	context.write(key, result);
	}
	}

	public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = Job.getInstance(conf, "word count");
	job.setJarByClass(WordCount.class);
	job.setMapperClass(TokenizerMapper.class);
	job.setCombinerClass(IntSumReducer.class);
	job.setReducerClass(IntSumReducer.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(IntWritable.class);

	Hello World Bye World
	FileInputFormat.addInputPath(job, new Path(args[0]));
	FileOutputFormat.setOutputPath(job, new Path(args[1]));
	System.exit(job.waitForCompletion(true) ? 0 : 1);

	Hello Hadoop Goodbye Hadoop
	}
	}
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 4
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop com.sun.tools.javac.Main WordCount.java
	/home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2366: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_USER: invalid variable name
	/home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2461: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_OPTS: invalid variable name
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ jar cf wc.jar WordCount*.class
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 20
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
	-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22 WordCount.class
	-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
	-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22 wc.jar
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /
	Found 1 items
	drwxrwx--- - huser supergroup 0 2022-03-21 10:07 /tmp
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/input
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/output
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
	Found 2 items
	drwxr-xr-x - huser supergroup 0 2022-03-21 10:24 /huser/data/wc/one/input
	drwxr-xr-x - huser supergroup 0 2022-03-21 10:24 /huser/data/wc/one/output
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file01.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file01.txt
	Hello World Bye World
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file02.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file02.txt
	Hello Hadoop Goodbye Hadoop
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 28
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
	-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22 WordCount.class
	-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
	-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22 wc.jar
	-rw-rw-r-- 1 huser huser 22 Mar 21 10:26 file01.txt
	-rw-rw-r-- 1 huser huser 28 Mar 21 10:26 file02.txt

	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcoun
	tSample/file01.txt /huser/data/wc/one/input/
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample/file02.txt /huser/data/wc/one/input/
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/input
	Found 2 items
	-rw-r--r-- 1 huser supergroup 22 2022-03-21 10:29 /huser/data/wc/one/input/file01.txt
	-rw-r--r-- 1 huser supergroup 28 2022-03-21 10:29 /huser/data/wc/one/input/file02.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$




	/home/huser/hadoop-3.2.2

	huser@thanoojWin10Home:~/hadoop-3.2.2/etc/hadoop$ cat mapred-site.xml
	<?xml version="1.0"?>
	<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
	<!--
	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License. See accompanying LICENSE file.
	-->

	<!-- Put site-specific property overrides in this file. -->

	<configuration>
	<property>
	<name>mapreduce.framework.name</name>
	<value>yarn</value>
	</property>
	<property>
	<name>yarn.app.mapreduce.am.env</name>
	<value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
	</property>
	<property>
	<name>mapreduce.map.env</name>
	<value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
	</property>
	<property>
	<name>mapreduce.reduce.env</name>
	<value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
	</property>
	</configuration>


	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -rm -r -f /huser/data/wc/one/output
	Deleted /huser/data/wc/one/output
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
	Found 1 items
	drwxr-xr-x - huser supergroup 0 2022-03-21 10:29 /huser/data/wc/one/input
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$


	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 28
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
	-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22 WordCount.class
	-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
	-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22 wc.jar
	-rw-rw-r-- 1 huser huser 22 Mar 21 10:26 file01.txt
	-rw-rw-r-- 1 huser huser 28 Mar 21 10:26 file02.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop jar wc.jar WordCount /huser/data/wc/one/input /huser/data/wc/one/output
	2022-03-21 10:41:32,060 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
	2022-03-21 10:41:32,655 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
	2022-03-21 10:41:32,692 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/huser/.staging/job_1647836809874_0002
	2022-03-21 10:41:33,064 INFO input.FileInputFormat: Total input files to process : 2
	2022-03-21 10:41:33,190 INFO mapreduce.JobSubmitter: number of splits:2
	2022-03-21 10:41:33,634 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1647836809874_0002
	2022-03-21 10:41:33,637 INFO mapreduce.JobSubmitter: Executing with tokens: []
	2022-03-21 10:41:33,933 INFO conf.Configuration: resource-types.xml not found
	2022-03-21 10:41:33,934 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.
	2022-03-21 10:41:34,052 INFO impl.YarnClientImpl: Submitted application application_1647836809874_0002
	2022-03-21 10:41:34,116 INFO mapreduce.Job: The url to track the job: http://thanoojWin10Home.localdomain:8088/proxy/application_1647836809874_0002/
	2022-03-21 10:41:34,118 INFO mapreduce.Job: Running job: job_1647836809874_0002
	2022-03-21 10:41:44,438 INFO mapreduce.Job: Job job_1647836809874_0002 running in uber mode : false
	2022-03-21 10:41:44,444 INFO mapreduce.Job: map 0% reduce 0%
	2022-03-21 10:41:53,673 INFO mapreduce.Job: map 100% reduce 0%
	2022-03-21 10:42:01,783 INFO mapreduce.Job: map 100% reduce 100%
	2022-03-21 10:42:01,805 INFO mapreduce.Job: Job job_1647836809874_0002 completed successfully
	2022-03-21 10:42:01,997 INFO mapreduce.Job: Counters: 54
	File System Counters
	FILE: Number of bytes read=79
	FILE: Number of bytes written=704452
	FILE: Number of read operations=0
	FILE: Number of large read operations=0
	FILE: Number of write operations=0
	HDFS: Number of bytes read=292
	HDFS: Number of bytes written=41
	HDFS: Number of read operations=11
	HDFS: Number of large read operations=0
	HDFS: Number of write operations=2
	HDFS: Number of bytes read erasure-coded=0
	Job Counters
	Launched map tasks=2
	Launched reduce tasks=1
	Data-local map tasks=2
	Total time spent by all maps in occupied slots (ms)=13631
	Total time spent by all reduces in occupied slots (ms)=4545
	Total time spent by all map tasks (ms)=13631
	Total time spent by all reduce tasks (ms)=4545
	Total vcore-milliseconds taken by all map tasks=13631
	Total vcore-milliseconds taken by all reduce tasks=4545
	Total megabyte-milliseconds taken by all map tasks=13958144
	Total megabyte-milliseconds taken by all reduce tasks=4654080
	Map-Reduce Framework
	Map input records=2
	Map output records=8
	Map output bytes=82
	Map output materialized bytes=85
	Input split bytes=242
	Combine input records=8
	Combine output records=6
	Reduce input groups=5
	Reduce shuffle bytes=85
	Reduce input records=6
	Reduce output records=5
	Spilled Records=12
	Shuffled Maps =2
	Failed Shuffles=0
	Merged Map outputs=2
	GC time elapsed (ms)=171
	CPU time spent (ms)=2510
	Physical memory (bytes) snapshot=785104896
	Virtual memory (bytes) snapshot=8188284928
	Total committed heap usage (bytes)=625999872
	Peak Map Physical memory (bytes)=291561472
	Peak Map Virtual memory (bytes)=2730262528
	Peak Reduce Physical memory (bytes)=213037056
	Peak Reduce Virtual memory (bytes)=2727878656
	Shuffle Errors
	BAD_ID=0
	CONNECTION=0
	IO_ERROR=0
	WRONG_LENGTH=0
	WRONG_MAP=0
	WRONG_REDUCE=0
	File Input Format Counters
	Bytes Read=50
	File Output Format Counters
	Bytes Written=41
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$