Input → InputFormat → RecordReader → Mapper → Partition → Spill → Sort → Shuffle → Merge → Reducer → OutputFormat → HDFS

// WordCount.java – Compile & run this today!
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {
    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        public void map(Object key, Text value, Context context) throws Exception {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while (itr.hasMoreTokens()) {
                word.set(itr.nextToken().replaceAll("[^a-zA-Z]", "").toLowerCase());
                if (!word.toString().isEmpty())
                    context.write(word, one);
            }
        }
    }

    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws Exception {
            int sum = 0;
            for (IntWritable val : values) sum += val.get();
            result.set(sum);
            context.write(key, result);
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "word count");
        job.setJarByClass(WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);   // ← saves network!
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

hadoop com.sun.tools.javac.Main WordCount.java
jar cf wc.jar WordCount*.class
hadoop jar wc.jar WordCount /input/shakespeare.txt /output/wc_2025

# mapper.py
#!/usr/bin/env python3
import sys
for line in sys.stdin:
    for word in line.strip().split():
        print(f"{word.lower()}\t1")

# reducer.py
#!/usr/bin/env python3
import sys
current_word = None
count = 0
for line in sys.stdin:
    word, cnt = line.strip().split('\t')
    if current_word == word:
        count += int(cnt)
    else:
        if current_word:
            print(f"{current_word}\t{count}")
        current_word = word
        count = int(cnt)
if current_word:
    print(f"{current_word}\t{count}")

hadoop jar $HADOOP_HOME/share/hadoop/tools/lib/hadoop-streaming-*.jar \
    -files mapper.py,reducer.py \
    -mapper mapper.py -reducer reducer.py \
    -input /data/books/* -output /output/wc_python

// Maven + MRUnit test
@Test
public void testMapper() {
    Text value = new Text("hadoop hadoop spark");
    new MapDriver<Object, Text, Text, IntWritable>()
        .withMapper(new TokenizerMapper())
        .withInput(new Object(), value)
        .withOutput(new Text("hadoop"), new IntWritable(1))
        .withOutput(new Text("hadoop"), new IntWritable(1))
        .withOutput(new Text("spark"), new IntWritable(1))
        .runTest();
}

Client → YARN ResourceManager → ApplicationMaster → Container (Mapper/Reducer)
                    ↓
              Task Attempt (with JVM reuse)
                    ↓
         Shuffle: Copy → Sort → Merge → Reduce input

# Option 1 – Fastest (Docker – 2025)
docker run -it --name hadoop-lab -p 9870:9870 -p 8088:8088 sequenceiq/hadoop-docker:3.3.6 /etc/bootstrap.sh -bash

# Inside container
hdfs dfs -mkdir /data
hdfs dfs -put /usr/local/hadoop/README.txt /data/
hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar wordcount /data/README.txt /output

# Option 2 – Cloud (Free)
https://labs.play-with-hadoop.com → pre-built cluster, login as “training”