Hadoop: Map Reduce: count number of lines in a file

Step 1: Following is the application that counts number of lines in a file.

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class LinesCount {
 public static class LineCountMapper extends
   Mapper<Object, Text, Text, IntWritable> {

  private final static IntWritable one = new IntWritable(1);
  private Text word = new Text("Total Lines");

  public void map(Object key, Text value, Context context)
    throws IOException, InterruptedException {
   context.write(word, one);
  }
 }

 public static class IntSumReducer extends
   Reducer<Text, IntWritable, Text, IntWritable> {
  private IntWritable result = new IntWritable();

  public void reduce(Text key, Iterable<IntWritable> values,
    Context context) throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
    sum += val.get();
   }
   result.set(sum);
   context.write(key, result);
  }
 }

 public static void main(String[] args) throws Exception {
  Configuration conf = new Configuration();
  Job job = Job.getInstance(conf, "lines count");
  job.setJarByClass(WordCount.class);
  job.setMapperClass(LineCountMapper.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  FileInputFormat.addInputPath(job, new Path(args[0]));
  FileOutputFormat.setOutputPath(job, new Path(args[1]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
 }
}

Step2: Compile above java file.

$ hadoop com.sun.tools.javac.Main LinesCount.java

Step 3: Create jar file

$ jar cf linecount.jar LinesCount*class

Step 4: Run jar file.

hadoop jar linecount.jar LinesCount /user/harikrishna_gurram/input.txt  /user/harikrishna_gurram/results

Open “/user/harikrishna_gurram/results” directory, you can see two files.

$ hadoop fs -ls /user/harikrishna_gurram/results

Found 2 items

-rw-r--r--   3 harikrishna_gurram supergroup          0 2015-06-23 09:30 /user/harikrishna_gurram/results/_SUCCESS

-rw-r--r--   3 harikrishna_gurram supergroup         14 2015-06-23 09:30 /user/harikrishna_gurram/results/part-r-00000

Open “part-r-00000” file, you can see the number of lines of given input file.

$ hadoop fs -cat /user/harikrishna_gurram/results/part-r-00000

Total Lines 999453

Previous                                                 Next                                                 Home