Hadoop: Map Reduce: count number of characters in a file

Step 1: Following is the application that counts number of lines in a file.

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class CharacterCount {
 public static class CharacterCountMapper extends
   Mapper<Object, Text, Text, IntWritable> {

  private Text word = new Text("Total Characters in file are ");

  public void map(Object key, Text value, Context context)
    throws IOException, InterruptedException {
   context.write(word, new IntWritable(value.getLength()));
  }
 }

 public static class IntSumReducer extends
   Reducer<Text, IntWritable, Text, IntWritable> {
  private IntWritable result = new IntWritable();

  public void reduce(Text key, Iterable<IntWritable> values,
    Context context) throws IOException, InterruptedException {
   int sum = 0;
   for (IntWritable val : values) {
    sum += val.get();
   }
   result.set(sum);
   context.write(key, result);
  }
 }

 public static void main(String[] args) throws Exception {
  Configuration conf = new Configuration();
  Job job = Job.getInstance(conf, "Character count");
  job.setJarByClass(CharacterCount.class);
  job.setMapperClass(CharacterCountMapper.class);
  job.setCombinerClass(IntSumReducer.class);
  job.setReducerClass(IntSumReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(IntWritable.class);
  FileInputFormat.addInputPath(job, new Path(args[0]));
  FileOutputFormat.setOutputPath(job, new Path(args[1]));
  System.exit(job.waitForCompletion(true) ? 0 : 1);
 }
}

Step2: Compile above java file.

$ hadoop com.sun.tools.javac.Main CharacterCount.java

Step 3: Create jar file

$ jar cf charcount.jar CharacterCount*class

Step 4: Run jar file.

hadoop jar charcount.jar CharacterCount /user/harikrishna_gurram/input.txt  /user/harikrishna_gurram/results1

Open “/user/harikrishna_gurram/results1” directory, you can see two files.

$ hadoop fs -ls /user/harikrishna_gurram/results1

Found 2 items

-rw-r--r--   3 harikrishna_gurram supergroup          0 2015-06-23 09:40 /user/harikrishna_gurram/results1/_SUCCESS

-rw-r--r--   3 harikrishna_gurram supergroup         34 2015-06-23 09:40 /user/harikrishna_gurram/results1/part-r-00000

Open “part-r-00000” file; you can see the number of characters of given input file.

$ hadoop fs -cat /user/harikrishna_gurram/results1/part-r-00000

Total Characters in file are    23999453

Previous                                                 Next                                                 Home