0% found this document useful (0 votes)
51 views15 pages

Customer - 3.java: Import Import Import Import Import Import Import Import

This document contains code for several Java classes used in Hadoop MapReduce jobs. The classes include mappers, reducers and main methods that configure and run jobs. The jobs process CSV and text data, extract values, perform filtering and grouping, and write outputs. Key aspects configured include input/output formats, mapper and reducer classes, and number of reduce tasks.

Uploaded by

Karthik Raj
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as DOCX, PDF, TXT or read online on Scribd
0% found this document useful (0 votes)
51 views15 pages

Customer - 3.java: Import Import Import Import Import Import Import Import

This document contains code for several Java classes used in Hadoop MapReduce jobs. The classes include mappers, reducers and main methods that configure and run jobs. The jobs process CSV and text data, extract values, perform filtering and grouping, and write outputs. Key aspects configured include input/output formats, mapper and reducer classes, and number of reduce tasks.

Uploaded by

Karthik Raj
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as DOCX, PDF, TXT or read online on Scribd
You are on page 1/ 15

Customer_3.

java

import java.io.IOException;
import java.util.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class customer_3 {


public static class Map extends Mapper<Text, Text, Text, Text>{
//String c="harsh";
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException
{
String[] line = value.toString().split(",");
String ukey = line[0];
String uvalue = line[1] + "," + line[2] + "," + line[3];
context.write(new Text(ukey), new Text(value));
}
}
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
conf.setInt(NLineInputFormat.LINES_PER_MAP, 4);

Job job = new Job(conf , "customer");


job.setJarByClass(customer_3.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(Map.class);

job.setInputFormatClass(NLineInputFormat.class);
job.setNumReduceTasks(0);

//job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}
}

Customer.java

import java.io.IOException;
import java.util.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class customer {


public static class Map extends Mapper<Text, Text, Text, Text>{
String c="harsh";
public void map(Text key, Text value, Context context) throws
IOException, InterruptedException
{
String line=key.toString();
if(c.equalsIgnoreCase (line))
context.write(key,value);
}
}
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();

conf.set("mapreduce.input.keyvaluelinerecordreader.key.value.separator"
,",");

Job job = new Job(conf , "customer");


job.setJarByClass(customer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(Map.class);

job.setInputFormatClass(KeyValueTextInputFormat.class);

//job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}

Customer_2.java

import java.io.IOException;
import java.util.*;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class customer_2 {


public static class Map extends Mapper<Text, Text, Text, Text>{
//String c="harsh";
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException
{
String[] line = value.toString().split(",");
String ukey = line[0];
String uvalue = line[1] + "," + line[2] + "," + line[3];
context.write(new Text(ukey), new Text(value));
}
}
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
conf.setInt(NLineInputFormat.LINES_PER_MAP, 4);

Job job = new Job(conf , "customer");


job.setJarByClass(customer_2.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(Map.class);

job.setInputFormatClass(NLineInputFormat.class);
job.setNumReduceTasks(0);

//job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}

CustomPartitioner.java

import java.util.*;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class dataset {


public static class Map extends Mapper<LongWritable, Text, Text,
IntWritable>{
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException{
String[] line = value.toString().split(";");
try {
//if((Integer.valueOf(line[0]) > 15 &&
Integer.valueOf(line[1]) > 60000)){
context.write(new Text(line[3] + " " + line[0] + "
"), new IntWritable(Integer.valueOf(line[1])));
//}
}catch(Exception e){
}
}
}
public static class dpart extends Partitioner<Text,IntWritable>
{
public int getPartition(Text key,IntWritable value,int nr)
{
if(value.get()<30000)
return 0;
if(value.get() < 50000)
return 1;
else
return 2;
}
}
/*
public static class Reduce extends Reducer<Text, IntWritable, Text
,IntWritable>{
int n = 0;
public void reduce(Text key ,Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException{
for (IntWritable val: values){
context.write(key , val);
n++;
}
}
public void cleanup(Context context) throws IOException,
InterruptedException{
context.write(new Text("Total records satifying = "), new
IntWritable(n));
}
}
*/

public static void main(String[] args) throws Exception{


Configuration conf = new Configuration();
Job job = new Job(conf, "dataset");
job.setJarByClass(dataset.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(Map.class);
//job.setCombinerClass(Reduce.class);
//job.setReducerClass(Reduce.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setPartitionerClass(dpart.class);
job.setNumReduceTasks(3);
job.waitForCompletion(true);
}

Lab1.java

import java.util.*;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class Lab1


{
public static class map extends Mapper <LongWritable , Text , Text
, IntWritable>
{
public void map(LongWritable key , Text value , Context
context)
{
String[] values = value.get().split(';');

context.write(new Text(values[4]) , new


IntWritable(values[5]));
}
}

public static class reduce extends Reducer <Text , IntWritable ,


Text , IntWritable>
{
public int male_sum = 0;
public int male_count = 0;
public int female_sum = 0;
public int female_count = 0;

public void reduce(Text key , Iterable<IntWritable> values ,


Context context)
{
String gender = key.get();
if(gender.equals(new String("Male")))
{
for(int value : values)
{
male_sum += value;
male_count += 1;
}
}
if(gender.equals(new String("Female")))
{
for(int value : values)
{
female_sum += value;
female_count += 1;
}
}
}

public void cleanup(Context context)


{
context.write(new Text("Male average : ") , new
IntWritable(male_sum / male_count));
context.write(new Text("Female average : ") , new
IntWritable(female_sum / female_count));
}
}

public static void main(Strings[] args)


{
Configuration conf = new Configuration();
Job job = new Job(conf , "Lab1");

job.setJarByClass(Lab1.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);

job.setMapperClass(map.class);
job.setReduceClass(reduce.class);

job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);

FileInputFormat.addInputPath(args[0]);
FileOutputFormat.setOutputPath(args[1]);

job.waitForCompletion(true);
}
}

Lab2.java

import java.util.*;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class Lab1


{
public static class map extends Mapper <LongWritable , Text , Text
, IntWritable>
{
public void map(LongWritable key , Text value , Context
context)
{
String[] values = value.get().split(';');

context.write(value , new IntWritable(values[1]));


}
}

public static class dpart extends Partitioner <Text , IntWritable>


{
public int getPartition(Text key , IntWritable value , int nr)
{
if(value.get() < 30000)
return 0;
else if(value.get() < 50000)
return 1;
else
return 2;
}
}

public static class reduce extends Reducer <Text , IntWritable ,


Text , IntWritable>
{

public void reduce(Text key , Iterable<IntWritable> values ,


Context context)
{
context.write(key , value);
}
}

public static void main(Strings[] args)


{
Configuration conf = new Configuration();
Job job = new Job(conf , "Lab1");

job.setJarByClass(Lab1.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);

job.setMapperClass(map.class);
job.setReduceClass(reduce.class);
job.setPartitionerClass(dpart.class);

job.setNumReduceTasks(3);

job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);

FileInputFormat.addInputPath(args[0]);
FileOutputFormat.setOutputPath(args[1]);

job.waitForCompletion(true);
}
}

SequentialIO.java

import java.util.*;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class SequentialIO {


public static class Map extends Mapper<LongWritable, Text,
LongWritable, Text>{
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException{
String[] line = value.toString().split(",");
try {

context.write(key , new Text(line[0] + " " + line[1] +


" " + line[2] + " " + line[3]));

}catch(Exception e){
}
}
}
/*
public static class Reduce extends Reducer<Text, IntWritable, Text
,IntWritable>{
int n = 0;
public void reduce(Text key ,Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException{
for (IntWritable val: values){
context.write(key , val);
n++;
}
}
public void cleanup(Context context) throws IOException,
InterruptedException{
context.write(new Text("Total records satifying = "), new
IntWritable(n));
}
}
*/

public static void main(String[] args) throws Exception{


Configuration conf = new Configuration();
Job job = new Job(conf, "SequentialIO");
job.setJarByClass(SequentialIO.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(Map.class);
//job.setCombinerClass(Reduce.class);
//job.setReducerClass(Reduce.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
//job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//job.setNumReduceTasks(3);
job.waitForCompletion(true);
}

WordCount.java
import java.util.*;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class wordcount


{
public static class map extends Mapper <LongWritable , Text , Text
, IntWritable>
{
public void map(LongWritable key, Text value, Context context)
{
context.write(value , new IntWritable("1"));
}
}

public static class reduce extends Reducer <Text , IntWritable ,


Text , IntWritable>
{
public int total_count = 0;
public void reduce(Text key , Iterable<IntWritable> values ,
Context context)
{
int count = 0;
for(IntWritable value : values)
{
count += value.get();
}
total_count += count;
context.write(key , new IntWritable(count));
}

public void cleanup(Context context) throws IOException ,


InterruptedException
{
context.write(new Text("Total no of word : ") , new
IntWritable(total_count));
}
}

public static void main(Strings[] args) throws Exception


{
Configuration conf = new Configuration();
Job job = new Job(conf , "wordcount");

// Setting Jar file...


job.setJarByClass(wordcount.class);

// Setting Output Key-Value


job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);

// Setting Mapper-Reducer class...


job.setMapperClass(map.class);
job.setReducerClass(reduce.class);

// Setting Input-Output format class...


job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextInputFormat.class);

// Setting Input-Output path...


FileInputFormat.addInputPath(job , new Path(args[0]));
FileInputFormat.setOutputPath(job , new Path(args[1]));

job.waitForCompletion(true);
}

Wordmax.java
import java.util.*;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;

public class wordmax


{
public static class map extends Mapper <LongWritable , Text , Text
, IntWritable>
{

public void map(LongWritable key , Text value , Context


context)
{
String values[] = value.get().split(',');

for(String value : values)


context.write(new Text(value) , new IntWritable(1));
}
}

public static class reduce extends Reducer <Text , IntWritable ,


Text , IntWritable>
{
public int max_count = 0;
public int min_count = 9999999;
public String max_key = "";
public String min_key = "";
public void reduce(Text key , Iterable<IntWritable> values ,
Context context)
{
int count = 0;

for(int value : values)


count += value;

if(count > max_count){


max_count = count;
max_key = key.get();
}

if(count < min_count){


min_count = count;
min_key = key.get();
}
}

public void cleanup(Context context) throws IOException ,


InterruptedException
{
context.write(new Text("Maximum " + max_key + " = ") , new
IntWritable(max_count));
context.write(new Text("Minimum " + min_key + " = ") , new
IntWritable(min_count));
}
}

public static void main(Strings[] args) throws Exception


{
Configuration conf = new Configuration();
Job job = new Job(conf , "wordmax");

// Setting Jar class...


job.setJarByClass(wordmax.class);

// Setting Output Key-Value class..


job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);

// Setting Mapper-Reducer class...


job.setMapperClass(map.class);
job.setReducerClass(reduce.class);

// Setting Input-Output Format Class..


job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);

// Setting Input-Output path...


FileInputFormat.addInputPath(job , new Path(args[0]));
FileInputFormat.setOutputPath(job , new Path(args[1]));

job.waitForCompletion(true);
}
}

You might also like