f

0.20.2 버전에서의 소스 기본 템플릿

2013.02.20 15:27 콤퓨타/Hadoop

/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package sssp_hadoop;

import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

/**
*
* @author Sanghyun Lee
*/
public class SSSP_Hadoop {

/**
* @param args the command line arguments
*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
// TODO code application logic here

Configuration conf = new Configuration();
// args[0] = input path;
// args[1] = output path;
Job job = getMakeADJListJob(conf, new Path(args[0]), new Path(args[1]));
job.waitForCompletion(true);
}

private static Job getMakeADJListJob(Configuration conf, Path input, Path output) throws IOException {
Job job = new Job(conf);

FileSystem fs = FileSystem.get(conf);
job.setJarByClass(SSSP_Hadoop.class);
job.setJobName("Make Adjacency List");

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);

job.setMapperClass(ADJMapper.class);
job.setReducerClass(ADJReducer.class);

if (fs.exists(output)) {
fs.delete(output, true);
}

job.setInputFormatClass(TextInputFormat.class);
// job.setOutputFormatClass(TextOutputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);

FileInputFormat.addInputPath(job, input);
FileOutputFormat.setOutputPath(job, output);

return job;
}

public static class ADJMapper extends Mapper<LongWritable, Text, Text, Text> {

@Override
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {


}
}

public static class ADJReducer extends Reducer<Text, Text, Text, Text> {

@Override
public void reduce(Text key, Iterable<Text> value, Context context) throws IOException, InterruptedException {

}
}
}

Trackbacks 0 / Comments 0

리듀스의 출력이 Text, Text 일 때, 알파벳 역순으로 출력되게 하는 Comparator

2013.02.20 15:27 콤퓨타/Hadoop

리듀스의 출력이 Text, Text 일 때, 알파벳 역순으로 출력되게 하는 Comparator

출처 : http://pkghosh.wordpress.com/2011/04/13/map-reduce-secondary-sort-does-it-all/

public static class KeyComprator extends WritableComparator {

02

03 protected KeyComprator() {

04 super(Text.class, true);

05 }

06

07 @Override

08 public int compare(WritableComparable w1, WritableComparable w2) {

09

10//ascending zone and day

11

12Text t1 = (Text) w1;

13 Text t2 = (Text) w2;

14 String[] t1Items = t1.toString().split(":");

15 String[] t2Items = t2.toString().split(":");

16 String t1Base = t1Items[0] + ":" + t1Items[1] + ":";

17 String t2Base = t2Items[0] + ":" + t2Items[1] + ":";

18 int comp = t1Base.compareTo(t2Base);

19

20//descending value

21 if (comp == 0) {

22 comp = t2Items[2].compareTo(t1Items[2]);

23 }

24

25 return comp;

26

27 }

28 }

Trackbacks 0 / Comments 0

Copyright © 어니엉월드 All Rights Reserved | JB All In One Version 0.4 Designed by CMSFactory.NET

티스토리 툴바