本文主要包含mapreduce倒排索引,mapreduce编程实例,mapreduce编程模型,mapreduce编程,mapreduce编程模板等服务器相关知识,网友希望可以进行参考
MapReduce编程之倒排索引，mapreduce编程索引
任务要求：
//输入文件格式
18661629496 110
13107702446 110
1234567 120
2345678 120
987654 110
2897839274 18661629496
//输出文件格式格式
11018661629496|13107702446|987654|18661629496|13107702446|987654|
1201234567|2345678|1234567|2345678|
186616294962897839274|2897839274|
mapreduce程序编写：
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class Test2 {
enum Counter
{
LINESKIP,//记录出错的行
}
public static class Map extends Mapper<LongWritable, Text, Text, Text>{
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();//读取源数据
try
{
//数据处理
String [] lineSplit = line.split(" ");//18661629496,110
String anum = lineSplit[0];
String bnum = lineSplit[1];
//输出格式：110,18661629496
context.write(new Text(bnum), new Text(anum));
}
catch(ArrayIndexOutOfBoundsException e)
{
context.getCounter(Counter.LINESKIP).increment(1);//出错时计数器+1
return;
}
}
}
|