\u4f7f\u7528MapReduce\u548cspark\u5bf9\u65e5\u5fd7\u6570\u636e\u8fdb\u884c\u7b80\u5355\u7684\u6e05\u7406\u548c\u603b\u7ed3 \u9996\u5148\u4f7f\u7528MapReduce\u5bf9\u65e5\u5fd7\u8fdb\u884c\u5206\u5272\uff0c\u5c06\u2026 Read More »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[3,9],"tags":[67],"class_list":["post-980","post","type-post","status-publish","format-standard","hentry","category-hadoop","category-mapreduce","tag-67"],"_links":{"self":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/980","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/comments?post=980"}],"version-history":[{"count":1,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/980\/revisions"}],"predecessor-version":[{"id":1398,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/980\/revisions\/1398"}],"wp:attachment":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/media?parent=980"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/categories?post=980"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/tags?post=980"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}hadoop jar rain-hadoop-1.0-SNAPSHOT.jar com.rain.mapreduce.BaiduLog \/data\/baidu.log \/data\/log\/clean5<\/code><\/p>\nBaiduLog.java<\/h2>\n
import org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.io.IntWritable;\nimport org.apache.hadoop.io.LongWritable;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.mapreduce.Job;\nimport org.apache.hadoop.mapreduce.Mapper;\nimport org.apache.hadoop.mapreduce.Reducer;\nimport org.apache.hadoop.mapreduce.lib.input.FileInputFormat;\nimport org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;\n\nimport java.io.IOException;\n\npublic class BaiduLog {\n public static class BaiduLogMapper extends Mapper<LongWritable,Text, Text, LogBean> {\n @Override\n protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {\n\/\/ super.map(key, value, context);\n String log = value.toString();\n String str = \"(cn.baidu.core.inteceptor.LogInteceptor:55)\";\n if (log.indexOf(str)!=-1){\n String[] log_arr = log.split(str);\n String time = log_arr[0].substring(1, 10);\n String[] log_arr2 = log_arr[1].split(\"\\t\");\n String ip = log_arr2[1];\n String url = log_arr2[2];\n if (url.equals(\"null\")){\n url = log_arr2[3];\n }\n LogBean logbean = new LogBean(time,ip,url);\n context.write(new Text(ip),logbean);\n }\n }\n }\n public static class BaiduLogReducer extends Reducer<Text,LogBean,IntWritable,Text>{\n\n @Override\n protected void reduce(Text key, Iterable<LogBean> values, Context context) throws IOException, InterruptedException {\n\/\/ super.reduce(key, values, context);\n int sum = 0;\n\n StringBuffer str = new StringBuffer();\n int flag = 0;\n for (LogBean logbean:values){\n sum++;\n if (flag==0){\n str.append(logbean.toString());\n flag = 1;\n }\n }\n context.write(new IntWritable(sum),new Text(str.toString()));\n\n }\n }\n public static void main(String[] args) throws Exception {\n Configuration conf = new Configuration();\n Job job = Job.getInstance(conf, \"avg\");\n\n job.setJarByClass(BaiduLog.class);\n job.setMapperClass(BaiduLog.BaiduLogMapper.class);\n job.setReducerClass(BaiduLog.BaiduLogReducer.class);\n\n\/\/ job.setCombinerClass(BaiduLog.BaiduLogReducer.class);\n\n job.setOutputKeyClass(Text.class);\n job.setOutputValueClass(LogBean.class);\n\n FileInputFormat.addInputPath(job,new Path(args[0]));\n FileOutputFormat.setOutputPath(job,new Path(args[1]));\n System.exit(job.waitForCompletion(true)?0:1);\n }\n\n}\n\n<\/code><\/pre>\nLogBean.java<\/h2>\n
import org.apache.hadoop.io.Writable;\nimport org.apache.hadoop.io.WritableComparable;\n\nimport java.io.DataInput;\nimport java.io.DataOutput;\nimport java.io.IOException;\n\npublic class LogBean implements WritableComparable<LogBean> {\n private String time;\n private String ip;\n private String url;\n\n public LogBean() {\n super();\n }\n\n public LogBean(String time, String ip, String url) {\n this.time = time;\n this.ip = ip;\n this.url = url;\n }\n\n @Override\n public String toString() {\n return \"LogBean{\" +\n \"time='\" + time + ' ' +\n \", ip='\" + ip + ' ' +\n \", url='\" + url + ' ' +\n '}';\n }\n\n public String getTime() {\n return time;\n }\n\n public void setTime(String time) {\n this.time = time;\n }\n\n public String getIp() {\n return ip;\n }\n\n public void setIp(String ip) {\n this.ip = ip;\n }\n\n public String getUrl() {\n return url;\n }\n\n public void setUrl(String url) {\n this.url = url;\n }\n\n\n @Override\n public int compareTo(LogBean o) {\n return 0;\n }\n\n @Override\n public void write(DataOutput out) throws IOException {\n out.writeUTF(time);\n out.writeUTF(ip);\n out.writeUTF(url);\n\n }\n\n @Override\n public void readFields(DataInput in) throws IOException {\n time = in.readUTF();\n ip = in.readUTF();\n url = in.readUTF();\n }\n}\n\n\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"