{"id":971,"date":"2018-08-03T19:06:07","date_gmt":"2018-08-03T11:06:07","guid":{"rendered":"http:\/\/www.rain1024.com\/?p=971"},"modified":"2023-08-07T20:57:25","modified_gmt":"2023-08-07T12:57:25","slug":"article131","status":"publish","type":"post","link":"http:\/\/rain1024.com\/index.php\/2018\/08\/03\/article131\/","title":{"rendered":"\u4f7f\u7528MapReduce\u7edf\u8ba1\u5929\u6c14\u6570\u636e\u4e2d\u7684\u6700\u9ad8\u6c14\u6e29"},"content":{"rendered":"<h1>\u4f7f\u7528MapReduce\u7edf\u8ba1\u5929\u6c14\u6570\u636e\u4e2d\u7684\u6700\u9ad8\u6c14\u6e29<\/h1>\n<h2>\u4f8b\u5b50\u662f\u4eceHadoop\u6743\u5a01\u6307\u5357\u4e0a\u6539\u5199\u7684\uff0c\u9700\u8981\u5148\u5c06\u4ee3\u7801\u751f\u6210jar\u5305\uff0c\u7136\u540e\u5c06\u6d4b\u8bd5\u6837\u4f8b\u548cjar\u5305\u4e0a\u4f20\u5230HDFS\u4e2d\uff0c\u4f7f\u7528\u547d\u4ee4\u8fd0\u884c\uff0c\u6700\u7ec8\u4f1a\u5f97\u51fa\u6bcf\u5e74\u7684\u6700\u9ad8\u6c14\u6e29\u7edf\u8ba1\u7ed3\u679c\uff0c\u8fd0\u884c\u547d\u4ee4\u4e0d\u5199\u51fa\u6765\u4e86\u3002\u4ee3\u7801\u4e5f\u5f88\u7b80\u5355\uff0c\u5c31\u662f\u5206\u522b\u5199\u51famap\u548creduce\u7684\u64cd\u4f5c\uff0cmap\u4e2d\u5bf9\u6570\u636e\u8fdb\u884c\u4e00\u4e2a\u5206\u5272\uff0c\u63d0\u53d6\uff0c\u5c06\u7ed3\u679c\u4f20\u7ed9reduce\u8fdb\u884c\u7edf\u8ba1\u3002<\/h2>\n<pre><code class=\"java\">package com.rain.mapreduce;\n\nimport org.apache.hadoop.fs.Path;\nimport org.apache.hadoop.io.IntWritable;\nimport org.apache.hadoop.io.LongWritable;\nimport org.apache.hadoop.io.Text;\nimport org.apache.hadoop.mapreduce.lib.input.FileInputFormat;\nimport org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;\nimport org.apache.hadoop.mapred.MapReduceBase;\nimport org.apache.hadoop.mapreduce.Job;\nimport org.apache.hadoop.mapreduce.Mapper;\nimport org.apache.hadoop.mapreduce.Reducer;\n\nimport java.io.IOException;\n\n\/**\n* Hadoop\u4e66\u4e0a\u7684\u6700\u9ad8\u6c14\u6e29\u5224\u65ad\n*\/\npublic class MaxTemperature {\npublic static class MaxTemperatureMapper extends Mapper&lt;LongWritable,Text, Text, IntWritable&gt;{\n\nprivate static final int MISSING = 9999;\n\n@Override\npublic void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {\nString line = value.toString();\nString year = line.substring(15,19);\nint airTemperature;\nif (line.charAt(87) == '+') {\nairTemperature = Integer.parseInt(line.substring(88,92));\n}else{\nairTemperature = Integer.parseInt(line.substring(87,92));\n}\nString quality = line.substring(92,93);\nif (airTemperature!=MISSING&amp;&amp;quality.matches(\"[01459]\")){\ncontext.write(new Text(year), new IntWritable(airTemperature));\n}\n\n}\n}\npublic static class MaxTemperatureReducer extends Reducer&lt;Text ,IntWritable,Text,IntWritable&gt;{\n@Override\nprotected void reduce(Text key, Iterable&lt;IntWritable&gt; values, Context context) throws IOException, InterruptedException {\nint maxValue = Integer.MIN_VALUE;\nfor (IntWritable value:values){\nmaxValue = Math.max(maxValue,value.get());\n}\ncontext.write(key,new IntWritable(maxValue));\n\n}\n}\npublic static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {\nif (args.length!=2){\nSystem.err.println(\"Usage:MaxTEmperature\");\nSystem.exit(-1);\n}\nJob job = new Job();\njob.setJarByClass(MaxTemperature.class);\njob.setJobName(\"Max temperature\");\n\nFileInputFormat.addInputPath(job,new Path(args[0]));\nFileOutputFormat.setOutputPath(job,new Path(args[1]));\n\njob.setMapperClass(MaxTemperatureMapper.class);\njob.setReducerClass(MaxTemperatureReducer.class);\n\njob.setOutputKeyClass(Text.class);\njob.setOutputValueClass(IntWritable.class);\n\nSystem.exit(job.waitForCompletion(true)?0:1);\n}\n}\n\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u4f7f\u7528MapReduce\u7edf\u8ba1\u5929\u6c14\u6570\u636e\u4e2d\u7684\u6700\u9ad8\u6c14\u6e29 \u4f8b\u5b50\u662f\u4eceHadoop\u6743\u5a01\u6307\u5357\u4e0a\u6539\u5199\u7684\uff0c\u9700\u8981\u5148\u5c06\u4ee3\u7801\u751f\u6210jar\u5305\u2026 <span class=\"read-more\"><a href=\"http:\/\/rain1024.com\/index.php\/2018\/08\/03\/article131\/\">Read More &raquo;<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[3,9],"tags":[42,54,67],"class_list":["post-971","post","type-post","status-publish","format-standard","hentry","category-hadoop","category-mapreduce","tag-hadoop","tag-mapreduce","tag-67"],"_links":{"self":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/971","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/comments?post=971"}],"version-history":[{"count":1,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/971\/revisions"}],"predecessor-version":[{"id":1402,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/971\/revisions\/1402"}],"wp:attachment":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/media?parent=971"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/categories?post=971"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/tags?post=971"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}