HDFSUtil.java<\/p>\n
package com.rain.hdfs;\n\nimport org.apache.commons.io.IOUtils;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.*;\nimport org.junit.Before;\nimport org.junit.Test;\n\nimport java.io.FileInputStream;\nimport java.io.FileNotFoundException;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.net.URI;\n\npublic class HDFSUtil {\n\n FileSystem fs = null;\n\n\n\n\/\/ \u8fd9\u662f\u5148\u884c\u8fd0\u884c\u7684\u51fd\u6570\uff0c\u5728\u5176\u4ed6\u51fd\u6570\u8fd0\u884c\u4e4b\u524d\uff0c\u5c06\u914d\u7f6e\u5199\u597d\n @Before\n public void init() throws Exception{\n\n \/\/\u8bfb\u53d6classpath\u4e0b\u7684xxx-site.xml \u914d\u7f6e\u6587\u4ef6\uff0c\u5e76\u89e3\u6790\u5176\u5185\u5bb9\uff0c\u5c01\u88c5\u5230conf\u5bf9\u8c61\u4e2d\n\/\/ \u53ef\u4ee5\u5c06core-site.xml\u5bfc\u5165\u5230\u9879\u76ee\u4e2d\n Configuration conf = new Configuration();\n\n \/\/\u4e5f\u53ef\u4ee5\u5728\u4ee3\u7801\u4e2d\u5bf9conf\u4e2d\u7684\u914d\u7f6e\u4fe1\u606f\u8fdb\u884c\u624b\u52a8\u8bbe\u7f6e\uff0c\u4f1a\u8986\u76d6\u6389\u914d\u7f6e\u6587\u4ef6\u4e2d\u7684\u8bfb\u53d6\u7684\u503c\n conf.set(\"fs.defaultFS\", \"hdfs:\/\/hmaster:9000\/\");\n\n \/\/\u6839\u636e\u914d\u7f6e\u4fe1\u606f\uff0c\u53bb\u83b7\u53d6\u4e00\u4e2a\u5177\u4f53\u6587\u4ef6\u7cfb\u7edf\u7684\u5ba2\u6237\u7aef\u64cd\u4f5c\u5b9e\u4f8b\u5bf9\u8c61\n fs = FileSystem.get(new URI(\"hdfs:\/\/hmaster:9000\/\"),conf,\"hadoop\");\n\n }\n\n\n\n \/**\n * \u4e0a\u4f20\u6587\u4ef6\uff0c\u6bd4\u8f83\u5e95\u5c42\u7684\u5199\u6cd5\n *\n * @throws Exception\n *\/\n @Test\n public void upload_old() throws Exception {\n\n Configuration conf = new Configuration();\n conf.set(\"fs.defaultFS\", \"hdfs:\/\/hmaster:9000\/\");\n\n FileSystem fs = FileSystem.get(conf);\n\n Path dst = new Path(\"hdfs:\/\/hmaster:9000\/data\/sample.txt\");\n\n FSDataOutputStream os = fs.create(dst);\n\n FileInputStream is = new FileInputStream(\"\/Users\/rain\/Downloads\/sample.txt\");\n\n IOUtils.copy(is, os);\n\n\n }\n\n \/**\n * \u4e0a\u4f20\u6587\u4ef6\uff0c\u5c01\u88c5\u597d\u7684\u5199\u6cd5\n * @throws Exception\n * @throws IOException\n *\/\n @Test\n public void upload() throws Exception, IOException{\n\n\/\/ \u524d\u9762\u4e3a\u4e0a\u4f20\u6587\u4ef6\u7684\u8def\u5f84\uff0c\u540e\u9762\u662fHDFS\u4e2d\u7684\u8def\u5f84\u548c\u6587\u4ef6\u7684\u65b0\u540d\u79f0\n fs.copyFromLocalFile(new Path(\"\/Users\/rain\/Downloads\/sample.txt\"), new Path(\"hdfs:\/\/hmaster:9000\/data\/test.txt\"));\n\n }\n\n\n \/**\n * \u4e0b\u8f7d\u6587\u4ef6\n * @throws Exception\n * @throws IllegalArgumentException\n *\/\n @Test\n public void download() throws Exception {\n\/\/ \u524d\u9762\u662fHDFS\u4e2d\u6587\u4ef6\u7684\u8def\u5f84\uff0c\u540e\u9762\u662f\u8981\u4e0b\u8f7d\u5230\u672c\u5730\u7684\u8def\u5f84\u4ee5\u53ca\u540d\u79f0\n fs.copyToLocalFile(new Path(\"hdfs:\/\/hmaster:9000\/sample.txt\"), new Path(\"\/Users\/rain\/Downloads\/sample.txt\"));\n\n }\n\n \/**\n * \u67e5\u770b\u6587\u4ef6\u4fe1\u606f\n * @throws IOException\n * @throws IllegalArgumentException\n * @throws FileNotFoundException\n *\n *\/\n @Test\n public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {\n\n \/\/ listFiles\u5217\u51fa\u7684\u662f\u6587\u4ef6\u4fe1\u606f\uff0c\u800c\u4e14\u63d0\u4f9b\u9012\u5f52\u904d\u5386\n RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path(\"\/data\"), true);\n\n while(files.hasNext()){\n\n LocatedFileStatus file = files.next();\n Path filePath = file.getPath();\n String fileName = filePath.getName();\n System.out.println(fileName);\n\n }\n\n System.out.println(\"---------------------------------\");\n\n \/\/listStatus \u53ef\u4ee5\u5217\u51fa\u6587\u4ef6\u548c\u6587\u4ef6\u5939\u7684\u4fe1\u606f\uff0c\u4f46\u662f\u4e0d\u63d0\u4f9b\u81ea\u5e26\u7684\u9012\u5f52\u904d\u5386\n FileStatus[] listStatus = fs.listStatus(new Path(\"\/data\"));\n for(FileStatus status: listStatus){\n\n String name = status.getPath().getName();\n System.out.println(name + (status.isDirectory()?\" is dir\":\" is file\"));\n\n }\n\n }\n\n \/**\n * \u521b\u5efa\u6587\u4ef6\u5939\n * @throws Exception\n * @throws IllegalArgumentException\n *\/\n @Test\n public void mkdir() throws IllegalArgumentException, Exception {\n\n fs.mkdirs(new Path(\"\/data\/test\"));\n\n\n }\n\n \/**\n * \u5220\u9664\u6587\u4ef6\u6216\u6587\u4ef6\u5939\n * @throws IOException\n * @throws IllegalArgumentException\n *\/\n @Test\n public void rm() throws IllegalArgumentException, IOException {\n\n fs.delete(new Path(\"\/data\/test\"), true);\n\n }\n\n\n public static void main(String[] args) throws Exception {\n\n\n }\n\n\n\n}\n<\/code><\/pre>\npom.xml<\/p>\n
<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http:\/\/maven.apache.org\/POM\/4.0.0\"\n xmlns:xsi=\"http:\/\/www.w3.org\/2001\/XMLSchema-instance\"\n xsi:schemaLocation=\"http:\/\/maven.apache.org\/POM\/4.0.0 http:\/\/maven.apache.org\/xsd\/maven-4.0.0.xsd\">\n <modelVersion>4.0.0<\/modelVersion>\n\n <groupId>com.rain.hadoop<\/groupId>\n <artifactId>rain-hadoop<\/artifactId>\n <version>1.0-SNAPSHOT<\/version>\n <build>\n <plugins>\n <plugin>\n <groupId>org.apache.maven.plugins<\/groupId>\n <artifactId>maven-compiler-plugin<\/artifactId>\n <configuration>\n <source>6<\/source>\n <target>6<\/target>\n <\/configuration>\n <\/plugin>\n <\/plugins>\n <\/build>\n\n\n <properties>\n <project.build.sourceEncoding>UTF-8<\/project.build.sourceEncoding>\n <hadoop.version>2.6.0-cdh5.7.0<\/hadoop.version>\n <\/properties>\n\n <repositories>\n <repository>\n <id>cloudera<\/id>\n <url>https:\/\/repository.cloudera.com\/artifactory\/cloudera-repos<\/url>\n <\/repository>\n <\/repositories>\n\n <dependencies>\n <!--\u6dfb\u52a0\u4f9d\u8d56-->\n <dependency>\n <groupId>org.apache.hadoop<\/groupId>\n <artifactId>hadoop-client<\/artifactId>\n <version>2.6.0-cdh5.7.0<\/version>\n <\/dependency>\n\n <dependency>\n <groupId>org.apache.hbase<\/groupId>\n <artifactId>hbase-server<\/artifactId>\n <version>1.2.0-cdh5.7.0<\/version>\n <\/dependency>\n <dependency>\n <groupId>org.apache.hbase<\/groupId>\n <artifactId>hbase-client<\/artifactId>\n <version>1.2.0-cdh5.7.0<\/version>\n <\/dependency>\n\n <dependency>\n <groupId>junit<\/groupId>\n <artifactId>junit<\/artifactId>\n <version>4.11<\/version>\n <scope>test<\/scope>\n <\/dependency>\n <\/dependencies>\n\n<\/project>\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"\u4f7f\u7528Java\u8c03\u7528HDFS\u7684API\u8fdb\u884c\u6587\u4ef6\u57fa\u672c\u64cd\u4f5c \u672c\u6837\u4f8b\u4e3b\u8981\u662f\u4ee3\u7801\uff0c\u5176\u4e2d\u5305\u62ec\u4e86Java\u8fdb\u884c\u4e0a\u4f20\uff0c\u4e0b\u8f7d\uff0c\u5220\u9664\uff0c\u521b\u2026 Read More »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[3,4],"tags":[43,67],"class_list":["post-974","post","type-post","status-publish","format-standard","hentry","category-hadoop","category-hdfs","tag-hdfs","tag-67"],"_links":{"self":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/974","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/comments?post=974"}],"version-history":[{"count":1,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/974\/revisions"}],"predecessor-version":[{"id":1400,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/posts\/974\/revisions\/1400"}],"wp:attachment":[{"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/media?parent=974"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/categories?post=974"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/rain1024.com\/index.php\/wp-json\/wp\/v2\/tags?post=974"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}