第九課 HDFS API 的使用

HDFS API的高級編程

HDFS的API就兩個：

測試用的hdfs-size.xml

<code>

<configuration>
    <property>
        <name>dfs.replication/<name>
        <value>1/<value>
    /<property>
/<configuration>
​/<code>

HDFS API 樣例

<code>package com.qinghe.hdfs;
​
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
​
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Test;
​
public class HdfsClientDemo {
​
    /**
     * 測試創建文件夾
     * 
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testMkdirs() throws IOException, InterruptedException, URISyntaxException {
​ 

        /**
         * 更改用戶操作有3種方式 1、直接設置運行換種的用戶名為bigdata VM arguments ; -DHADOOP_USER_NAME=bigdata
         * 
         * 2、在代碼中進行聲明 System.setProperty("HADOOP_USER_NAME", "bigdata");
         * 
         * 3、在每次讀取文件時設置 FileSystem fs = FileSystem.get(new
         * URI("hdfs://bigdata101:9000"), configuration, "bigdata");
         */
​
        // 1 獲取文件系統
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), configuration, "bigdata");
​
        // 2 創建目錄
        fs.mkdirs(new Path("/user/bigdata/input"));
​
        // 3 關閉資源
        fs.close();
​
        System.out.println("over");
    }
​
    /**
     * HDFS文件上傳（測試參數優先級）
     * 
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testCopyFromLocalFile() throws IOException, InterruptedException, URISyntaxException {
​
        // 1 獲取文件系統
        Configuration configuration = new Configuration();
​
        System.setProperty("HADOOP_USER_NAME", "bigdata");
​
        /**
         * 分別註釋下面兩行代碼測試優先級
         * 
         * 參數優先級排序： （1）客戶端代碼中設置的值 >（2）ClassPath下的用戶自定義配置文件 >（3）然後是服務器的默認配置 

         * 
         */
        configuration.set("dfs.replication", "2");
        configuration.addResource("hdfs-size.xml");
​
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), configuration);
​
        // 2 上傳文件
        fs.copyFromLocalFile(new Path("C:/bigdata/first.txt"), new Path("/user/bigdata/input"));
​
        // 3 關閉資源
        fs.close();
​
        System.out.println("over");
    }
​
    /**
     * HDFS文件下載
     * 
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testCopyToLocalFile() throws IOException, InterruptedException, URISyntaxException {
        // 1 獲取文件系統
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), conf, "bigdata");
​
        /**
         * 執行下載操作
         * 
         * boolean delSrc whether to delete Path src Path dst boolean
         * useRawLocalFileSystem whether to use RawLocalFileSystem as local file system
         * or not
         */
        fs.copyToLocalFile(false, new Path("/user/bigdata/input/first.txt"), new Path("c:/bigdata/downfirst.txt"),
                true);
​
        // 3 關閉資源
        fs.close();
​
        System.out.println("over");
    }
​
    /**
     * HDFS文件夾刪除 

     * 
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testDeletePath() throws IOException, InterruptedException, URISyntaxException {
        // 1 獲取文件系統
        Configuration conf = new Configuration();
​
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), conf, "bigdata");
​
        // 2 執行刪除
        fs.delete(new Path("/user/bigdata/input"), true);
​
        // 3 關閉資源
        fs.close();
​
        System.out.println("over");
    }
​
    /**
     * HDFS文件名更改
     * 
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testRenameFileName() throws IOException, InterruptedException, URISyntaxException {
​
        // 1 獲取文件系統
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), conf, "bigdata");
​
        // 2 修改文件名稱
        fs.rename(new Path("/user/bigdata/input/first.txt"), new Path("/user/bigdata/input/new_first.txt"));
​
        // 3 關閉資源
        fs.close();
​
        System.out.println("over");
    }
​
    /**
     * 查看文件名稱、權限、長度、塊信息 

     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testListFiles() throws IOException, InterruptedException, URISyntaxException {
        // 1 獲取文件系統
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), conf, "bigdata");
​
        // 2 獲取文件詳情
        RemoteIterator<locatedfilestatus> listFiles = fs.listFiles(new Path("/"), true);
​
        while (listFiles.hasNext()) {
            LocatedFileStatus status = listFiles.next();
​
            /**
             * 文件名稱\\t長度\\t權限\\t分組
             */
            System.out.println(status.getPath().getName() + "\\t" + status.getLen() + "\\t" + status.getPermission()
                    + "\\t" + status.getGroup());
            
            // 獲取存儲的塊信息
            BlockLocation[] blockLocations = status.getBlockLocations();
            
            for (BlockLocation blockLocation : blockLocations) {
                
                // 獲取塊存儲的主機節點
                String[] hosts = blockLocation.getHosts();
                
                for (String host : hosts) {
                    System.out.println(host);
                }
            }
            System.out.println("--------華麗的分割線---------");
        }
        
        // 3 關閉資源
        fs.close();
    }
    
    /**
     * HDFS文件和文件夾判斷
     * 
     * @throws IOException 

     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Test
    public void testListClass() throws IOException, InterruptedException, URISyntaxException {
        
        // 1 獲取文件系統
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://bigdata101:9000"), conf, "bigdata");
        
        // 2 判斷是文件還是文件夾
        FileStatus[] listStatus = fs.listStatus(new Path("/"));
        
        for (FileStatus fileStatus : listStatus) {
            // 如果是文件
            if (fileStatus.isFile()) {
                System.out.println("f:" + fileStatus.getPath().getName());
            } else {
                System.out.println("d:" + fileStatus.getPath().getName());
            }
        }
        
        // 3 關閉資源
        fs.close();
        
    }
​
}
​/<locatedfilestatus>/<code>

閱讀更多 大數據漫路求索 的文章

關鍵字: HDFS 第九課 樣例