HDFS （Hadoop Distributed File System）

分布式存儲(chǔ)系統(tǒng)

支持海量數(shù)據(jù)的存儲(chǔ)，成百上千的計(jì)算機(jī)組成存儲(chǔ)集群，HDFS可以運(yùn)行在低成本的硬件之上，具有的高容錯(cuò)、高可靠性、高可擴(kuò)展性、高吞吐率等特征，非常適合大規(guī)模數(shù)據(jù)集上的應(yīng)用。

優(yōu)點(diǎn)

• 高容錯(cuò)性
• 適合批處理
• 適合大數(shù)據(jù)處理
• 流式文件訪問(wèn)
• 可構(gòu)建在廉價(jià)機(jī)器上

缺點(diǎn)

• 不適合低延遲數(shù)據(jù)訪問(wèn)
• 不適合小文件存取
• 不適合并發(fā)寫(xiě)入、文件隨機(jī)修改

HDFS操作

命令操作HDFS

# 顯示目錄 / 下的文件
hdfs dfs -ls /
# 新建文件夾，絕對(duì)路徑
hdfs dfs -mkdir /test
# 上傳文件
hdfs dfs -put test.txt /test/
# 下載文件
hdfs dfs -get /test/test.txt
# 輸出文件內(nèi)容
hdfs dfs -cat /test/test.txt

Web端操作HDFS

打開(kāi)http://192.168.9.200:9870/，可以對(duì)HDFS進(jìn)行操作

文件在這里管理

可視化操作還是簡(jiǎn)單一些

Java操作HDFS

1. 添加依賴

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>3.3.2</version>
</dependency>

2. 建立連接

public void setUp() throws Exception {
    System.out.println("開(kāi)始建立與HDFS的連接");
    configuration = new Configuration();
    fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
}

3. 文件操作

    Configuration configuration = null;
    FileSystem fileSystem = null;
    public static final String HDFS_PATH = "hdfs://192.168.9.200:9000";

    /**
     * 在 hdfs中新建文件夾
     *
     * @throws Exception
     */
    @Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/JavaDemo/test"));
    }

    /**
     * 創(chuàng)建文件
     *
     * @throws Exception
     */

    @Test
    public void create() throws Exception {
        FSDataOutputStream outputStream = fileSystem.create(new Path("/JavaDemo/test/haha.txt"));
        outputStream.write("hello bigdata from javaDemo".getBytes());
        outputStream.flush();
        outputStream.close();
    }


    /**
     * 查看文件 hdfs -fs -cat file
     *
     * @throws Exception
     */
    @Test
    public void cat() throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/JavaDemo/test/haha.txt"));
        IOUtils.copyBytes(in, System.out, 1024);
        in.close();
    }

    /**
     * 重命名文件
     *
     * @throws Exception
     */
    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/JavaDemo/test/haha.txt");
        Path newPath = new Path("/JavaDemo/test/hehe.txt");
        fileSystem.rename(oldPath, newPath);
    }

    /**
     * 上傳文件到HDFS
     *
     * @throws Exception
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Path loacalPath = new Path("hello.txt");
        Path hdfsPath = new Path("/");
        fileSystem.copyFromLocalFile(loacalPath, hdfsPath);
    }


    /**
     * 上傳文件到HDFS帶進(jìn)度信息
     *
     * @throws Exception
     */

    @Test
    public void copyFromLocalFileWithProgress() throws Exception {
        InputStream in = new BufferedInputStream(Files.newInputStream(new File("hbase-2.2.7-bin.tar.gz").toPath()));
        FSDataOutputStream ouput = fileSystem.create(new Path("/JavaDemo/test/hbase-2.2.7-bin.tar.gz"), () -> {
            System.out.print(".");
        });
        IOUtils.copyBytes(in, ouput, 4096);
    }

    /**
     * 下載文件到HDFS
     *
     * @throws Exception
     */
    @Test
    public void copyToLocalFile() throws Exception {
        Path hdfsPath = new Path("/JavaDemo/test/haha.txt");
        Path loacalPath = new Path("./haha.txt");
        //  useRawLocalFileSystem
        fileSystem.copyToLocalFile(false, hdfsPath, loacalPath, true);
    }

    /**
     * 查看某個(gè)目錄下所有文件
     *
     * @throws Exception
     */
    @Test
    public void listFiles() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus f : fileStatuses) {
            String isDir = f.isDirectory() ? "文件夾" : "文件";
            short replication = f.getReplication();
            long len = f.getLen();
            String path = f.getPath().toString();

            System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);

        }
    }

    /**
     * 刪除文件
     *
     * @throws Exception
     */
    @Test
    public void delete() throws IOException {
        fileSystem.delete(new Path("/JavaDemo/haha.txt"), true);
    }

4. 關(guān)閉連接

public void tearDown() {
    configuration = null;
    fileSystem = null;
    System.out.println("關(guān)閉與HDFS的連接");
}

完整測(cè)試文件

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;
import java.nio.file.Files;

/**
 * @author Gettler
 * Java 操作HDFS
 */
public class HdfsDemo {
    Configuration configuration = null;
    FileSystem fileSystem = null;
    public static final String HDFS_PATH = "hdfs://192.168.9.200:9000";

    /**
     * 在 hdfs中新建文件夾
     *
     * @throws Exception
     */
    @Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/JavaDemo/test"));
    }

    /**
     * 創(chuàng)建文件
     *
     * @throws Exception
     */

    @Test
    public void create() throws Exception {
        FSDataOutputStream outputStream = fileSystem.create(new Path("/JavaDemo/test/haha.txt"));
        outputStream.write("hello bigdata from javaDemo".getBytes());
        outputStream.flush();
        outputStream.close();
    }


    /**
     * 查看文件 hdfs -fs -cat file
     *
     * @throws Exception
     */
    @Test
    public void cat() throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/JavaDemo/test/haha.txt"));
        IOUtils.copyBytes(in, System.out, 1024);
        in.close();
    }

    /**
     * 重命名文件
     *
     * @throws Exception
     */
    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/JavaDemo/test/haha.txt");
        Path newPath = new Path("/JavaDemo/test/hehe.txt");
        fileSystem.rename(oldPath, newPath);
    }

    /**
     * 上傳文件到HDFS
     *
     * @throws Exception
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Path loacalPath = new Path("hello.txt");
        Path hdfsPath = new Path("/");
        fileSystem.copyFromLocalFile(loacalPath, hdfsPath);
    }


    /**
     * 上傳文件到HDFS帶進(jìn)度信息
     *
     * @throws Exception
     */

    @Test
    public void copyFromLocalFileWithProgress() throws Exception {
        InputStream in = new BufferedInputStream(Files.newInputStream(new File("hbase-2.2.7-bin.tar.gz").toPath()));
        FSDataOutputStream ouput = fileSystem.create(new Path("/JavaDemo/test/hbase-2.2.7-bin.tar.gz"), () -> {
            System.out.print(".");
        });
        IOUtils.copyBytes(in, ouput, 4096);
    }

    /**
     * 下載文件到HDFS
     *
     * @throws Exception
     */
    @Test
    public void copyToLocalFile() throws Exception {
        Path hdfsPath = new Path("/JavaDemo/test/haha.txt");
        Path loacalPath = new Path("./haha.txt");
        //  useRawLocalFileSystem
        fileSystem.copyToLocalFile(false, hdfsPath, loacalPath, true);
    }

    /**
     * 查看某個(gè)目錄下所有文件
     *
     * @throws Exception
     */
    @Test
    public void listFiles() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus f : fileStatuses) {
            String isDir = f.isDirectory() ? "文件夾" : "文件";
            short replication = f.getReplication();
            long len = f.getLen();
            String path = f.getPath().toString();

            System.out.println(isDir + "\t" + replication + "\t" + len + "\t" + path);

        }
    }

    /**
     * 刪除文件
     *
     * @throws Exception
     */
    @Test
    public void delete() throws IOException {
        fileSystem.delete(new Path("/JavaDemo/haha.txt"), true);
    }


    //測(cè)試之前執(zhí)行的代碼
    @Before
    public void setUp() throws Exception {
        System.out.println("開(kāi)始建立與HDFS的連接");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "hadoop");
    }

    //測(cè)試之完執(zhí)行的代碼
    @After
    public void tearDown() {
        configuration = null;
        fileSystem = null;
        System.out.println("關(guān)閉與HDFS的連接");
    }
}

9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999ssssssssss99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999文章來(lái)源地址http://www.zghlxwxcb.cn/news/detail-468973.html

到了這里，關(guān)于Hadoop學(xué)習(xí)筆記之HDFS的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！