HDFS Java API access mode example code
- 2021-01-22 05:10:29
- OfStack
This paper mainly studies the access mode of Java API of HDFS, the specific code is shown below, with detailed comments.
Recently, the rhythm is a little fast, and when you are free, I will put this package 1
The implementation code
Packages to import:
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
Entity method:
/**
* To obtain HDFS The file system
* @return
* @throws IOException
* @throws URISyntaxException
*/
public static FileSystem getFileSystem() throws IOException, URISyntaxException{
//read config file
Configuration conf = new Configuration();
// Returns the default file system
// If the Hadoop Run in a cluster, using this method can directly get the default file system
//FileSystem fs = FileSystem.get(conf);
// The specified file system address
URI uri = new URI("hdfs://hy:9000");
// Returns the specified file system
// If you are testing locally, you need to use this method to get the file system
FileSystem fs = FileSystem.get(uri, conf);
return fs;
}
/**
* Create file directory
* @throws Exception
*/
public static void mkdir() throws Exception{
// Get the file system
FileSystem fs = getFileSystem();
// Create file directory
fs.mkdirs(new Path("hdfs://hy:9000/hy/weibo"));
// Release resources
fs.close();
}
/**
* Delete files or file directories
* @throws Exception
*/
public static void rmdir() throws Exception{
// Get the file system
FileSystem fs = getFileSystem();
// Delete files or file directories
fs.delete(new Path("hdfs://hy:9000/hy/weibo"), true);
// Release resources
fs.close();
}
/**
* Gets all the files in the directory
* @throws Exception
*/
public static void listAllFile() throws Exception{
// Get the file system
FileSystem fs = getFileSystem();
// List the contents of the directory
FileStatus[] status = fs.listStatus(new Path("hdfs://hy:9000/hy/"));
// Gets the path of all files in the directory
Path[] listedPaths = FileUtil.stat2Paths(status);
// Loop through each file
for (Path path : listedPaths) {
System.out.println(path);
}
// Release resources
fs.close();
}
/**
* Upload the file to HDFS
* @throws Exception
*/
public static void copyToHDFS() throws Exception{
// Get the file object
FileSystem fs = getFileSystem();
// The source file path is Linux The path of the Path srcPath = new Path("/home/hadoop/temp.jar");
// If you need to windows Under the test, need to change Windows The path down here, for example E://temp.jar
Path srcPath = new Path("E://temp.jar");
// Objective the path
Path dstPath = new Path("hdfs://hy:9000/hy/weibo");
// Implement file upload
fs.copyFromLocalFile(srcPath, dstPath);
// Release resources
fs.close();
}
/**
* from HDFS Download files from
* @throws Exception
*/
public static void getFile() throws Exception{
// Get the file system
FileSystem fs = getFileSystem();
// Source file path
Path srcPath = new Path("hdfs://hy:9000/hy/weibo/temp.jar");
// Destination path, default is Linux Under the
// If the Windows Under the test, need to change Windows The following path, such as C://User/andy/Desktop/
Path dstPath = new Path("D://");
// download HDFS The file on the
fs.copyToLocalFile(srcPath, dstPath);
// Release resources
fs.close();
}
/**
* To obtain HDFS Cluster point information
* @throws Exception
*/
public static void getHDFSNodes() throws Exception{
// Get the file system
FileSystem fs = getFileSystem();
// Get a distributed file system
DistributedFileSystem hdfs = (DistributedFileSystem)fs;
// Get all nodes
DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();
// Loop over traversal
for (int i = 0; i < dataNodeStats.length; i++) {
System.out.println("DataNote_" + i + "_Name:" + dataNodeStats[i].getHostName());
}
// Release resources
fs.close();
}
/**
* Find a file in HDFS Cluster location
* @throws Exception
*/
public static void getFileLocal() throws Exception{
// Get the file system
FileSystem fs = getFileSystem();
// The file path
Path path = new Path("hdfs://hy:9000/hy/weibo/temp.jar");
// Get the file directory
FileStatus fileStatus = fs.getFileStatus(path);
// Gets a list of file block locations
BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
// Loop out block information
for (int i = 0; i < blockLocations.length; i++) {
String[] hosts = blockLocations[i].getHosts();
System.out.println("block_" + i + "_location:" + hosts[0]);
}
// Release resources
fs.close();
}
conclusion
Above is this article about HDFS Java API access mode example code of all content, hope to be helpful to you. Interested friends can continue to refer to the site of other related topics, if there are shortcomings, welcome to leave a message to point out. Thank you for your support!