HDFS Java API access mode example code

  • 2021-01-22 05:10:29
  • OfStack

This paper mainly studies the access mode of Java API of HDFS, the specific code is shown below, with detailed comments.

Recently, the rhythm is a little fast, and when you are free, I will put this package 1

The implementation code

Packages to import:


import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;

Entity method:


/**
   *  To obtain HDFS The file system 
   * @return
   * @throws IOException 
   * @throws URISyntaxException 
   */
public static FileSystem getFileSystem() throws IOException, URISyntaxException{
	//read config file
	Configuration conf = new Configuration();
	// Returns the default file system 
	// If the Hadoop Run in a cluster, using this method can directly get the default file system 
	//FileSystem fs = FileSystem.get(conf);
	// The specified file system address 
	URI uri = new URI("hdfs://hy:9000");
	// Returns the specified file system 
	// If you are testing locally, you need to use this method to get the file system 
	FileSystem fs = FileSystem.get(uri, conf);
	return fs;
}
/**
   *  Create file directory 
   * @throws Exception
   */
public static void mkdir() throws Exception{
	// Get the file system 
	FileSystem fs = getFileSystem();
	// Create file directory 
	fs.mkdirs(new Path("hdfs://hy:9000/hy/weibo"));
	// Release resources 
	fs.close();
}
/**
   *  Delete files or file directories 
   * @throws Exception
   */
public static void rmdir() throws Exception{
	// Get the file system 
	FileSystem fs = getFileSystem();
	// Delete files or file directories 
	fs.delete(new Path("hdfs://hy:9000/hy/weibo"), true);
	// Release resources 
	fs.close();
}
/**
   *  Gets all the files in the directory 
   * @throws Exception
   */
public static void listAllFile() throws Exception{
	// Get the file system 
	FileSystem fs = getFileSystem();
	// List the contents of the directory 
	FileStatus[] status = fs.listStatus(new Path("hdfs://hy:9000/hy/"));
	// Gets the path of all files in the directory 
	Path[] listedPaths = FileUtil.stat2Paths(status);
	// Loop through each file 
	for (Path path : listedPaths) {
		System.out.println(path);
	}
	// Release resources 
	fs.close();
}
/**
   *  Upload the file to HDFS
   * @throws Exception
   */
public static void copyToHDFS() throws Exception{
	// Get the file object 
	FileSystem fs = getFileSystem();
	// The source file path is Linux The path of the  Path srcPath = new Path("/home/hadoop/temp.jar");
	// If you need to windows Under the test, need to change Windows The path down here, for example  E://temp.jar
	Path srcPath = new Path("E://temp.jar");
	// Objective the path 
	Path dstPath = new Path("hdfs://hy:9000/hy/weibo");
	// Implement file upload 
	fs.copyFromLocalFile(srcPath, dstPath);
	// Release resources 
	fs.close();
}
/**
   *  from HDFS Download files from 
   * @throws Exception
   */
public static void getFile() throws Exception{
	// Get the file system 
	FileSystem fs = getFileSystem();
	// Source file path 
	Path srcPath = new Path("hdfs://hy:9000/hy/weibo/temp.jar");
	// Destination path, default is Linux Under the 
	// If the Windows Under the test, need to change Windows The following path, such as C://User/andy/Desktop/
	Path dstPath = new Path("D://");
	// download HDFS The file on the 
	fs.copyToLocalFile(srcPath, dstPath);
	// Release resources 
	fs.close();
}
/**
   *  To obtain HDFS Cluster point information 
   * @throws Exception
   */
public static void getHDFSNodes() throws Exception{
	// Get the file system 
	FileSystem fs = getFileSystem();
	// Get a distributed file system 
	DistributedFileSystem hdfs = (DistributedFileSystem)fs;
	// Get all nodes 
	DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();
	// Loop over traversal 
	for (int i = 0; i < dataNodeStats.length; i++) {
		System.out.println("DataNote_" + i + "_Name:" + dataNodeStats[i].getHostName());
	}
	// Release resources 
	fs.close();
}
/**
   *  Find a file in HDFS Cluster location 
   * @throws Exception
   */
public static void getFileLocal() throws Exception{
	// Get the file system 
	FileSystem fs = getFileSystem();
	// The file path 
	Path path = new Path("hdfs://hy:9000/hy/weibo/temp.jar");
	// Get the file directory 
	FileStatus fileStatus = fs.getFileStatus(path);
	// Gets a list of file block locations 
	BlockLocation[] blockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
	// Loop out block information 
	for (int i = 0; i < blockLocations.length; i++) {
		String[] hosts = blockLocations[i].getHosts();
		System.out.println("block_" + i + "_location:" + hosts[0]);
	}
	// Release resources 
	fs.close();
}

conclusion

Above is this article about HDFS Java API access mode example code of all content, hope to be helpful to you. Interested friends can continue to refer to the site of other related topics, if there are shortcomings, welcome to leave a message to point out. Thank you for your support!


Related articles: