SpringBoot +Hadoop3.1.0 hdfs文件操作流程：

小灰灰 2023-10-18 20:53 95阅读 0赞

1、项目结构:

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70][]

核心pom.xml 文件:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    	<modelVersion>4.0.0</modelVersion>
    	<parent>
    		<groupId>com.zzg</groupId>
    		<artifactId>boot-stream-file-system</artifactId>
    		<version>0.0.1-SNAPSHOT</version>
    	</parent>
    	<artifactId>hadoop-hdfs</artifactId>
    
    	<dependencies>
    		
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-client</artifactId>
    			<version>3.1.0</version>
    		</dependency>
    		
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-common</artifactId>
    			<version>3.1.0</version>
    		</dependency>
    		
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-hdfs</artifactId>
    			<version>3.1.0</version>
    		</dependency>
    	</dependencies>
    </project>

hdfs 配置参数对象：

package com.zzg.hdfs.config;
    
    import java.io.Serializable;
    
    /**
     * 
     * @ClassName:  HdfsConfig   
     * @Description: Hadoop Hdfs配置对象 
     * @author:  -zzg
     * @date:   2019年7月2日 下午5:00:49   
     *     
     * @Copyright: 2019 www.digipower.cn 
     * 注意：本内容仅限于内部使用，禁止用于其他的商业目的
     */
    public class HdfsConfig implements Serializable {
    
    	/**   
    	 * @Fields serialVersionUID : TODO(用一句话描述这个变量表示什么)   
    	 */   
    	private static final long serialVersionUID = -3927708731917979149L;
    	// hdfs 服务器地址
    	private String hostname;
    	// hdfs 服务器端口
    	private String port;
    	// hdfs 服务器账户
    	private String username;
    	
    	// 构造函数
    	public HdfsConfig() {
    		super();
    		// TODO Auto-generated constructor stub
    	}
    	
    	public HdfsConfig(String hostname, String port, String username) {
    		super();
    		this.hostname = hostname;
    		this.port = port;
    		this.username = username;
    	}
    
    	//set 和 get 
    	public String getHostname() {
    		return hostname;
    	}
    	public void setHostname(String hostname) {
    		this.hostname = hostname;
    	}
    	public String getPort() {
    		return port;
    	}
    	public void setPort(String port) {
    		this.port = port;
    	}
    	public String getUsername() {
    		return username;
    	}
    	public void setUsername(String username) {
    		this.username = username;
    	}
    	
    	
    
    }

hdfs 文件工具类：

package com.zzg.hdfs.util;
    
    import java.io.FileInputStream;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.OutputStream;
    import java.net.URI;
    import java.net.URISyntaxException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IOUtils;
    
    import com.zzg.hdfs.config.HdfsConfig;
    
    /**
     * 
     * @ClassName: HdfsUtil
     * @Description: Hdfs 工具类封装
     * @author:  -zzg
     * @date: 2019年7月2日 下午5:04:13
     * 
     * @Copyright: 2019 www.digipower.cn 注意：本内容仅限于内部使用，禁止用于其他的商业目的
     */
    public class HdfsUtil {
    
    	/**
    	 * 
    	 * @Title: upload @Description: 文件上传 @param: @param config @param: @param
    	 * source @param: @param destination @return: void @throws
    	 */
    	public static void upload(HdfsConfig config, String source, String destination) {
    
    		try {
    			// 获得FileSystem对象，指定使用root用户上传
    			FileSystem fileSystem = FileSystem.get(new URI(getHdfsUrl(config)), new Configuration(),
    					config.getUsername());
    			// 创建输入流，参数指定文件输出地址
    			InputStream in = new FileInputStream(source);
    			// 调用create方法指定文件上传，参数HDFS上传路径
    			OutputStream out = fileSystem.create(new Path(destination));
    			// 使用Hadoop提供的IOUtils，将in的内容copy到out，设置buffSize大小，是否关闭流设置true
    			IOUtils.copyBytes(in, out, 4096, true);
    
    		} catch (IOException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		} catch (InterruptedException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		} catch (URISyntaxException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		}
    	}
    	
    	/**
    	 * 
    	 * @Title: download @Description: 文件上传 @param: @param config @param: @param
    	 * source @param: @param destination @return: void @throws
    	 */
    	public static void download(HdfsConfig config, String source, String destination) {
    
    		try {
    			// 获得FileSystem对象，指定使用root用户上传
    			FileSystem fileSystem = FileSystem.get(new URI(getHdfsUrl(config)), new Configuration(),
    					config.getUsername());
    			// 调用open方法进行下载，参数HDFS路径
    			InputStream in = fileSystem.open(new Path(source));
    			// 创建输出流，参数指定文件输出地址
    			OutputStream out = new FileOutputStream(destination);
    			IOUtils.copyBytes(in, out, 4096, true);
    
    		} catch (IOException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		} catch (InterruptedException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		} catch (URISyntaxException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    		}
    	}
    
    	/**
    	 * 
    	 * @Title: delete @Description: 文件删除 @param: @param config @param: @param
    	 * target @param: @return @return: boolean @throws
    	 */
    	public static boolean delete(HdfsConfig config, String target) {
    		boolean flag = false;
    		try {
    			// 获得FileSystem对象，指定使用root用户上传
    			FileSystem fileSystem = FileSystem.get(new URI(getHdfsUrl(config)), new Configuration(), config.getUsername());
    			// 调用delete方法，删除指定的文件。参数:false:表示是否递归删除
    			flag = fileSystem.delete(new Path(target), false);
    		} catch (IOException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    			return false;
    		} catch (InterruptedException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    			return false;
    		} catch (URISyntaxException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    			return false;
    		}
    		return flag;
    	}
    
    	public static boolean mkdir(HdfsConfig config, String directory) {
    		boolean flag = false;
    		try {
    			// 获得FileSystem对象
    			FileSystem 	fileSystem = FileSystem.get(new URI(getHdfsUrl(config)), new Configuration(), config.getUsername());
    			// 调用mkdirs方法，在HDFS文件服务器上创建文件夹。
    			flag = fileSystem.mkdirs(new Path(directory));
    
    		} catch (IOException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    			return false;
    		} catch (InterruptedException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    			return false;
    		} catch (URISyntaxException e) {
    			// TODO Auto-generated catch block
    			e.printStackTrace();
    			return false;
    		};
    		
    		return flag;
    	}
    
    	private static String getHdfsUrl(HdfsConfig config) {
    		StringBuilder builder = new StringBuilder();
    		builder.append("hdfs://").append(config.getHostname()).append(":").append(config.getPort());
    		return builder.toString();
    	}
    
    }

hdfs 测试方法：

package com.zzg.hdfs.test;
    
    import com.zzg.hdfs.config.HdfsConfig;
    import com.zzg.hdfs.util.HdfsUtil;
    
    public class Test {
    //  文件上传
    //	public static void main(String[] args) {
    //		// TODO Auto-generated method stub
    //		HdfsConfig config = new HdfsConfig("192.168.60.201", "9000", "root");
    //		String source = "C:\\data\\febs.sql"; // windows 文件
    //		String destination = "/hadoop/febs.sql"; //centos7 hdfs 文件存储地址
    //		HdfsUtil.upload(config, source, destination);
    //	}
    	// 文件下载
    	public static void main(String[] args) {
    		// TODO Auto-generated method stub
    		HdfsConfig config = new HdfsConfig("192.168.60.201", "9000", "root");
    		String source = "/hadoop/febs.sql"; // windows 文件
    		String destination = "C:\\data\\febs.sql"; //centos7 hdfs 文件存储地址
    		HdfsUtil.download(config, source, destination);
    	}
    	
    
    	
    
    }

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20190702182308999.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n,size_16,color_FFFFFF,t_70