濮阳建网站/dw如何制作网页
java远程操作hadoop的分布式文件系统hdfs
需要导入的hadoop的jar包
hadoop解压包中有所需的jar包
hadoop-2.5.2/share/hadoop/common/hadoop-common-2.5.2.jar
hadoop-2.5.2/share/hadoop/common/lib/*.jar(lib下的全部jar包)
hadoop-2.5.2/share/hadoop/hdfs/hadoop-hdfs-2.5.2.jar
如果是集群内部用eclipse中的java程序访问hdfs的话，可以直接写代码操作，如果是远程操作的话需要配置hdfs-site.xml文件，关闭权限检查，不配置的话会报错Permission denied
hdfs-site.xml
dfs.permissions.enabled
false
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
/**
* java远程或本地操作hdfs文件系统(eclipse)
* @author root
*
*/
public class HDFSFile {
public static void main(String [] args) throws IOException {
/**
* java读取hdfs文件信息
*/
/* 方法一 */
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL("hdfs://master:9000/hello.txt");
InputStream in = url.openStream();
IOUtils.copyBytes(in,System.out,4096,true);//in是输入流，System.out是标准输出，4096是指定缓存区大小，true是读完后把输入流自动关闭
/* 方法二 */
Configuration conf = new Configuration();
conf.set("fs.defaultFS","hdfs://master:9000");
FileSystem filesystem = FileSystem.get(conf);
boolean success = filesystem.mkdirs(new Path("/test"));
System.out.println(success);
// 判断文件夹是否存在
success = filesystem.exists(new Path("/hello.txt"));
System.out.println(success);
// 删除文件夹或文件
success = filesystem.delete(new Path("/test"),true);
System.out.println(success);
// true --> now false --> delete home
success = filesystem.exists(new Path("/test"));
System.out.println(success);
// 向hdfs文件中写入数据
FSDataOutputStream out = filesystem.create(new Path("/test.data"),true);
FileInputStream in = new FileInputStream("/root/a.txt");
byte [] buf = new byte[4096];
int len = in.read(buf);
while (len != -1) {
out.write(buf,0,len);
len = in.read(buf);
}
in.close();
out.close();
// 查看文件hdfs文件目录
FileStatus[] fs = filesystem.listStatus(new Path("/"));
for(FileStatus status :fs) {
System.out.println(status.getPath());
System.out.println(status.getPermission());
System.out.println(status.getReplication());//copy how much
}
}
}