hdfs文件工具类代码,Java代码库,德仔网

代码语言
代码分类
【Java】 hdfs文件工具类
作者:my_name_123 / 发布于2015/4/2/ 345
hdfs 读中文乱码解决方案

	package tools;
	
	import java.io.BufferedReader;
	import java.io.IOException;
	import java.io.InputStreamReader;
	import java.util.ArrayList;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;
	
	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.FSDataOutputStream;
	import org.apache.hadoop.fs.FileSystem;
	import org.apache.hadoop.fs.Path;
	
	public class OpHDFSTools {
	
	 /**
	 * @param args
	 */
	 public static void main(String[] args) {
	 // TODO Auto-generated method stub
	 try {
	 //getStrListFormHDFSFile("/dsap/");
	 } catch (Exception e) {
	 // TODO Auto-generated catch block
	 e.printStackTrace();
	 }
	 }
	
	 public static List<String> getStrListFormHDFSFile(String fpath) throws Exception
	 {
	
	
	 BufferedReader fileBR = makeBufferReader(fpath);
	
	 List<String> lineList=new ArrayList<String>();
	
	 //遍历文件
	 String strLineBuff="";
	 try
	 {
	 while((strLineBuff=fileBR.readLine()) != null)
	 {
	 strLineBuff = strLineBuff.trim();
	// System.out.println("strLineBuff:"+strLineBuff);
	 if(strLineBuff!=null && !"".equals(strLineBuff))
	 {
	 lineList.add(strLineBuff);
	 }
	 }
	 }
	 catch(Exception e)
	 {
	 e.printStackTrace();
	 System.out.println(fpath);
	 }
	
	 killBufferReader(fileBR);
	
	// System.out.println(fileBR.readLine());
	
	 return lineList;
	 }
	
	 public static Map<String,String> getFormatMapFormHDFSFile(String fpath,String sep,int num) throws Exception
	 {
	 BufferedReader fileBR = makeBufferReader(fpath);
	
	 Map<String,String> lineInfo=new HashMap<String,String>();
	
	 //遍历文件
	 String strLineBuff="";
	 try
	 {
	 while((strLineBuff=fileBR.readLine()) != null)
	 {
	 if(strLineBuff!=null && !"".equals(strLineBuff))
	 {
	 try
	 {
	 String [] strArr=strLineBuff.split(sep,2);
	 String valArr[]=strArr[1].split("\t",num);
	 for(int i=0;i<valArr.length;i++)
	 {
	 if("".equals(valArr[i])||valArr[i]==null)
	 {
	 valArr[i]="all";
	 }
	 }
	 strArr[1]=OpStrTools.arrayToString(valArr);
	 lineInfo.put(strArr[0],strArr[1]);
	 }
	 catch(Exception e)
	 {}
	 }
	 }
	 }
	 catch(Exception e)
	 {
	 e.printStackTrace();
	 System.out.println(fpath);
	 }
	
	 killBufferReader(fileBR);
	 return lineInfo;
	 }
	
	 public static Map<String,String> getMapFormHDFSFile(String fpath,String sep) throws Exception
	 {
	 BufferedReader fileBR = makeBufferReader(fpath);
	
	 Map<String,String> lineInfo=new HashMap<String,String>();
	
	 //遍历文件
	 String strLineBuff="";
	 try
	 {
	 while((strLineBuff=fileBR.readLine()) != null)
	 {
	 if(strLineBuff!=null && !"".equals(strLineBuff))
	 {
	 try
	 {
	 String [] strArr=strLineBuff.split(sep,2);
	 lineInfo.put(strArr[0],strArr[1]);
	 }
	 catch(Exception e)
	 {}
	 }
	 }
	 }
	 catch(Exception e)
	 {
	 e.printStackTrace();
	 System.out.println(fpath);
	 }
	
	 killBufferReader(fileBR);
	 return lineInfo;
	 }
	
	
	
	 public static void killBufferReader(BufferedReader reader)
	 {
	 try
	 {
	 if(reader!=null)
	 {
	 reader.close();
	 }
	 }
	 catch (IOException e)
	 {
	 // TODO Auto-generated catch block
	 e.printStackTrace();
	 }
	 }
	
	
	 public static BufferedReader makeBufferReader(String ifpath) throws Exception
	 {
	 //打开读文件
	 String charset="utf-8";
	 return makeBufferReader(ifpath,charset);
	 }
	
	 public static BufferedReader makeBufferReader(String ifpath,String charset) throws Exception
	 {
	 //打开读文件
	 Configuration config = new Configuration();
	 FileSystem hdfs = FileSystem.get(config);
	 Path path = new Path(ifpath);
	 InputStreamReader isr = null;
	 BufferedReader fileBR = null;
	
	 try
	 {
	 isr = new InputStreamReader( hdfs.open(path),charset);
	 fileBR = new BufferedReader(isr);
	 return fileBR;
	 }
	 catch(Exception e)
	 {
	 e.printStackTrace();
	 System.out.println("open file error!");
	 if(fileBR!=null)
	 {
	 try
	 {
	 fileBR.close();
	 }
	 catch(IOException e1)
	 {
	 e1.printStackTrace();
	 }
	 }
	 if(isr!=null)
	 {
	 try
	 {
	 isr.close();
	 }
	 catch(IOException e1)
	 {
	 e1.printStackTrace();
	 }
	 }
	 return null;
	 }
	
	 }
	
	
	 //创建目录
	 public static void mkdir(String path) throws IOException
	 {
	 Configuration conf = new Configuration();
	 FileSystem fs = FileSystem.get(conf);
	 Path srcPath = new Path(path);
	 boolean isok = fs.mkdirs(srcPath);
	 if(isok){
	 System.out.println("create dir ok!");
	 }else{
	 System.out.println("create dir failure");
	 }
	 fs.close();
	 }
	
	 //创建新文件
	 public static void createFile(String dst , byte[] contents) throws IOException
	 {
	 Configuration conf = new Configuration();
	 FileSystem fs = FileSystem.get(conf);
	 Path dstPath = new Path(dst); //目标路径
	 //打开一个输出流
	 FSDataOutputStream outputStream = fs.create(dstPath);
	 outputStream.write(contents);
	 outputStream.close();
	 fs.close();
	 System.out.println("文件创建成功！");
	 }
	
	}
评论列表
本站所提供的代码，版权归原作者所有，若有侵犯作者版权，请与我们联系，我们将立即删除或修改。谢谢!
本站所有代码发布及提供者。
试试其它关键字
　文件工具　
同语言下
可能有用的
my_name_123贡献的其它代码(2)