Hadoop HDFS文件系统通过java FileSystem 实现上传下代码,Java代码库,德仔网

代码语言
代码分类
【Java】 Hadoop HDFS文件系统通过java FileSystem 实现上传下
作者: / 发布于2017/5/3/ 249
package linlintest;  
  
import java.io.File;  
import java.io.FileOutputStream;  
import java.io.IOException;  
import java.net.URI;  
  
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.FSDataInputStream;  
import org.apache.hadoop.fs.FileStatus;  
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IOUtils;  
  
public class HdfsFileSystem {  
      
     public static void main(String[] args) throws Exception  {  
         //String uri="hdfs://LL-167:8020/";   //hdfs 地址  
         // String remote="hdfs://LL-167:8020/lin/in1/1.txt"; //hdfs上的路径  
         String uri="hdfs://192.168.0.151:8020/";   //hdfs 地址  
         String local="C:/Users/Administrator/Desktop/a.txt";  //本地路径  
         String remote="hdfs://192.168.0.151:8020/Workspace/houlinlin";  
         Configuration conf = new Configuration();  
           
         //cat(conf ,uri,"hdfs://LL-167:8020/lin/in1/1.txt");  
         //download(conf ,uri,remote,local);  
        //  delete(conf ,uri,"hdfs://192.168.0.173:8020/Workspace/houlinlin");  
         // markDir(conf ,uri,"hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");  
          
        //  checkDir(uri,"d:/file");  
        // getFile(conf ,uri,"","hdfs://192.168.0.151:8020/Workspace/houlinlin/a.txt");  
         copyFile(conf,uri,"C:/Users/Administrator/Desktop/8a8380c745d05d370145d06719aa3c89.txt","hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");  
       //   ls(conf ,"hdfs://fulonghadoop","hdfs://fulonghadoop/");  
    }  
    /** 
     * 上传文件 
     * @param conf 
     * @param local 
     * @param remote 
     * @throws IOException 
     */  
    public static void copyFile(Configuration conf , String uri , String local, String remote) throws IOException {  
        FileSystem fs = FileSystem.get(URI.create(uri), conf);  
        fs.copyFromLocalFile(new Path(local), new Path(remote));  
        System.out.println("copy from: " + local + " to " + remote);  
        fs.close();  
    }  
      
    /** 
     * 获取hdfs上文件流 
     * @param conf 
     * @param uri 
     * @param local 
     * @param remote 
     * @throws IOException 
     */  
    public static  void getFileStream(Configuration conf , String uri , String local, String remote) throws IOException{  
        FileSystem fs = FileSystem.get(URI.create(uri), conf);  
        Path path= new Path(remote);  
        FSDataInputStream in = fs.open(path);//获取文件流  
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/b.txt");//输出流  
        int ch = 0;  
        while((ch=in.read()) != -1){    
            fos.write(ch);    
        }    
        System.out.println("-----");  
        in.close();   
        fos.close();    
    }  
      
    /** 
     * 创建文件夹 
     * @param conf 
     * @param uri 
     * @param remoteFile 
     * @throws IOException 
     */  
    public static void markDir(Configuration conf , String uri , String remoteFile ) throws IOException{  
        FileSystem fs = FileSystem.get(URI.create(uri), conf);  
        Path path = new Path(remoteFile);  
          
        fs.mkdirs(path);  
        System.out.println("创建文件夹"+remoteFile);  
           
    }  
    /** 
     * 查看文件 
     * @param conf 
     * @param uri 
     * @param remoteFile 
     * @throws IOException 
     */  
     public static void cat(Configuration conf , String uri ,String remoteFile) throws IOException {  
            Path path = new Path(remoteFile);  
            FileSystem fs = FileSystem.get(URI.create(uri), conf);  
            FSDataInputStream fsdis = null;  
            System.out.println("cat: " + remoteFile);  
            try {  
                fsdis = fs.open(path);  
                IOUtils.copyBytes(fsdis, System.out, 4096, false);  
            } finally {  
                IOUtils.closeStream(fsdis);  
                fs.close();  
            }  
    }  
     /** 
      * 下载 hdfs上的文件 
      * @param conf 
      * @param uri 
      * @param remote 
      * @param local 
      * @throws IOException 
      */  
     public static void download(Configuration conf , String uri ,String remote, String local) throws IOException {  
            Path path = new Path(remote);  
            FileSystem fs = FileSystem.get(URI.create(uri), conf);  
            fs.copyToLocalFile(path, new Path(local));  
            System.out.println("download: from" + remote + " to " + local);  
            fs.close();  
    }  
     /** 
      * 删除文件或者文件夹 
      * @param conf 
      * @param uri 
      * @param filePath 
      * @throws IOException 
      */  
     public static void delete(Configuration conf , String uri,String filePath) throws IOException {  
            Path path = new Path(filePath);  
            FileSystem fs = FileSystem.get(URI.create(uri), conf);  
            fs.deleteOnExit(path);  
            System.out.println("Delete: " + filePath);  
            fs.close();  
    }  
     /** 
      * 查看目录下面的文件 
      * @param conf 
      * @param uri 
      * @param folder 
      * @throws IOException 
      */  
     public static  void ls(Configuration conf , String uri , String folder) throws IOException {  
            Path path = new Path(folder);  
            FileSystem fs = FileSystem.get(URI.create(uri), conf);  
            FileStatus[] list = fs.listStatus(path);  
            System.out.println("ls: " + folder);  
            System.out.println("==========================================================");  
            for (FileStatus f : list) {  
                System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(),f.isDirectory() , f.getLen());  
            }  
            System.out  
                    .println("==========================================================");  
            fs.close();  
    }  
  
        /** 
         *  
         * @param parentName 绝对路径地址 
         * @throws Exception 
         */  
        public static void checkDir(String uri,String parentName)   throws Exception{  
            //D:\file  
            Configuration conf = new Configuration();  
            File file = new File(parentName);  
            boolean flag = true;  
            while (flag)    {  
                //查出parentName下的所有文件  
                File[] fileNames = file.listFiles(new FileFilter());  
                if(fileNames != null)   {  
                    for (int i = 0; i < fileNames.length; i++) {  
                        File f = fileNames[i];  
                        //System.out.println("parent directory:"+f.getParent()+",file name:"+f.getName());  
                        System.out.println("parent directory:"+f.getParent().replace("\\", "/").substring(2)+",file name:"+f.getName());  
                        String remoteFolrd= "hdfs://192.168.0.173:8020/Workspace/houlinlin"+f.getParent().replace("\\", "/").substring(2);  
                        markDir(conf ,uri,remoteFolrd);  
                        copyFile(conf ,uri,f.getParent()+"\\"+f.getName(),remoteFolrd);  
                    }                 
                }  
                //查出parentName下的所有目录  
                File[] directories = file.listFiles(new DirectortyFilter());  
                if(directories != null) {  
                    for (int i = 0; i < directories.length; i++) {  
                        File dir = directories[i];  
                        //绝对路径  
                        String path =  dir.getAbsolutePath();  
                        //递归  
                        checkDir(uri,path);  
                    }  
                }  
                flag = false;  
            }  
        }  
       
}
评论列表
本站所提供的代码，版权归原作者所有，若有侵犯作者版权，请与我们联系，我们将立即删除或修改。谢谢!
本站所有代码发布及提供者。
试试其它关键字
同语言下
可能有用的
贡献的其它代码Label