java批量生成sqoop任务代码,Java代码库,德仔网

代码语言
代码分类
【Java】 java批量生成sqoop任务
作者: / 发布于2017/8/29/ 256
用Java写了类，用于批量生成sqoop脚本任务，sqoop一行行的写不得不说很累
package data.clear;

import java.io.File;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import org.apache.commons.io.FileUtils;
import java.io.FileOutputStream;
import   java.io.PrintStream;


public class GeneratePartitions {
	public static void main(String[] args) throws IOException {
		List<String> lines = FileUtils.readLines(new File("d:\\wilson.zhou\\Desktop\\testtest.txt"));
		String template = "sqoop import --connect jdbc:postgresql://10.1.2.130:5432/xmo_dw --table %s --fields-terminated-by '\\t' --null-string '\\\\N' --null-non-string '\\\\N' --target-dir /staging/db_backup/greenplum/%s/%s  --username xmo_summarizer --password YCt452uz";
		String gp_table = "";
		String table = "";
		String partition = "";
		  PrintStream ps=new PrintStream(new FileOutputStream("d:\\wilson.zhou\\Desktop\\adgrouphists_2015.sh"));  
		  System.setOut(ps);  
		for(String line: lines){
			line = line.trim();
			gp_table = line;
			if(line != null && !line.contains("2016") &&line.contains("2015")){
				if(line.startsWith("rtb_summary_ext")){
					table = "rtb_summary_ext";
					partition = line.substring(line.lastIndexOf("_")+1);
					
					
					partition = partition.replace("p", "");
					
//					System.out.println("echo `date '+%m-%d %-H:%-M:%-S'` "+gp_table);
//					System.out.println("mkdir -p /mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"");
//					System.out.println("psql xmo_dw -c \"copy xmo_dw."+gp_table+" to '/mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"/"+partition+".csv'\"");

					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition)  +" -m 1 --fetch-size 50000");
				}else if(line.startsWith("rtb_summary")){
					table = "rtb_summary";
					partition = line.substring(line.lastIndexOf("_20")+1);
					
					partition = partition.replace("p", "");
					
//					System.out.println("echo `date '+%m-%d %-H:%-M:%-S'` "+gp_table);
//					System.out.println("mkdir -p /mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"");
//					System.out.println("psql xmo_dw -c \"copy xmo_dw."+gp_table+" to '/mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"/"+partition+".csv'\"");
					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition)  +" -m 1 --fetch-size 50000");
				}
				else if(line.startsWith("eventdatas_old")){
					table = "eventdatas_old";
					partition = line.substring(line.lastIndexOf("_20")+1);
					
					partition = partition.replace("p", "");
					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition) + " --split-by id --fetch-size 50000");
					
				}else if(line.startsWith("eventdatas_1")){
						table = "eventdatas";
						partition = line.substring(line.lastIndexOf("_p")+1);
						
						partition = partition.replace("p", "");
						System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
						System.out.println(String.format(template, gp_table, table, partition) + " --split-by id --fetch-size 50000");
						
				
				}else if(line.startsWith("rtb_datas")){
					table = "rtb_datas";
					partition = line.substring(line.lastIndexOf("_p")+2);
				
					partition = partition.replace("p", "");
					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition)  + " --split-by id --fetch-size 50000");
				}else if(line.startsWith("clickdatas")){
					table = "clickdatas";
					
					partition = line.substring(line.lastIndexOf("_")+1);
				
					partition = partition.replace("p", "");
//					System.out.println("echo "+gp_table);
//					System.out.println("mkdir -p /mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"");
//					System.out.println("psql xmo_dw -c \"copy xmo_dw."+gp_table+" to '/mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"/"+partition+".csv'\"");
					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition)  + " --split-by id --fetch-size 50000");
	
				}else if(line.startsWith("imageviews")){
					table = "imageviews";
					partition = line.substring(line.lastIndexOf("_p")+2);
				
					partition = partition.replace("p", "");
					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition)  + " --split-by id --fetch-size 50000");
			
				}else if(line.startsWith("adgrouphists")){
					table = "adgrouphists";
					partition = line.substring(line.lastIndexOf("_")+1);
					
					partition = partition.replace("p", "");
					
//					System.out.println("echo `date '+%m-%d %-H:%-M:%-S'` "+gp_table);
//					System.out.println("mkdir -p /mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"");
//					System.out.println("psql xmo_dw -c \"copy xmo_dw."+gp_table+" to '/mapr/hkidc.hadoop.iclick/staging/db_backup/greenplum/"+table+"/"+partition+"/"+partition+".csv'\"");

					System.out.println("echo \""+gp_table+"\t"+table+"\t" + partition+"\"");
					System.out.println(String.format(template, gp_table, table, partition)  + " --split-by id --fetch-size 50000");
			
				}
				
			}
		}
	}
}
评论列表
本站所提供的代码，版权归原作者所有，若有侵犯作者版权，请与我们联系，我们将立即删除或修改。谢谢!
本站所有代码发布及提供者。
试试其它关键字
同语言下
可能有用的
贡献的其它代码Label