瀏覽代碼

hdfs.conf.path support comma-separated list

zqq90 8 年之前
父節點
當前提交
28200731e1

+ 3 - 1
hdata-hdfs/src/main/java/com/github/stuxuhai/hdata/plugin/reader/hdfs/HDFSReader.java

@@ -61,7 +61,9 @@ public class HDFSReader extends Reader {
 	public void execute(RecordCollector recordCollector) {
 		Configuration conf = new Configuration();
 		if (readerConfig.containsKey(HDFSReaderProperties.HDFS_CONF_PATH)) {
-			conf.addResource(new Path("file://" + readerConfig.getString(HDFSReaderProperties.HDFS_CONF_PATH)));
+      for (String path: readerConfig.getString(HDFSReaderProperties.HDFS_CONF_PATH).split(",")) {
+        conf.addResource(new Path("file://" + path));
+      }
 		}
 
 		CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf);

+ 3 - 1
hdata-hdfs/src/main/java/com/github/stuxuhai/hdata/plugin/reader/hdfs/HDFSSplitter.java

@@ -45,7 +45,9 @@ public class HDFSSplitter extends Splitter {
 
 		Configuration conf = new Configuration();
 		if (readerConfig.containsKey(HDFSReaderProperties.HDFS_CONF_PATH)) {
-			conf.addResource(new Path("file://" + readerConfig.getString(HDFSReaderProperties.HDFS_CONF_PATH)));
+      for (String path: readerConfig.getString(HDFSReaderProperties.HDFS_CONF_PATH).split(",")) {
+        conf.addResource(new Path("file://" + path));
+      }
 		}
 		try {
 			FileSystem fs = dir.getFileSystem(conf);

+ 3 - 1
hdata-hdfs/src/main/java/com/github/stuxuhai/hdata/plugin/writer/hdfs/HDFSWriter.java

@@ -154,7 +154,9 @@ public class HDFSWriter extends Writer {
 			Path hdfsPath = createFilePath(fileNum++);
 			conf = new Configuration();
 			if (writerConfig.containsKey(HDFSWriterProperties.HDFS_CONF_PATH)) {
-				conf.addResource(new Path("file://" + writerConfig.getString(HDFSWriterProperties.HDFS_CONF_PATH)));
+        for (String path: writerConfig.getString(HDFSWriterProperties.HDFS_CONF_PATH).split(",")) {
+          conf.addResource(new Path("file://" + path));
+        }
 			}
 
 			try {