学步园

public class ProcedureJob {
        public static class MapClass extends  Mapper<Object, Text, Text, Text>{
                private Path[] localFiles;
                private URI[] uris;
                @Override
                protected void setup(Context context) throws IOException,
                                InterruptedException {
                       localFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());
                       uris = DistributedCache.getCacheFiles(context.getConfiguration());
                       System.out.println(uris[0].toString()+"+++++++++++++++"+localFiles+"----------------");
                       if(uris==null || uris.length==0){
                               throw new IOException("Error reading file from distributed cache. No URIs found.");
                       }
                       FileSystem fs = FileSystem.get(URI.create("hdfs://namenode:9000"),context.getConfiguration());
                       Path localPath = new Path("~/abc.txt");
                       fs.copyToLocalFile(new Path(uris[0].toString()),localPath );
                       BufferedReader br = new BufferedReader(new FileReader(localPath.toString()));
                       String str="";
                       while((str=br.readLine())!=null){
                               System.out.println(str+"---------------------------------------------");
                       }
                       br.close();
                }

                protected void map(Object key, Text value, Context context)
                                throws IOException, InterruptedException {
                      context.write(new Text(key.toString()), new Text("1"));
                }
        }
        
        public static void main(String[] args) throws Exception {
                Configuration conf = new Configuration();
                
                DistributedCache.addCacheFile(new Path(args[0]).toUri(), conf);  
                
                Job job = new Job(conf,"Test");
                job.setJarByClass(ProcedureJob.class);
                FileInputFormat.setInputPaths(job,new Path(args[1]));
                FileOutputFormat.setOutputPath(job, new Path(args[2]));
                job.setJobName("Test");
                job.setMapperClass(MapClass.class);
                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(Text.class);
                job.setNumReduceTasks(0);
                job.waitForCompletion(true);
        }
}

说明：事先将本地文件上传到HDFS中，/data/pubic/test.txt。

注意点：
1.Job对象必须在使用configuration对象把缓存文件添加之后才能new，否则会无法添加，出现java.lang.NullPointerException。
2.在setup中获取Path[] localFiles = DistributedCache.getLocalCacheFiles(context.getConfiguration());得到的对象是null，可能版本问题。
3.在setup方法中获取文件系统,FileSystem fs = FileSystem.get(URI.create("hdfs://namenode:9000"),context.getConfiguration());
如果直接是FileSystem fs = FileSystem.get(context.getConfiguration());会出现java.lang.IllegalArgumentException: Wrong FS: hdfs://localhost:54310/user/Hadoop/b, expected: file:/// 异常。