hadoop - 独立的 map reduce 作业一个接一个地执行

Question

是否有可能执行独立的 map reduce 作业(不在 reducer 输出的链接中

1. 成为映射器的输入。
2. 可以一个接一个地执行。

Answer 1

在你的驱动代码中调用两个方法runfirstjob,runsecondjob.就像这样。这只是一个提示，根据你的需要做修改

public class ExerciseDriver {


static Configuration conf;

public static void main(String[] args) throws Exception {

    ExerciseDriver ED = new ExerciseDriver();
    conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    if(args.length < 4) {
        System.out.println("Too few arguments. Arguments should be:  <hdfs input folder> <hdfs output folder> <N configurable Integer Value>");
        System.exit(0);
    }

    String pathin1stmr = args[0];
    String pathout1stmr = args[1];
    String pathin2ndmr = args[2];
    String pathout2ndmr = args[3];

    ED.runFirstJob(pathin1stmr, pathout1stmr);

    ED.runSecondJob(pathin2ndmr, pathout2ndmr);

}

public int runFirstJob(String pathin, String pathout)  

 throws Exception {

    Job job = new Job(conf);
    job.setJarByClass(ExerciseDriver.class);
    job.setMapperClass(ExerciseMapper1.class);
    job.setCombinerClass(ExerciseCombiner.class);
    job.setReducerClass(ExerciseReducer1.class);
    job.setInputFormatClass(ParagrapghInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class); 
    FileInputFormat.addInputPath(job, new Path(pathin));
    FileOutputFormat.setOutputPath(job, new Path(pathout));

   job.submit();  

   job.getMaxMapAttempts();

   /*
   JobContextImpl jc = new JobContextImpl();
   TaskReport[] maps = jobclient.getMapTaskReports(job.getJobID());

    */

    boolean success = job.waitForCompletion(true);
    return success ? 0 : -1;

}

  public int runSecondJob(String pathin, String pathout) throws Exception { 
    Job job = new Job(conf);
    job.setJarByClass(ExerciseDriver.class);
    job.setMapperClass(ExerciseMapper2.class);
    job.setReducerClass(ExerciseReducer2.class);
    job.setInputFormatClass(KeyValueTextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);    
    FileInputFormat.addInputPath(job,new Path(pathin));
    FileOutputFormat.setOutputPath(job, new Path(pathout));
    boolean success = job.waitForCompletion(true);
    return success ? 0 : -1;
}

 }