java - 空指针异常Hadoop

Question

您好，我目前正在编写一个 map reduce 作业来运行包含图像的 hdfs 并记录输出，其中文件包含或不包含红色。我目前遇到 NullPointerException 错误，我似乎无法弄清楚它来自哪里。下面是我的代码

{

package hipi.examples.redfilter;


import java.awt.Color;
import java.awt.image.BufferedImage;
import java.io.IOException;

import javax.imageio.ImageIO;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class RedFilter extends Configured implements Tool{
    private static String inputPath;
    public static class RedFilterMapper extends Mapper<NullWritable, BufferedImage, Text, BooleanWritable> {


        public void map(IntWritable key, BufferedImage currImage, Context content) throws IOException, InterruptedException
        {
            System.out.println("I am in the mapper method");
             FileSystem fs = FileSystem.get(new Configuration());
             FileStatus[] status = fs.listStatus(new Path(inputPath));
             System.out.println("I am in the mapper method");

             for (int i=0;i<status.length;i++){
                 String fileName = fs.open(status[i].getPath()).toString();
                  currImage= ImageIO.read(fs.open(status[i].getPath()));


               int width = currImage.getWidth();
               int height = currImage.getHeight();  
               boolean exit = false;

               for(int x = 0; x < width; x++) {
                   for(int y = 0; y < height; y++) {
                       Color c = new Color(currImage.getRGB(x, y));
                       int red = c.getRed();

                       if ( red > 200 && c.getBlue() < 100 && c.getGreen() < 100) {
                           content.write(new Text(fileName), new BooleanWritable(true));
                           exit = true;
                           break;

                       }

                   }
                   if(exit) {
                       break;
                   }
               }
               if(!exit) {
                 content.write(new Text(fileName), new BooleanWritable(false)); 
               }


         }

        }       

    }

    public int run(String[] args) throws Exception {
        Configuration conf = new Configuration();
        if (args.length < 3) {
            System.out.println("Usage: dumphib <input hib> <outputdir>");
            System.exit(0);
        }
        inputPath = args[1];
        String outputPath = args[2];

        Job job = new Job(conf, "refilter");
        System.out.println("Jar Files r being set");
        job.setJarByClass(RedFilter.class);
        System.out.println("Mapper is being set");
        job.setMapperClass(RedFilterMapper.class);


        // Set formats
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Boolean.class);
//      job.setInputFormatClass(ImageBundleInputFormat.class);

        // Set out/in paths
        removeDir(outputPath, conf);
        FileOutputFormat.setOutputPath(job, new Path(outputPath));
        FileInputFormat.setInputPaths(job, new Path(inputPath));    

        job.setNumReduceTasks(1);
        System.exit(job.waitForCompletion(true) ? 0 : 1);
        return 0;

    }

    public static void main(String[] args) throws Exception {
        int exitCode = ToolRunner.run(new RedFilter(), args);
        System.exit(exitCode);
    }

    public static void removeDir(String path, Configuration conf) throws IOException {
        Path output_path = new Path(path);

        FileSystem fs = FileSystem.get(conf);

        if (fs.exists(output_path)) {
            fs.delete(output_path, true);
        }
    }


}

}

这是作业跟踪器的日志结果

{

stdout logs


stderr logs
2014-11-23 23:25:19.979 java[6176:1003] Unable to load realm info from SCDynamicStore


syslog logs
2014-11-23 23:25:20,740 WARN org.apache.hadoop.util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2014-11-23 23:25:21,569 WARN org.apache.hadoop.metrics2.impl.MetricsSystemImpl: Source name ugi already exists!
2014-11-23 23:25:21,727 INFO org.apache.hadoop.mapred.Task:  Using ResourceCalculatorPlugin : null
2014-11-23 23:25:21,759 INFO org.apache.hadoop.mapred.MapTask: Processing split: hdfs://localhost/user/oladotunopasina/redfilterinput/31b5ea5d982cf2b8b4ce27744d812d285b9e3.jpg:0+684033
2014-11-23 23:25:21,778 INFO org.apache.hadoop.mapred.MapTask: io.sort.mb = 100
2014-11-23 23:25:22,119 INFO org.apache.hadoop.mapred.MapTask: data buffer = 79691776/99614720
2014-11-23 23:25:22,119 INFO org.apache.hadoop.mapred.MapTask: record buffer = 262144/327680
2014-11-23 23:25:22,222 INFO org.apache.hadoop.mapred.TaskLogsTruncater: Initializing logs' truncater with mapRetainSize=-1 and reduceRetainSize=-1
2014-11-23 23:25:22,481 WARN org.apache.hadoop.mapred.Child: Error running child
java.lang.NullPointerException
    at org.apache.hadoop.io.serializer.SerializationFactory.getSerializer(SerializationFactory.java:73)
    at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.<init>(MapTask.java:970)
    at org.apache.hadoop.mapred.MapTask$NewOutputCollector.<init>(MapTask.java:673)
    at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:756)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:364)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:394)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1190)
    at org.apache.hadoop.mapred.Child.main(Child.java:249)
2014-11-23 23:25:22,485 INFO org.apache.hadoop.mapred.Task: Runnning cleanup for the task

}

请帮忙。

Answer 1

我在一行中发现了一个错误:

job.setOutputValueClass(Boolean.class);

您应该将其替换为:

job.setOutputValueClass(BooleanWritable.class);

我认为 hadoop 在找不到 Boolean 对象的序列化程序时会失败。