hadoop - 为什么最后一个 reducer 在合并步骤期间因 java 堆错误而停止

Question

我不断增加 reducer 的数量，我发现除了一个 reducer 之外的所有 reducer 都运行迅速并完成了它们的工作，最后一个 reducer 只是在合并步骤挂起，并在其 tasktracker 日志中显示以下消息:

Down to the last merge-pass, with 3 segments left of total size: 171207264 bytes

...在这个语句停留很长时间后，它抛出一个 java 堆错误并开始一些清理，但没有完成。

我将 child.opts 内存增加到 3.5GB(无法超过此限制)并压缩了 map 输出。

可能是什么原因？

驱动代码如下:

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    conf.set("mapred.task.timeout", "6000000");
    conf.set("mapred.compress.map.output", "true");
    Job job = new Job(conf, "FreebasePreprocess_Phase2");
    job.setNumReduceTasks(6);
    job.setJarByClass(FreebasePreprocess.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    FileInputFormat.addInputPath(job, new Path("/user/watsonuser/freebase_data100m120m_output"));
    FileOutputFormat.setOutputPath(job, new Path("/user/watsonuser/freebase_data100m120m_output_2"));

    job.waitForCompletion(true);
}

这是映射器:

import java.io.IOException;
import java.util.Arrays;
import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;


public class Map extends Mapper<LongWritable, Text, Text, Text>{

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 
{
    String[] entities = value.toString().split("\\t");
    String[] strings = {"/type/object/type", "/common/topic/notable_for", "/type/user/usergroup"};
    List<String> filteredPredicates = Arrays.asList(strings);

    FileSplit fileSplit = (FileSplit)context.getInputSplit();
    String filename = fileSplit.getPath().getName();
    //      System.out.println("File name "+filename);

    if(filename.startsWith("part-r")) {
        //      if(filename.equalsIgnoreCase("quad.tsv")) {
        //this is a quad dump file
        String name = null;
        String predicate = null;
        String oid = null;
        String outVal = null;
        String outKey = null;
        if(entities.length==3) {
            oid = entities[0].trim();
            predicate = entities[1].trim();
            name = entities[2].trim();

            /*if(predicate.contains("/type/object/name/lang"))
            {
                if(predicate.endsWith("/en")) 
                {*/
                /*outKey = sid;
                outVal = oid+"#-#-#-#"+"topic_name";
                context.write(new Text(outKey), new Text(outVal));*/
            /*  }
            }*/
                outKey = oid;
                outVal = predicate+"#-#-#-#"+name;
                context.write(new Text(outKey), new Text(outVal));

        }
    }

    else if(filename.equalsIgnoreCase("freebase-simple-topic-dump.tsv")) {
        //this is a simple topic dump file
        String sid = null;
        String name = null;
        String outKey = null;
        String outVal = null;
        if(entities.length>1) {
            sid = entities[0];
            name = entities[1];
            outKey = sid;
            outVal = name+"#-#-#-#"+"topic_name";
            context.write(new Text(outKey), new Text(outVal));
        }
    }
}

}

这里是 reducer

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class Reduce extends Reducer<Text, Text, Text, Text> 
{

public void reduce(Text key, Iterable<Text> values, Context context) 
        throws IOException, InterruptedException 
        {
            String name = null; 
            String sid = null;
            String predicate = null;
            String oid = null;
            String id = null;
            String outKey = null;
            String outVal = null;

            ArrayList<Text> valuesList = new ArrayList<Text>();
            Iterator<Text> ite = values.iterator();
            while(ite.hasNext()) {
                Text t = ite.next();
                Text txt = new Text();
                txt.set(t.toString());
                valuesList.add(txt);
                String[] entities = t.toString().split("#-#-#-#");
                if(entities[entities.length-1].equalsIgnoreCase("topic_name"))
                {
                    name = entities[0];
                }
            }

            for(int i=0; i<valuesList.size(); i++) { 
{ 

                Text t2 = valuesList.get(i);
                String[] entities = t2.toString().split("#-#-#-#");
                if(!entities[entities.length-1].contains("topic_name"))
                {
                    if(name!=null) {
                        outKey = entities[1]+"\t"+entities[0]+"\t"+name;
                    }
                    else {
                        outKey = entities[1]+"\t"+entities[0]+"\t"+key.toString();
                    }
                    context.write(new Text(outKey), null);
                }
            }
        }
}

Answer 1

我的猜测是您有一个包含大量值的键，而您的 reducer 中的以下行给您带来了问题:

valuesList.add(txt);

假设您有一个具有 100m 值的键，您正在尝试构建一个大小为 100m 的数组列表 - 在某个阶段，您的 reducer JVM 将耗尽内存。

您可能可以通过进行一些调试并检查永不结束的 reducer 的日志来确认这一点:

valuesList.add(txt);
if (valuesList.size() % 10000 == 0) {
  System.err.println(key + "\t" + valueList.size());
}