java - Java Eclipse 内存不足

Question

public class Functions {

public Functions (){

}

Map<String, Set<String>> task() throws Exception{

    System.out.println("start reading for strain file");

    String value= "/home/charles/Documents/mmp.txt";
    File file = new File(value);


    FileInputStream fin = new FileInputStream(file);

    FileChannel fc = fin.getChannel();
    MappedByteBuffer mapByteBuff = fc.map(FileChannel.MapMode.READ_ONLY, 0, file.length());

    byte[] buffer= new byte[(int)fc.size()];
    mapByteBuff.get(buffer);
    BufferedReader br=new BufferedReader(new InputStreamReader(new ByteArrayInputStream(buffer)));
    Map<String, Set<String>> strainHashMap = new HashMap<String, Set<String>>(824*12+1,2);


    String line = br.readLine();

    //  String delim="[\t]";
    String[] tokens=new String[3];
    //      String delim="[\t]";

    for (line = br.readLine(); line != null; line = br.readLine()) {

        //          tokens=new String[3];
        tokens=line.split("[\t]",3);
                     //here is where I am stuck.. I get java out of heap memory

        Set<String> strainSet = strainHashMap.get(tokens[1]);
        if(strainSet==null){
            strainSet = new HashSet<String>();
            strainSet.add(tokens[2]);
            strainHashMap.put(tokens[1],strainSet);
        }
        else{
            strainSet.add(tokens[2]);
            strainHashMap.put(tokens[1], strainSet);
        }


        if(strainHashMap.size() % 600000 ==0){
            System.out.println("strain"+strainHashMap.size());
        }


        if(strainHashMap.size() % 610000 ==0){
            System.out.println("strain"+strainHashMap.size());
        }

        if(strainHashMap.size() % 620000 ==0){
            System.out.println("strain"+strainHashMap.size());
        }


        if(strainHashMap.size() % 650000 ==0){
            System.out.println("strain"+strainHashMap.size());
        }

        if(strainHashMap.size() % 700000 ==0){
            System.out.println("strain"+strainHashMap.size());
        }


        if(strainHashMap.size() % 851000 ==0){
            System.out.println("strain"+strainHashMap.size());
        }

    }

    fc.close();
    fin.close();
    System.out.println("The hash stain is " + strainHashMap.size());

    //  System.out.println("The Genotype is " + mapGenotype.size());

    System.out.println("moving out of strain function");
    return strainHashMap;

}

public Map<String, Genotype> genotype() throws Exception {
    // TODO Auto-generated method stub

    System.out.println("entering genotype data function");
    // TODO Auto-generated method stub
    String value= "/home/charles/Documents/mmp.txt";
    File file = new File(value);
    FileInputStream fin = new FileInputStream(file);
    FileChannel fc = fin.getChannel();
    MappedByteBuffer mapByteBuff = fc.map(FileChannel.MapMode.READ_ONLY, 0, file.length()); 

    byte[] buffer= new byte[(int)fc.size()];
    mapByteBuff.get(buffer);

    BufferedReader br=new BufferedReader(new InputStreamReader(new ByteArrayInputStream(buffer)));
    Map<String, Genotype> mapGenotype = new HashMap<String, Genotype>(824*12+1,2);

    String line = br.readLine();
    //  String delim="[\t]";
    String[] tokens=new String[8];
    //  String delim="[\t]";

    for (line = br.readLine(); line != null; line = br.readLine()) {
        tokens=new String[8];
        tokens=line.split("[\t]",8);

        //      tokens=line.split(delim);
        //      Genotype genotypeObject = new Genotype();

        //      genotypeObject.setChromsomeName(tokens[3]);

        //      genotypeObject.setMutation(tokens[6]);

        //      genotypeObject.setPosition(Double.parseDouble(tokens[4]));
        //genotypeObject.setPosition(Double.parseDouble(tokens[4]));

        //      genotypeObject.setRef(tokens[5]);

        if(!mapGenotype.containsKey(tokens[1])){


            Genotype genotypeObject = new Genotype();

            genotypeObject.setChromsomeName(tokens[3]);

            genotypeObject.setMutation(tokens[6]);

            genotypeObject.setPosition(Double.parseDouble(tokens[4]));
            //genotypeObject.setPosition(Double.parseDouble(tokens[4]));

            genotypeObject.setRef(tokens[5]);
            mapGenotype.put(tokens[1], genotypeObject);

        }

        if(mapGenotype.size() % 700000 ==0){
            System.out.println(mapGenotype.size());
        }


        if(mapGenotype.size() % 650000 ==0){
            System.out.println(mapGenotype.size());
        }

        if(mapGenotype.size() % 851000 ==0){
            System.out.println(mapGenotype.size());
        }

    }
    fc.close();
    fin.close();
    System.out.println("leaving Genotype method of function");
    return mapGenotype;

}

}

奇怪的是，在task 方法中hash map 添加了650000 个值后，内存不足。 我用类似的文件运行相同的方法，它工作得很好 genotype。

我使用的是 jdk 1.6 Ubuntu 12.04，Eclipse 开普勒。 .ini 文件有 -XX:MaxPermSize=256m -Xms740m -Xmx1412m

Run as configuration 如下: -Xms912m -Xmx2124m 交换内存超过 3.5 GB。

我曾经能够在某个时候完成这项任务，但不幸的是，我丢失了所有数据，我正在尝试重新设置所有这些。 我花了很多时间来解决这个问题，但似乎没有任何帮助。

这是我得到的错误:Exception in thread "main"java.lang.OutOfMemoryError: Java heap space

Answer 1

您的 JVM 设置对我来说似乎很慷慨。我看不出让它们变大会有什么帮助。

我是否正确理解您的代码 - 您是否将整个文件读入内存，然后遍历所有行以解析 Map 中的条目？

如果是，我会重写它一次只在内存中保留一行:读取一行，解析它，添加到 map ，重复。

运行 VisualVM查看运行时内存消耗的位置。这可能会让人大开眼界。