c# - 系统内存不足异常大txt文件放入数组

Question

以下代码适用于小型 txt 文件，但如果我们有大型 txt 文件，则会在 string[] array = File.ReadAllLines("hash.txt"); 处给出内存不足异常

hash.txt 文件大小为 500 mb

我尝试了互联网上的一些建议，但没有成功。

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
namespace Hash_Parser
{
    internal class Program
    {
        private static List<string> users = new List<string>();
        private static Dictionary<string, int> hash_original = new Dictionary<string, int>();
        private static List<string> hash_found = new List<string>();
        private static List<string> pass = new List<string>();
        private static string hash_path = "split.txt";
        private static void split()
        {
            Console.WriteLine("Splitting...");
            StreamWriter streamWriter = new StreamWriter("user.txt");
            StreamWriter streamWriter2 = new StreamWriter("hash.txt");
            string[] array = File.ReadAllLines(Program.hash_path);
            for (int i = 0; i < array.Length; i++)
            {
                string text = array[i];
                string[] array2 = text.Split(new char[]
               {
                   ':'
               }, 2);
                if (array2.Count<string>() >= 2)
                {
                    streamWriter.WriteLine(array2[0]);
                    streamWriter2.WriteLine(array2[1]);
                }
            }
            streamWriter.Close();
            streamWriter2.Close();
            Console.WriteLine("Saved as user.txt and hash.txt");
        }
        private static void populate()
        {
            Console.WriteLine("Populating lists...");
            Program.users.AddRange(File.ReadAllLines("user.txt"));
            Program.pass.AddRange(File.ReadAllLines("pass.txt"));
            Program.hash_found.AddRange(File.ReadAllLines("found.txt"));
            int num = 0;
            string[] array = File.ReadAllLines("hash.txt");
            for (int i = 0; i < array.Length; i++)
            {
                string key = array[i];
                Program.hash_original.Add(key, num);
                num++;
            }
        }
        private static void seek()
        {
            StreamWriter streamWriter = new StreamWriter("userpass.txt");
            int num = 0;
            int num2 = 100;
            foreach (string current in Program.hash_found)
            {
                if (Program.hash_original.ContainsKey(current))
                {
                    streamWriter.WriteLine(Program.users[Program.hash_original[current]] + ":" + Program.pass[num]);
                }
                num++;
                if (num >= num2)
                {
                    Console.Title = string.Concat(new object[]
                   {
                       "Processed: ",
                       num,
                       " : ",
                       Program.hash_found.Count
                   });
                    num2 += 1000;
                }
            }
            Console.Title = string.Concat(new object[]
           {
               "Processed: ",
               num,
               " : ",
               Program.hash_found.Count
           });
            streamWriter.Close();
        }
        private static void Main(string[] args)
        {
            Console.WriteLine("Split hash /split");
            Console.WriteLine("'split.txt'\n");
            Console.WriteLine("Parse hashes /parse");
            Console.WriteLine("'user.txt' | 'found.txt' | 'hash.txt' | 'pass.txt'");
            string a = Console.ReadLine();
            if (a == "/split")
            {
                Program.split();
            }
            else
            {
                if (a == "/parse")
                {
                    Program.populate();
                    Console.WriteLine("Processing...");
                    Stopwatch stopwatch = new Stopwatch();
                    stopwatch.Start();
                    Program.seek();
                    stopwatch.Stop();
                    Console.WriteLine("Saved as userpass.txt");
                    Console.WriteLine("Time elapsed: " + stopwatch.Elapsed);
                    Console.ReadKey();
                }
            }
        }
    }
}

感谢您的帮助。

Answer 1

试试这个代码:

foreach (var line in File.ReadLines(_filePath))
{
    //Don't put "line" into a list or collection.
    //Just make your processing on it.
}

引用的文本:只需使用 File.ReadLines 即可返回 IEnumerable，并且不会一次将所有行加载到内存中。

报价链接:https://stackoverflow.com/a/13416225/3041974

希望对您有所帮助。