c# - 改进我的代码以循环遍历 1000 个 csv 文件并更快地导入数百万条记录？

Question

我有 1000 个 csv 文件，每个 csv 文件包含 5500 行数据和 8 列。我下面的代码花了 5 分钟将 1 个 csv 文件导入 MySQL 数据库。我知道外面有很多这样的问题，我一直在研究它们。但是，根据我的代码，您是否有任何改进此导入过程的建议？

private void btn_fetchSharePrices_Click(object sender, EventArgs e)
{
    string[] fileCSV = Directory.GetFiles(sourceDirCSV);
    foreach (string csv in fileCSV)
    {
        try
        {
            string[] lines = File.ReadAllLines(csv);
            foreach (var line in lines)
            {
                var data = line.Split(new[] { ',' }, 8);
                DateTime prices_date = DateTime.Parse(data[0].Trim());
                DateTime prices_time = DateTime.Parse(data[1].Trim());
                string open = data[2].Trim();
                string high = data[3].Trim();
                string low = data[4].Trim();
                string close = data[5].Trim();
                int volume = int.Parse(data[6].Trim());
                int tickers_ticker_id = int.Parse(data[7].Trim());
                StoreRecord_FetchSharePrices(prices_date, prices_time, open, high, low, close, volume, tickers_ticker_id);
            }
        }
        catch (Exception ex)
        {
            MessageBox.Show(ex.Message);
        }  
    }
}

private void StoreRecord_FetchSharePrices(DateTime prices_date, DateTime prices_time, string open, string high, string low, string close, int volume, int tickers_ticker_id)
{
    using (var connection = new MySqlConnection(strProvider))
    using (var command = connection.CreateCommand())
    {
        connection.Open();
        command.CommandText = @"INSERT IGNORE INTO prices (Prices_Date, Prices_Time, Prices_Open, Prices_High, Prices_Low, Prices_Close, Prices_Volume, Tickers_Ticker_ID) VALUES (@Prices_Date, @Prices_Time, @Prices_Open, @Prices_High, @Prices_Low, @Prices_Close, @Prices_Volume, @Tickers_Ticker_ID)";
        command.Parameters.AddWithValue("@Prices_Date", prices_date);
        command.Parameters.AddWithValue("@Prices_Time", prices_time);
        command.Parameters.AddWithValue("@Prices_Open", open);
        command.Parameters.AddWithValue("@Prices_High", high);
        command.Parameters.AddWithValue("@Prices_Low", low);
        command.Parameters.AddWithValue("@Prices_Close", close);
        command.Parameters.AddWithValue("@Prices_Volume", volume);
        command.Parameters.AddWithValue("@Tickers_Ticker_ID", tickers_ticker_id);
        command.ExecuteNonQuery();
    }
}

Answer 1

更快？在 MySQL 中执行

load data local infile 'file.csv' into table table_name
 fields terminated by ','
 enclosed by '"'
 lines terminated by '\n'
 (column1, column2, column3,...)

对文件夹 Directory.EnumerateFiles(Folder_path) 中的文件运行 foreach 循环，并为每个具有完整文件路径的文件运行上述命令(代替 file.csv )