MYSQL插入很慢

Question

我正在将一个包含大约 900000 行的数据表插入到 MySQL 数据库中，现在 13 小时后，我的程序已插入 185000 行。我觉得那太慢了^^

这是我如何将 DataTable 插入 MYSQL DB 的代码

//data is typeof DataTable and filled with my data.

using (System.Data.SqlClient.SqlConnection srcConnection = new SqlConnection(Settings.Default.SourceConnection))
using (MySql.Data.MySqlClient.MySqlConnection dstConnection = new MySqlConnection(Settings.Default.TargetConnection))
{
    if (dstConnection.State != ConnectionState.Open)
        dstConnection.Open();
    if (srcConnection.State != ConnectionState.Open)
        srcConnection.Open();

    var builder = SqlBuilder(tableName, data)

    foreach (DataRow toCopy in data)
    {
        using (var cmdUpdate = builder.GenerateUpdateCommand(dstConnection, toCopy))
        {
            if (cmdUpdate.ExecuteNonQuery() == 0)
            {
                using (var cmdInsert = builder.GenerateInsertCommand(dstConnection, toCopy))
                {
                    cmdInsert.ExecuteNonQuery;
                }
            }
        }
    }
}

builder 是我自己的类：

public class SqlBuilder
{
    private readonly string _tableName;
    private readonly DataColumnCollection _columns;
    private readonly List<string> _columnStrings = new List<string>();
    private DataColumn _primaryDataColumn;
    private readonly Logger _logger = LogManager.GetLogger(typeof(SqlBuilder).FullName);

    public SqlBuilder(string tableName, DataTable table)
    {
        if (tableName == null) throw new ArgumentNullException("tableName");
        if (table == null) throw new ArgumentNullException("table");
        if (table.PrimaryKey.Count() != 1) throw new ArgumentException("Only one primary key supported", "table");

        _tableName = tableName;
        _columns = table.Columns;
        _primaryDataColumn = table.PrimaryKey[0];

        foreach (DataColumn column in _columns)
            _columnStrings.Add(column.ColumnName);
    }

    public SqlBuilder()
    {
    }

    public string TableName
    {
        get { return _tableName; }
    }

    public String[] ColumnStrings
    {
        get { return _columnStrings.ToArray(); }
    }


    public IDbCommand GenerateCountCommand(IDbConnection con)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = String.Format("SELECT COUNT(*) FROM {0}", TableName);
        return result;
    }

    public IDbCommand GenerateDeleteTableCommand(IDbConnection con)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = String.Format("DELETE FROM {0}", TableName);
        return result;
    }

    public IDbCommand GenerateDeleteChangeLogTableCommand(IDbConnection con)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = String.Format("DELETE FROM ChangeLog");
        return result;
    }

    public String GenerateInsertCommandText()
    {
        String result;

        result = String.Format(
            "INSERT INTO {0} ({1}) VALUES ({2})",
            TableName,
            String.Join(",", _columnStrings.ToArray()),
            String.Join(",", _columnStrings.Select(s => "@" + s).ToArray()));

        return result;
    }

    public string GenerateUpdateCommandText()
    {
        String result;

        result = String.Format(
            "UPDATE {0} SET {1} WHERE {2}",
            TableName,
            String.Join(", ", _columnStrings.Where((s =>!_primaryDataColumn.ColumnName.Equals(s))).Select(s => s+"=@" + s).ToArray()),
            _primaryDataColumn.ColumnName+"=@"+_primaryDataColumn.ColumnName
            );

        return result;
    }

    public string GenerateDeleteCommandText(int id)
    {
        string result = String.Format("DELETE FROM {0} WHERE {1} = {2}", _tableName , _columnStrings[0], id);
        return result;
    }

    public IDbCommand GenerateInsertCommand(IDbConnection con, DataRow row)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = GenerateInsertCommandText();

        result = FillParameters(row, result);

        if (_logger.IsTraceEnabled) _logger.Trace(result.CommandText);

        return result;
    }

    public IDbCommand GenerateUpdateCommand(IDbConnection connection, DataRow dataRow)
    {
        IDbCommand result = connection.CreateCommand();
        result.CommandText = GenerateUpdateCommandText();

        result = FillParameters(dataRow, result);

        if (_logger.IsTraceEnabled) _logger.Trace(result.CommandText);

        return result;
    }

    public IDbCommand GenerateDeleteCommand(IDbConnection con, int id)
    {
        IDbCommand result = con.CreateCommand();

        result.CommandText = GenerateDeleteCommandText(id);

        return result;
    }

    private IDbCommand FillParameters(DataRow row, IDbCommand result)
    {
        foreach (var curColumn in _columnStrings)
        {
            IDbDataParameter parameter = result.CreateParameter();
            parameter.ParameterName = curColumn;
            parameter.Value = row[curColumn];
            result.Parameters.Add(parameter);
        }

        return result;
    }
}

任何人知道如何使这个更快？？？

Answer 1

在一个语句中插入多行，如下所示：

INSERT INTO table (...) VALUES 
(values for first record),
(values for second record),
...

您可能可以在这样的一条语句中插入几十行，具体取决于记录大小。

Answer 2

我在类似的参数构建过程中发现的是设置参数的“SourceColumn”...在您的“FillParameters”方法中尝试添加

parameter.SourceColumn = curColumn;

然后，在您的 INSERT 例程中，而不是

foreach( DataRow )
   Build The Insert Command
      Execute Insert

改成

Build The Insert Command
  ForEach( DataRow )
      Execute Insert

这样，您不必一遍又一遍地构建命令。该参数会转到正在处理的数据行的源列并为您拉入。

当我编写我的版本时，更新和删除也是如此。我预先构建了命令和参数，并在我正在使用的 DataTable 期间保留它们。

因此，您实际上可以在 foreach(DataRow) 之前构建插入和更新命令，然后直接使用它们。一遍又一遍。显然，在假设它会全部通过之前，只测试上传几条记录以确认它是否适用于您的其他记录。