Azure DB 上 C# 中的 SQL 插入语句 - 运行速度非常慢

Question

我正在我们的网络应用程序中开发一个导入器。使用我目前拥有的代码，当您通过本地 SQL 服务器连接时，它运行良好且在合理范围内。我还在创建一个.sql 脚本，他们也可以下载它

示例 1

4 万条记录，8 列，从 1 分 30 秒到 2 分钟

当我将其移至生产和 Azure 应用服务时，它的运行速度非常慢。

示例 2

40k 条记录，8 列，从 15 分钟到 18 分钟

当前数据库设置为：定价层：标准 S2：50 个 DTU

代码如下：

using (var sqlConnection = new SqlConnection(connectionString))
{
    try
    {
        var generatedScriptFilePathInfo = GetImportGeneratedScriptFilePath(trackingInfo.UploadTempDirectoryPath, trackingInfo.FileDetail);

        using (FileStream fileStream = File.Create(generatedScriptFilePathInfo.GeneratedScriptFilePath))
        {
            using (StreamWriter writer = new StreamWriter(fileStream))
            {
                sqlConnection.Open();
                sqlTransaction = sqlConnection.BeginTransaction();
                        
                await writer.WriteLineAsync("/* Insert Scripts */").ConfigureAwait(false);

                foreach (var item in trackingInfo.InsertSqlScript)
                {
                    errorSqlScript = item;

                    using (var cmd = new SqlCommand(item, sqlConnection, sqlTransaction))
                    {
                        cmd.CommandTimeout = 800;
                        cmd.CommandType = CommandType.Text;
                        await cmd.ExecuteScalarAsync().ConfigureAwait(false);
                    }

                    currentRowLine++;
                    rowsProcessedUpdateEveryXCounter++;
                    rowsProcessedTotal++;

                    // append insert statement to the file
                    await writer.WriteLineAsync(item).ConfigureAwait(false);
                }

                // write out a couple of blank lines to separate insert statements from post scripts (if there are any)
                await writer.WriteLineAsync(string.Empty).ConfigureAwait(false);
                await writer.WriteLineAsync(string.Empty).ConfigureAwait(false);
            }
        }
    }
    catch (OverflowException exOverFlow)
    {
        sqlTransaction.Rollback();
        sqlTransaction.Dispose();
        trackingInfo.IsSuccessful = false;
        trackingInfo.ImportMetricUpdateError = new ImportMetricUpdateErrorDTO(trackingInfo.ImportMetricId)
                {
                    ErrorLineNbr = currentRowLine + 1, // add one to go ahead and count the record we are on to sync up with the file
                    ErrorMessage = string.Format(CultureInfo.CurrentCulture, "{0}", ImporterHelper.ArithmeticOperationOverflowFriendlyErrorText),
                    ErrorSQL = errorSqlScript,
                    RowsProcessed = currentRowLine
                };

        await LogImporterError(trackingInfo.FileDetail, exOverFlow.ToString(), currentUserId).ConfigureAwait(false);
        await UpdateImportAfterFailure(trackingInfo.ImportMetricId, exOverFlow.Message, currentUserId).ConfigureAwait(false);

        return trackingInfo;
    }
    catch (Exception ex)
    {
        sqlTransaction.Rollback();
        sqlTransaction.Dispose();

        trackingInfo.IsSuccessful = false;
        trackingInfo.ImportMetricUpdateError = new ImportMetricUpdateErrorDTO(trackingInfo.ImportMetricId)
                {
                    ErrorLineNbr = currentRowLine + 1, // add one to go ahead and count the record we are on to sync up with the file
                    ErrorMessage = string.Format(CultureInfo.CurrentCulture, "{0}", ex.Message),
                    ErrorSQL = errorSqlScript,
                    RowsProcessed = currentRowLine
                };

        await LogImporterError(trackingInfo.FileDetail, ex.ToString(), currentUserId).ConfigureAwait(false);
        await UpdateImportAfterFailure(trackingInfo.ImportMetricId, ex.Message, currentUserId).ConfigureAwait(false);

        return trackingInfo;
    }
}

问题

1. 有没有办法在 Azure 上加快这个速度？或者是升级DTUs的唯一方法？
2. 我们也在研究 SQL 批量复制。这是否会帮助或仍然会导致 Azure 运行缓慢：https://docs.microsoft.com/en-us/dotnet/api/system.data.sqlclient.sqlbulkcopy?redirectedfrom=MSDN&view=dotnet-plat-ext-5.0

期望的结果

在本地 SQL Server 数据库中运行时以相同的速度运行

Answer 1

现在，我更新了我的代码以根据记录的数量对插入语句进行批处理。如果记录数超过 10k，那么它将通过将总数除以 10 来对它们进行批处理。

这有助于在我们的 Azure 实例上实现 BIG TIME。我能够在 30 秒内添加 40k 条记录。我还认为其中一些问题是有多少不同的插槽在 Azure 上使用我们的应用服务。

由于用户需要导入更大的 excel 文件，我们稍后也可能会转移到 SQLBulkCopy。

感谢大家的帮助和见解！

        // apply the create table SQL script if found.
        if (string.IsNullOrWhiteSpace(trackingInfo.InsertSqlScript.ToString()) == false)
        {
            int? updateEveryXRecords = GetProcessedEveryXTimesForApplyingInsertStatementsValue(trackingInfo.FileDetail);
            trackingInfo.FileDetail = UpdateImportMetricStatus(trackingInfo.FileDetail, ImportMetricStatus.ApplyingInsertScripts, currentUserId);

            int rowsProcessedUpdateEveryXCounter = 0;
            int rowsProcessedTotal = 0;

            await UpdateImportMetricsRowsProcessed(trackingInfo.ImportMetricId, rowsProcessedTotal, trackingInfo.FileDetail.ImportMetricStatusHistories).ConfigureAwait(false);

            bool isBulkMode = trackingInfo.InsertSqlScript.Count >= 10000;

            await writer.WriteLineAsync("/* Insert Scripts */").ConfigureAwait(false);
            int insertCounter = 0;
            int bulkCounter = 0;
            int bulkProcessingAmount = 0;
            int lastInsertCounter = 0;
            if (isBulkMode == true)
            {
                bulkProcessingAmount = trackingInfo.InsertSqlScript.Count / 10;
            }

            await LogInsertBulkStatus(trackingInfo.FileDetail, isBulkMode, trackingInfo.InsertSqlScript.Count, bulkProcessingAmount, currentUserId).ConfigureAwait(false);

            StringBuilder sbInsertBulk = new StringBuilder();
            foreach (var item in trackingInfo.InsertSqlScript)
            {
                if (isBulkMode == false)
                {
                    errorSqlScript = item;

                    using (var cmd = new SqlCommand(item, sqlConnection, sqlTransaction))
                    {
                        cmd.CommandTimeout = 800;
                        cmd.CommandType = CommandType.Text;
                        await cmd.ExecuteScalarAsync().ConfigureAwait(false);
                    }

                    currentRowLine++;
                    rowsProcessedUpdateEveryXCounter++;
                    rowsProcessedTotal++;

                    // append insert statement to the file
                    await writer.WriteLineAsync(item).ConfigureAwait(false);

                    // Update database with the insert statement created count to alert the user of the status.
                    if (updateEveryXRecords.HasValue)
                    {
                        if (updateEveryXRecords.Value == rowsProcessedUpdateEveryXCounter)
                        {
                            await UpdateImportMetricsRowsProcessed(trackingInfo.ImportMetricId, rowsProcessedTotal, trackingInfo.FileDetail.ImportMetricStatusHistories).ConfigureAwait(false);
                            rowsProcessedUpdateEveryXCounter = 0;
                        }
                    }
                }
                else
                {
                    sbInsertBulk.AppendLine(item);

                    if (bulkCounter < bulkProcessingAmount)
                    {
                        errorSqlScript = string.Format(CultureInfo.CurrentCulture, "IsBulkMode is True | insertCounter = {0}", insertCounter);

                        bulkCounter++;
                    }
                    else
                    {
                        // display to the end user
                        errorSqlScript = string.Format(CultureInfo.CurrentCulture, "IsBulkMode is True | currentInsertCounter value = {0} | lastInsertCounter (insertCounter when the last bulk insert occurred): {1}", insertCounter, lastInsertCounter);

                        await ApplyBulkInsertStatements(sbInsertBulk, writer, sqlConnection, sqlTransaction, trackingInfo, rowsProcessedTotal).ConfigureAwait(false);

                        bulkCounter = 0;
                        sbInsertBulk.Clear();
                        lastInsertCounter = insertCounter;
                    }

                    rowsProcessedTotal++;
                }

                insertCounter++;
            }

            // get the remaining records after finishing the forEach insert statement
            if (isBulkMode == true)
            {
                await ApplyBulkInsertStatements(sbInsertBulk, writer, sqlConnection, sqlTransaction, trackingInfo, rowsProcessedTotal).ConfigureAwait(false);
            }
        }

        /// <summary>
        /// Applies the bulk insert statements.
        /// </summary>
        /// <param name="sbInsertBulk">The sb insert bulk.</param>
        /// <param name="wrtier">The wrtier.</param>
        /// <param name="sqlConnection">The SQL connection.</param>
        /// <param name="sqlTransaction">The SQL transaction.</param>
        /// <param name="trackingInfo">The tracking information.</param>
        /// <param name="rowsProcessedTotal">The rows processed total.</param>
        /// <returns>Task</returns>
        private async Task ApplyBulkInsertStatements(
                                                     StringBuilder sbInsertBulk,
                                                     StreamWriter wrtier,
                                                     SqlConnection sqlConnection,
                                                     SqlTransaction sqlTransaction,
                                                     ProcessImporterTrackingDTO trackingInfo,
                                                     int rowsProcessedTotal)
        {
            var bulkInsertStatements = sbInsertBulk.ToString();
            using (var cmd = new SqlCommand(bulkInsertStatements, sqlConnection, sqlTransaction))
            {
                cmd.CommandTimeout = 800;
                cmd.CommandType = CommandType.Text;
                await cmd.ExecuteScalarAsync().ConfigureAwait(false);
            }

            // append insert statement to the file
            await wrtier.WriteLineAsync(bulkInsertStatements).ConfigureAwait(false);

            // Update database with the insert statement created count to alert the user of the status.
            await UpdateImportMetricsRowsProcessed(trackingInfo.ImportMetricId, rowsProcessedTotal, trackingInfo.FileDetail.ImportMetricStatusHistories).ConfigureAwait(false);
        }