【发布时间】：2016-04-15 13:49:01
【问题描述】：
这是我的脚本，它从 CSV 中获取 50k 条记录并对其进行 CRUD 操作。不幸的是，性能存在很多问题。
CSV 字段为：msisdn, resellerid, product
以下是执行的步骤：
- 读取 CSV。
- 创建临时表。
- 选择前1000条记录并用随机字符串标记状态以锁定它。
- 执行这些记录，同时另一个请求来到这个文件，然后在锁定状态下进行下一个 1000 批次。
- 它读取锁定状态记录并进行API调用。如果调用是肯定的，那么它会创建客户帐户的记录，否则继续下一个记录的 API 调用。
如果我们上传包含 1k、2k、最多 7k 条记录的 CSV，则代码可以正常工作。
当我们上传 10k 及以上的相同记录时，处于锁定状态的相同记录仍会被调用两次和多次，从而导致需要避免不必要的 API 调用。有时，当 API 调用成功时，即使没有创建客户帐户。
所以我特此粘贴代码，以便熟练的 PHP 和 Mysql 专家可以查看代码并提出改进意见。
<?php
/*********************************************************************************
** The contents of this file are subject to the crm CRM Public License Version 1.0
* ("License"); You may not use this file except in compliance with the License
* The Original Code is: crm CRM Open Source
* The Initial Developer of the Original Code is crm.
* Portions created by crm are Copyright (C) crm.
* All Rights Reserved.
*
********************************************************************************/
ini_set('max_execution_time', 30000);
set_time_limit(0);
global $adb, $log;
require_once('include/utils/utils.php');
require_once('include/database/PearDatabase.php');
include_once 'includes/main/WebUI.php';
$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer where `statusrecord` = 'inprogress'");
$noOfRows = $adb->num_rows($result);
if (!$noOfRows) {
$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer where `statusrecord` = 'new'");
$noOfRows = $adb->num_rows($result);
}
if (!$noOfRows) {
// add inner join with crm_crmentity for created date interval day for one day.
//$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer where `statusrecord` = 'closedwithfailure'");
$result = $adb->pquery("SELECT bulkcustomerid,billingarea,apilogid FROM crm_bulkcustomer inner join crm_crmentity on
crm_bulkcustomer.bulkcustomerid=crm_crmentity.crmid where `statusrecord` = 'closedwithfailure'");
$noOfRows = $adb->num_rows($result);
$isClosedWf = TRUE;
}
$process_data = 1000;
$log->debug('TOTAL NUMBEROFCSVS ' . $noOfRows);
$length = 9;
$randomString = substr(str_shuffle("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"), 0, $length);
if ($noOfRows > 0) { // for number of csv files .
for ($i = 0; $i < $noOfRows; $i++) {
$bulkcustomerid = $adb->query_result($result, $i, 'bulkcustomerid');
$apilogid = $adb->query_result($result, $i, 'apilogid'); // update the records for processing
$billingarea = $adb->query_result($result, $i, 'billingarea');
if (!$apilogid) {
$apilogid = 0;
}
$adb->query("UPDATE crm_bulkcustomer SET `statusrecord` = 'inprogress' where `bulkcustomerid` = '$bulkcustomerid'");
if ($isClosedWf) {
$log->debug('IN CLOSED WITH FAILURE PART');
//sleep(6800);
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '$randomString' where STATUS = '0' OR STATUS REGEXP '[a-z]' ORDER BY id ASC LIMIT $process_data");
$servicetype = "Prepaid";
$log->debug('CRON TASK - After validating Call Create Customer API for Record ID ' . $bulkcustomerid);
$log->debug('select msisdn,package from temp_csv_$bulkcustomerid where status=' . $randomString . ' limit' . $process_data . ' FOR UPDATE');
//$transaction = "START TRANSACTION;";
//$adb->pquery($transaction, array());
$result2 = $adb->query("select msisdn,package from temp_csv_$bulkcustomerid where status='$randomString' limit $process_data");
$noOfRows2 = $adb->num_rows($result2);
$log->debug('Total row for this table is:- ' . $noOfRows2);
if ($noOfRows2 > 0) {
for ($j = 0; $j < $noOfRows2; $j++) {
$msisdnnum = $adb->query_result($result2, $j, 'msisdn');
$package = $adb->query_result($result2, $j, 'package');
/***** commented MSISDN
$log->debug('CRON TASK - Call Create Customer API for Table ID temp_csv_' . $bulkcustomerid . 'and mobile number is' . $msisdnnum);
$log->debug("for IF part BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum,$apilogid)");
$result1 = BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum, $apilogid, $bulkcustomerid);
$message = $result1['message'];
$code = $result1['code'];
if ($code == 0) {
$log->debug('MESSAGE IS--for code 0 ' . $message . 'code is--' . $code);
$log->debug('Sucess UPDATE crm_bulkcustomer SET statusrecord = closed where bulkcustomerid=' . $bulkcustomerid);
******/
$bulkresult = BulkCustomer_CreateCustomer_Action::Create_Customer($msisdnnum, $bulkcustomerid, $package, $servicetype,$billingarea,$apilogid);
$log->debug('CUSTOMER WITH ALL ACCOUNTS CREATED SUCCESSFULLY FOR MOBILENO=' . $msisdnnum . 'and current j value is=' . $j . 'and no of records to parse are' . $noOfRows2);
$bulkcode = $bulkresult['code'];
$bulkmessage = $bulkresult['message'];
if ($bulkcode == '0') {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '1',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
} else {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-6',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
}
/****** search MSISDN } else {
$log->debug('MESSAGE IS-- for code -1 ' . $message . 'code is--' . $code);
$log->debug('Failure UPDATE temp_csv_' . $bulkcustomerid . 'SET status = Failed where msisdn =' . $msisdnnum);
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-9',remarks='$message' where `msisdn` = '$msisdnnum'");
}
search MSISDN *****/
$log->debug('j value is for loop' . $j);
}
}else{
$log->debug('No data to parse');
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closed' where `bulkcustomerid` = '$bulkcustomerid'");
}
} else { // for locking the columns with -7
$log->debug('IN PROGESS OR NEW PART');
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '$randomString' where `status` = '0' ORDER BY id ASC LIMIT $process_data");
$servicetype = "prepaid";
$log->debug('CRON TASK - After validating Call Create Customer API for Record ID ' . $bulkcustomerid);
$log->debug('select msisdn,package from temp_csv_$bulkcustomerid where status=' . $randomString . ' limit' . $process_data . 'FOR UPDATE');
//$transaction = "START TRANSACTION;";
//$adb->pquery($transaction, array());
$result2 = $adb->query("select msisdn,package from temp_csv_$bulkcustomerid where status='$randomString' limit $process_data");
$noOfRows2 = $adb->num_rows($result2);
$log->debug('Total row for this table is:- ' . $noOfRows2);
if ($noOfRows2 > 0) {
for ($j = 0; $j < $noOfRows2; $j++) {
$msisdnnum = $adb->query_result($result2, $j, 'msisdn');
$package = $adb->query_result($result2, $j, 'package');
/***** commented MSISDN
$log->debug("ELSE BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum,$apilogid)");
$result1 = BulkCustomer_SearchSim_Action::search_msisdn($msisdnnum, $apilogid, $bulkcustomerid);
$message = $result1['message'];
$code = $result1['code'];
if ($code == 0) {
$log->debug('MESSAGE IS--for code 0 ' . $message . 'code is--' . $code);
$log->debug('Sucess UPDATE crm_bulkcustomer SET statusrecord = closed where bulkcustomerid=' . $bulkcustomerid);
******/
$bulkresult = BulkCustomer_CreateCustomer_Action::Create_Customer($msisdnnum, $bulkcustomerid, $package, $servicetype,$billingarea,$apilogid);
$log->debug('CUSTOMER WITH ALL ACCOUNTS CREATED SUCCESSFULLY FOR MOBILENO=' . $msisdnnum . 'and current j value is=' . $j . 'and no of records to parse are' . $noOfRows2);
$bulkcode = $bulkresult['code'];
$bulkmessage = $bulkresult['message'];
if ($bulkcode == '0') {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '1',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
} else {
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-6',remarks='$bulkmessage' where `msisdn` = '$msisdnnum'");
}
/****** } else {
$log->debug('Failure UPDATE temp_csv_' . $bulkcustomerid . 'SET status = Failed where msisdn =' . $msisdnnum);
$adb->query("UPDATE temp_csv_$bulkcustomerid SET `status` = '-9',remarks='$message' where `msisdn` = '$msisdnnum'");
}
comments ends for MSISDN ****/
$log->debug('j value is for loop' . $j);
}
}else{
$log->debug('No data to parse inprogress or new part');
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closedwithfailure' where `bulkcustomerid` = '$bulkcustomerid'");
}
//commit
} // for else part
// close the csv file
// get count of sucess and failure.
$sucesscount = $adb->pquery("SELECT * FROM temp_csv_$bulkcustomerid where `status` = '1'"); // fetch record for new
$noofsucess = $adb->num_rows($sucesscount);
$adb->query("UPDATE `crm_bulkcustomer` SET `successcount` = '$noofsucess' where `bulkcustomerid` = '$bulkcustomerid'");
// failure count
$failurecount = $adb->pquery("SELECT * FROM temp_csv_$bulkcustomerid where status!='1'"); // fetch record for fails
$nooffailure = $adb->num_rows($failurecount);
$reprocesscount = $adb->pquery("SELECT * FROM temp_csv_$bulkcustomerid where STATUS = '0' OR STATUS REGEXP '[a-z]'");
// fetch record for fails
//$noofreprocesscount = 0;
$noofreprocesscount = $adb->num_rows($reprocesscount);
$log->debug("UPDATE `crm_bulkcustomer` SET `failurecount` = '$nooffailure' where `bulkcustomerid` = '$bulkcustomerid'");
if ($nooffailure == '0') {
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closed' where `bulkcustomerid` = '$bulkcustomerid'");
$adb->query("UPDATE `crm_bulkcustomer` SET `failurecount` = '' where `bulkcustomerid` = '$bulkcustomerid'");
} elseif ($noofreprocesscount) {
// check if it is being closed before all records are executed
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'inprogress' where `bulkcustomerid` = '$bulkcustomerid'");
} else {
$adb->query("UPDATE `crm_bulkcustomer` SET `failurecount` = '$nooffailure' where `bulkcustomerid` = '$bulkcustomerid'");
$adb->query("UPDATE `crm_bulkcustomer` SET `statusrecord` = 'closedwithfailure' where `bulkcustomerid` = '$bulkcustomerid'");
$filename = "$bulkcustomerid" . "-csv";
$file = fopen("tempcsv/$filename.csv", "w");
$val = array(
'MSISDN',
'ResellerID',
'Package',
'Remarks'
);
fputcsv($file, $val, ';', ' ');
$csvresult = $adb->query("select msisdn,package,resellerid,remarks from temp_csv_$bulkcustomerid where status!='1' ");
$csvrows = $adb->num_rows($csvresult);
$log->debug('Total row for this table in CSV PART is:- ' . $csvrows);
if ($csvrows > 0) {
for ($m = 0; $m < $csvrows; $m++) {
$msisdnnum = $adb->query_result($csvresult, $m, 'msisdn');
$package = $adb->query_result($csvresult, $m, 'package');
$resellerid = $adb->query_result($csvresult, $m, 'resellerid');
$remarks = $adb->query_result($csvresult, $m, 'remarks');
$remarks = preg_replace('/\s+/', '--', $remarks);
$arrayval = array(
"$msisdnnum",
"$resellerid",
"$package",
"$remarks"
);
fputcsv($file, $arrayval, ';', ' ');
}
}
}
}
} else {
$log->debug('No status with NEW or Pending of CSV RECORDS found');
}
?>
【问题讨论】：
-
你看过MySql函数LOAD DATA INFILE吗？您将能够在几秒钟内读取 100k 行的 csv 文件。它一直是我的数据导入朋友
-
我投票结束这个问题，因为这个问题属于codereview.stackexchange.com
-
@pierre，阅读现在不是问题，而是在问题中执行整个 crud 操作。所以，如果您可以查看代码并提出专家建议，那就太好了。
-
@riggs ，这可以帮助想要相同解决方案的人，投票反对某些问题将不是查询的解决方案。如果您有解决方案，请将其用于我们。
-
在问题被搁置之前，需要 4 个人同意我的观点。如果 5 个人都同意这个问题不是这个网站的主题，那么它几乎肯定不是主题。我确实建议您可以在哪里获得帮助，我会在那里发布您的问题，看看他们是否准备帮助您。另请注意，在 2 小时内，您没有收到任何答案，这应该告诉您一些事情。通常，无论白天还是晚上，您都会在 5 分钟内在这里得到某种答案