【发布时间】:2011-06-21 03:31:17
【问题描述】:
在我的网站中,我使用线程在后台执行重复过程。现在,更新数据库的过程大约需要 30 到 45 秒。
该函数基本上执行以下操作:
1) 从网站下载 CSV。
2) 解析 CSV 并更新数据库表。
我想优化性能并将所需时间减少到 15 秒。
我该怎么做?
编辑:
代码如下:
// MAIN FUNCTION IN THE THREAD
private static void downloadAndParse()
{
NewHive.MyServ newServe = new NewHive.MyServ();
NewHive.CsvDownload newService = new NewHive.CsvDownload();
//NewHive.MyServ newServe = new NewHive.MyServ();
string downloadSuccess = newService.CsvDownloader();
if (downloadSuccess == "Success")
{
string parseSuccess = newService.CsvParser();
}
newServe.updateOthersInPosition();
}
//CSV DOWNLOAD FUNCTION
public string CsvDownloader()
{
Byte[] inBuf = null;
// HttpWebRequest wr = Convert.ChangeType(WebRequestFactory.Create("http://finance.yahoo.com/d/quotes.csv?s=RHT+MSFT&f=sb2b3jk"),HttpWebRequest);
// HttpWebResponse ws = Convert.ChangeType(wr.GetResponse(),HttpWebResponse);
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create("http://download.finance.yahoo.com/d/quotes.csv?s=INDU+INDU+^N225+^GSPC+^GDAXI+^FCHI+^HSI+^IXIC+^STOXX50E+^FTSE&f=l1bd14na");
HttpWebResponse ws = (HttpWebResponse)wr.GetResponse();
Stream str = ws.GetResponseStream();
inBuf = new Byte[100000000];
int bytesToRead = (int)inBuf.Length;
int bytesRead=0;
while(bytesToRead>0)
{
int n = str.Read(inBuf,bytesRead,bytesToRead);
if(n==0)
{
break;
}
bytesRead += n;
bytesToRead -= n;
}
FileStream fstr = new FileStream("D:\\Hosting\\7312812\\html\\News.csv", FileMode.Create, FileAccess.Write);
// FileStream fstr = new FileStream("News.csv", FileMode.Create, FileAccess.Write);
// FileStream fstr = new FileStream("C:\\VSS Working Folder\\27 Jan 11 NewHive\\NewHive\\CSV\\new.csv", FileMode.Create, FileAccess.Write);
fstr.Write(inBuf,0,bytesRead);
str.Close();
fstr.Close();
return "Success";
}
//CSV PARSER FUNCTION
public string CsvParser()
{
int _nNrRowsProccessed = 0;
string connectionString = @"Driver={Microsoft Text Driver (*.txt; *.csv)};Dbq=" + ConfigurationManager.AppSettings["CSVFolder"] + ";";
OdbcConnection conn = new OdbcConnection(connectionString);
try
{
conn.Open();
string strFileName = ConfigurationManager.AppSettings["CSVFile"];
string strSQL = "Select * from " + strFileName;
OdbcCommand cmd = new OdbcCommand();
cmd.Connection = conn;
cmd.CommandText = strSQL;
cmd.CommandType = CommandType.Text;
OdbcDataReader reader = cmd.ExecuteReader();
NewHive.MyServ newService = new NewHive.MyServ();
// MasterCalendar_DB.OpenMySQLConnection();
while (reader.Read())
{
decimal LastTradePrice;
decimal Bid;
string MarketOpen;
string IndexCode;
string Index;
decimal Offer;
decimal LTPDatabase=0.1M;
IndexCode = reader[3].ToString();
String addSQL = "Select LastTradePrice from `jsontest`.`tbl_MarketData` where IndexCode = '" + IndexCode + "'";
MySqlConnection objMyCon = new MySqlConnection(strProvider);
objMyCon.Open();
MySqlCommand command = objMyCon.CreateCommand();
command.CommandText = addSQL;
MySqlDataReader result = command.ExecuteReader();
//int j = command.ExecuteNonQuery();
while (result.Read())
{
LTPDatabase = Convert.ToDecimal(result[0]);
// LTPDatabase = Math.Round(LTPTemp, 2);
}
objMyCon.Close();
decimal LTPTemp = Convert.ToDecimal(reader[0].ToString());
LastTradePrice = Math.Round(LTPTemp, 2);
if (reader[1].ToString() != "N/A")
{
Bid = Convert.ToDecimal(reader[1].ToString());
}
else
{
Bid = 10.0M;
}
if (LastTradePrice != LTPDatabase)
{
MarketOpen = "Open";
}
else
{
MarketOpen = "Close";
}
Index = reader[4].ToString();
if (reader[5].ToString() != "N/A")
{
Offer = Convert.ToDecimal(reader[5].ToString());
}
else
{
Offer = 20.0M;
}
//}
// string[] arLine = strLine.Split(';');
// string strAgencyPropertyID = arLine[0];
// DateTime dt = DateTime.Parse(arLine[1]);
// Int64 nDate = (Int64)Util.ConvertToUnixTimestamp(dt);
// String strAvailability = (arLine[2]);
_nNrRowsProccessed++;
newService.CSVInsert(IndexCode,Index,MarketOpen,Bid,Offer,LastTradePrice);
// MasterCalendar_DB.Insert(strAgencyPropertyID, nDate, strAvailability);
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
conn.Close();
// MasterCalendar_DB.CloseMySQLConnection();
}
return "Success";
}
}
【问题讨论】:
-
您可能应该从分析开始以确定哪个方面花费的时间最多......然后专注于那个?没有任何代码或更多细节,这个问题是无法回答的。
-
您是否获得了三个主要进程(文件传输、解析算法、数据访问)的相对时间的指标来确定瓶颈?
-
您用于下载和解析 csv 的算法/代码是什么?哪个步骤需要更多时间?
-
我要做的第一件事就是从 ASP.NET 应用程序中删除该后台线程。它不属于那里。
-
@John Rasch:我有
edited我的问题,并添加了我的主要功能以及为下载和解析功能而调用的功能。
标签: .net asp.net multithreading .net-3.5