您可以通过大块写入文件来减少文件 I/O 的开销,从而减少单个写入操作的数量。
#define CHUNK_SIZE 4096
char file_buffer[CHUNK_SIZE + 64] ; // 4Kb buffer, plus enough
// for at least one one line
int buffer_count = 0 ;
int i = 0 ;
while( i < cnt )
{
buffer_count += sprintf( &file_buffer[buffer_count], "%d %d %d\n", a[i], b[i], c[i] ) ;
i++ ;
// if the chunk is big enough, write it.
if( buffer_count >= CHUNK_SIZE )
{
fwrite( file_buffer, buffer_count, 1, f ) ;
buffer_count = 0 ;
}
}
// Write remainder
if( buffer_count > 0 )
{
fwrite( file_buffer, buffer_count, 1, f ) ;
}
在单次写入中恰好写入 4096 字节(或其他 2 的幂)可能有一些优势,但这在很大程度上取决于文件系统,并且执行此操作的代码变得有点更复杂:
#define CHUNK_SIZE 4096
char file_buffer[CHUNK_SIZE + 64] ;
int buffer_count = 0 ;
int i = 0 ;
while( i < cnt )
{
buffer_count += sprintf( &file_buffer[buffer_count], "%d %d %d\n", a[i], b[i], c[i] ) ;
i++ ;
// if the chunk is big enough, write it.
if( buffer_count >= CHUNK_SIZE )
{
fwrite( file_buffer, CHUNK_SIZE, 1, f ) ;
buffer_count -= CHUNK_SIZE ;
memcpy( file_buffer, &file_buffer[CHUNK_SIZE], buffer_count ) ;
}
}
// Write remainder
if( buffer_count > 0 )
{
fwrite( file_buffer, 1, buffer_count, f ) ;
}
您可以尝试使用不同的 CHUNK_SIZE 值 - 较大可能是最佳选择,或者您可能会发现它几乎没有什么区别。我建议至少 512 字节。
测试结果:
使用 VC++ 2015,在以下平台上:
配备希捷 ST1000DM003 1TB 64MB 高速缓存 SATA 6.0Gb/s 硬盘。
写入 100000 行的单个测试的结果非常多变,正如您在运行多个进程共享同一硬盘驱动器的桌面系统上所期望的那样,因此我每次运行测试 100 次并选择最短时间结果(蜜蜂可以见下面的代码结果):
使用默认的“调试”构建设置(4K 块):
line_by_line: 0.195000 seconds
block_write1: 0.154000 seconds
block_write2: 0.143000 seconds
使用默认的“发布”构建设置(4K 块):
line_by_line: 0.067000 seconds
block_write1: 0.037000 seconds
block_write2: 0.036000 seconds
优化对所有三种实现都有类似的影响,固定大小的块写入速度略快于“参差不齐”的块。
当使用 32K 块时,性能仅略高一些,固定版本和不规则版本之间的差异可以忽略不计:
使用默认的“发布”构建设置(32K 块):
block_write1: 0.036000 seconds
block_write2: 0.036000 seconds
使用 512 字节块与 4K 块没有明显区别:
使用默认的“发布”构建设置(512 字节块):
block_write1: 0.036000 seconds
block_write2: 0.037000 seconds
以上都是 32 位 (x86) 版本。构建 64 位代码 (x64) 产生了有趣的结果:
使用默认的“发布”构建设置(4K 块)- 64 位代码:
line_by_line: 0.049000 seconds
block_write1: 0.038000 seconds
block_write2: 0.032000 seconds
不规则块稍微慢一些(尽管可能没有统计学意义),固定块明显快于逐行写入(但不足以使其比任何块写入更快)。
测试代码(4K块版):
#include <stdio.h>
#include <string.h>
#include <time.h>
void line_by_line_write( int count )
{
FILE* f = fopen("line_by_line_write.txt", "w");
for( int i = 0; i < count; i++)
{
fprintf(f, "%d %d %d\n", 1234, 5678, 9012 ) ;
}
fclose(f);
}
#define CHUNK_SIZE (4096)
void block_write1( int count )
{
FILE* f = fopen("block_write1.txt", "w");
char file_buffer[CHUNK_SIZE + 64];
int buffer_count = 0;
int i = 0;
while( i < count )
{
buffer_count += sprintf( &file_buffer[buffer_count], "%d %d %d\n", 1234, 5678, 9012 );
i++;
// if the chunk is big enough, write it.
if( buffer_count >= CHUNK_SIZE )
{
fwrite( file_buffer, buffer_count, 1, f );
buffer_count = 0 ;
}
}
// Write remainder
if( buffer_count > 0 )
{
fwrite( file_buffer, 1, buffer_count, f );
}
fclose(f);
}
void block_write2( int count )
{
FILE* f = fopen("block_write2.txt", "w");
char file_buffer[CHUNK_SIZE + 64];
int buffer_count = 0;
int i = 0;
while( i < count )
{
buffer_count += sprintf( &file_buffer[buffer_count], "%d %d %d\n", 1234, 5678, 9012 );
i++;
// if the chunk is big enough, write it.
if( buffer_count >= CHUNK_SIZE )
{
fwrite( file_buffer, CHUNK_SIZE, 1, f );
buffer_count -= CHUNK_SIZE;
memcpy( file_buffer, &file_buffer[CHUNK_SIZE], buffer_count );
}
}
// Write remainder
if( buffer_count > 0 )
{
fwrite( file_buffer, 1, buffer_count, f );
}
fclose(f);
}
#define LINES 100000
int main( void )
{
clock_t line_by_line_write_minimum = 9999 ;
clock_t block_write1_minimum = 9999 ;
clock_t block_write2_minimum = 9999 ;
for( int i = 0; i < 100; i++ )
{
clock_t start = clock() ;
line_by_line_write( LINES ) ;
clock_t t = clock() - start ;
if( t < line_by_line_write_minimum ) line_by_line_write_minimum = t ;
start = clock() ;
block_write1( LINES ) ;
t = clock() - start ;
if( t < block_write1_minimum ) block_write1_minimum = t ;
start = clock() ;
block_write2( LINES ) ;
t = clock() - start ;
if( t < block_write2_minimum ) block_write2_minimum = t ;
}
printf( "line_by_line: %f seconds\n", (float)(line_by_line_write_minimum) / CLOCKS_PER_SEC ) ;
printf( "block_write1: %f seconds\n", (float)(block_write1_minimum) / CLOCKS_PER_SEC ) ;
printf( "block_write2: %f seconds\n", (float)(block_write2_minimum) / CLOCKS_PER_SEC ) ;
}