Stan/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

tweaks for thesis and minor fixes.

Browse Source 
file: change block size to 32kb (=> yields highest throughput)
archive, file_io: add additional block cache forwarding path when
reading uncompressed data from archive (avoids repetitive IOs)
file_io, file_stats: fix inaccurate and incorrect IO throughput
calculation
file_stats: refactor some calculations to avoid division by 0

This was SVN commit r3665.
This commit is contained in:
janwas 2006-03-18 21:28:47 +00:00
parent 2e0a312913
commit f5f4670d7f
6 changed files with 47 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
source/lib/res/file/archive.cpp
View File

@ -295,6 +295,8 @@ LibError afile_open(const Handle ha, const char* fn, uintptr_t memento, int flag
return ERR_NO_MEM;
}
flags |= FILE_CACHE_BLOCK;
af->fc.flags = flags;
af->fc.size = ent->ucsize;
af->fc.atom_fn = atom_fn;
@ -431,11 +433,7 @@ LibError afile_io_validate(const AFileIo* io)
}
///////////////////////////////////////////////////////////////////////////////
//-----------------------------------------------------------------------------
class Decompressor
{

2
source/lib/res/file/file.h
View File

@ -325,7 +325,7 @@ extern LibError file_io_validate(const FileIo* io);
// all transfers are expanded to naturally aligned, whole blocks
// (this makes caching parts of files feasible; it is also much faster
// for some aio implementations, e.g. wposix).
const size_t FILE_BLOCK_SIZE = 16*KiB;
const size_t FILE_BLOCK_SIZE = 32*KiB;
typedef const u8* FileIOBuf;

38
source/lib/res/file/file_io.cpp
View File

@ -70,7 +70,7 @@ static AiocbAllocator aiocb_allocator;
// no attempt is made at aligning or padding the transfer.
LibError file_io_issue(File* f, off_t ofs, size_t size, void* p, FileIo* io)
{
debug_printf("FILE| issue ofs=%d size=%d\n", ofs, size);
debug_printf("FILE| issue ofs=0x%X size=0x%X\n", ofs, size);
// zero output param in case we fail below.
memset(io, 0, sizeof(FileIo));
@ -108,7 +108,6 @@ LibError file_io_issue(File* f, off_t ofs, size_t size, void* p, FileIo* io)
cb->aio_fildes = f->fd;
cb->aio_offset = ofs;
cb->aio_nbytes = size;
debug_printf("FILE| issue2 io=%p nbytes=%u\n", io, cb->aio_nbytes);
int err = lio_listio(LIO_NOWAIT, &cb, 1, (struct sigevent*)0);
if(err < 0)
{
@ -138,7 +137,7 @@ int file_io_has_completed(FileIo* io)
LibError file_io_wait(FileIo* io, void*& p, size_t& size)
{
debug_printf("FILE| wait io=%p\n", io);
// debug_printf("FILE| wait io=%p\n", io);
// zero output params in case something (e.g. H_DEREF) fails.
p = 0;
@ -153,7 +152,7 @@ LibError file_io_wait(FileIo* io, void*& p, size_t& size)
// query number of bytes transferred (-1 if the transfer failed)
const ssize_t bytes_transferred = aio_return(cb);
debug_printf("FILE| bytes_transferred=%d aio_nbytes=%u\n", bytes_transferred, cb->aio_nbytes);
// debug_printf("FILE| bytes_transferred=%d aio_nbytes=%u\n", bytes_transferred, cb->aio_nbytes);
// see if actual transfer count matches requested size.
// note: most callers clamp to EOF but round back up to sector size
@ -325,11 +324,6 @@ class IOManager
else
dst = (void*)*pbuf;
double start_time = 0.0; // required for stats
FileOp op = is_write? FO_WRITE : FO_READ;
BlockId disk_pos = block_cache_make_id(f->fc.atom_fn, start_ofs);
stats_io_start(FI_LOWIO, op, size, disk_pos, &start_time);
//
ssize_t total_transferred;
if(is_write)
total_transferred = write(fd, dst, size);
@ -340,8 +334,6 @@ class IOManager
free(dst_mem);
WARN_RETURN(LibError_from_errno());
}
//
stats_io_finish(FI_LOWIO, op, &start_time);
size_t total_processed;
LibError ret = file_io_call_back(dst, total_transferred, cb, cb_ctx, total_processed);
@ -408,7 +400,6 @@ class IOManager
else
buf = (char*)*pbuf + total_issued;
stats_io_start(FI_AIO, is_write? FO_WRITE : FO_READ, issue_size, slot.block_id, &slot.start_time);
LibError ret = file_io_issue(f, ofs, issue_size, buf, &slot.io);
// transfer failed - loop will now terminate after
// waiting for all pending transfers to complete.
@ -430,9 +421,15 @@ class IOManager
else
{
LibError ret = file_io_wait(&slot.io, block, block_size);
stats_io_finish(FI_AIO, is_write? FO_WRITE : FO_READ, &slot.start_time);
if(ret < 0)
err = ret;
if(pbuf != FILE_BUF_TEMP && f->fc.flags & FILE_CACHE_BLOCK)
{
slot.temp_buf = block_cache_alloc(slot.block_id);
memcpy2(slot.temp_buf, block, block_size);
// block_cache_mark_completed will be called in process()
}
}
// first time; skip past padding
@ -472,7 +469,8 @@ class IOManager
else
{
file_io_discard(&slot.io);
if(pbuf == FILE_BUF_TEMP)
if(slot.temp_buf)
block_cache_mark_completed(slot.block_id);
}
}
@ -536,10 +534,13 @@ public:
{
RETURN_ERR(prepare());
ssize_t ret = no_aio? lowio() : aio();
/// FILE_STATS_NOTIFY_IO(fi, is_write? FO_WRITE : FO_READ, user_size, total_issued, start_time);
debug_printf("FILE| err=%d, total_processed=%u\n", err, total_processed);
const FileIOImplentation fi = no_aio? FI_LOWIO : FI_AIO;
const FileOp fo = is_write? FO_WRITE : FO_READ;
const BlockId disk_pos = block_cache_make_id(f->fc.atom_fn, start_ofs);
double start_time = 0.0;
stats_io_start(disk_pos, &start_time);
ssize_t ret = no_aio? lowio() : aio();
stats_io_finish(fi, fo, ret, &start_time);
// we allocated the memory: skip any leading padding
if(pbuf != FILE_BUF_TEMP && !is_write)
@ -572,7 +573,6 @@ public:
ssize_t file_io(File* f, off_t ofs, size_t size, FileIOBuf* pbuf,
FileIOCB cb, uintptr_t ctx) // optional
{
debug_printf("FILE| io: size=%u ofs=%u fn=%s\n", size, ofs, f->fc.atom_fn);
CHECK_FILE(f);
// note: do not update stats/trace here: this includes Zip IOs,

32
source/lib/res/file/file_stats.cpp
View File

@ -157,14 +157,8 @@ void stats_user_io(size_t user_size)
user_io_size_total += user_size;
}
void stats_io_start(FileIOImplentation fi, FileOp fo, size_t actual_size,
BlockId disk_pos, double* start_time_storage)
void stats_io_start(BlockId disk_pos, double* start_time_storage)
{
debug_assert(fi < FI_MAX_IDX);
debug_assert(fo == FO_READ || FO_WRITE);
io_actual_size_total[fi][fo] += actual_size;
if(disk_pos.atom_fn != io_disk_pos_cur.atom_fn ||
disk_pos.block_num != io_disk_pos_cur.block_num+1)
io_seeks++;
@ -173,12 +167,17 @@ void stats_io_start(FileIOImplentation fi, FileOp fo, size_t actual_size,
timer_start(start_time_storage);
}
void stats_io_finish(FileIOImplentation fi, FileOp fo, double* start_time_storage)
void stats_io_finish(FileIOImplentation fi, FileOp fo, ssize_t user_size, double* start_time_storage)
{
debug_assert(fi < FI_MAX_IDX);
debug_assert(fo == FO_READ || FO_WRITE);
io_elapsed_time[fi][fo] += timer_reset(start_time_storage);
// ignore IOs that failed (nothing we can do)
if(user_size > 0)
{
io_actual_size_total[fi][fo] += user_size;
io_elapsed_time[fi][fo] += timer_reset(start_time_storage);
}
}
void stats_cb_start()
@ -223,6 +222,13 @@ void stats_block_cache(CacheRet cr)
//-----------------------------------------------------------------------------
template<typename T> int percent(T num, T divisor)
{
if(!divisor)
return 0;
return (int)(100*num / divisor);
}
void stats_dump()
{
const double KB = 1e3; const double MB = 1e6; const double ms = 1e-3;
@ -249,7 +255,7 @@ void stats_dump()
"Accessed files: %u (%g MB) -- %u%% of data set\n"
"Max. concurrent: %u; leaked: %u.\n",
unique_names, unique_name_len_total/1000,
opened_files.size(), opened_file_size_total/MB, 100u*opened_files.size()/vfs_files,
opened_files.size(), opened_file_size_total/MB, percent(opened_files.size(), vfs_files),
open_files_max, open_files_cur
);
@ -261,7 +267,7 @@ void stats_dump()
"Internal fragmentation: %d%%\n",
extant_bufs_total, buf_user_size_total/MB,
extant_bufs_max, extant_bufs_cur,
(int)(100*(buf_padded_size_total-buf_user_size_total)/buf_user_size_total)
percent(buf_padded_size_total-buf_user_size_total, buf_user_size_total)
);
// file_io
@ -286,7 +292,7 @@ void stats_dump()
"Hit ratio: %u%%; conflict misses: %u%%\n"
"Block hits: %u; misses: %u; ratio: %u%%\n",
cache_count[CR_HIT], cache_size_total[CR_HIT]/MB, cache_count[CR_MISS], cache_size_total[CR_MISS]/MB,
100u*cache_count[CR_HIT]/cache_count[CR_MISS], 100u*conflict_misses/cache_count[CR_MISS],
block_cache_count[CR_HIT], block_cache_count[CR_MISS], 100u*block_cache_count[CR_HIT]/(block_cache_count[CR_HIT]+block_cache_count[CR_MISS])
percent(cache_count[CR_HIT], cache_count[CR_MISS]), percent(conflict_misses, cache_count[CR_MISS]),
block_cache_count[CR_HIT], block_cache_count[CR_MISS], percent(block_cache_count[CR_HIT], block_cache_count[CR_HIT]+block_cache_count[CR_MISS])
);
}

9
source/lib/res/file/file_stats.h
View File

@ -29,9 +29,8 @@ extern void stats_buf_ref();
// file_io
extern void stats_user_io(size_t user_size);
extern void stats_io_start(FileIOImplentation fi, FileOp fo,
size_t actual_size, BlockId disk_pos, double* start_time_storage);
extern void stats_io_finish(FileIOImplentation fi, FileOp fo, double* start_time_storage);
extern void stats_io_start(BlockId disk_pos, double* start_time_storage);
extern void stats_io_finish(FileIOImplentation fi, FileOp fo, ssize_t user_size, double* start_time_storage);
extern void stats_cb_start();
extern void stats_cb_finish();
@ -54,8 +53,8 @@ extern void stats_dump();
#define stats_buf_free()
#define stats_buf_ref()
#define stats_user_io(user_size)
#define stats_io_start(fi, fo, actual_size, disk_pos, start_time_storage)
#define stats_io_finish(fi, fo, start_time_storage)
#define stats_io_start(disk_pos, start_time_storage)
#define stats_io_finish(fi, fo, user_size, start_time_storage)
#define stats_cb_start()
#define stats_cb_finish()
#define stats_cache(cr, size, atom_fn)

2
source/lib/res/file/trace.h
View File

@ -58,7 +58,7 @@ enum TraceRunFlags
// carry out all operations specified in the trace.
// if flags&TRF_SYNC_TO_TIMESTAMP, waits until timestamp for each event is
// reached; otherwise, they are run as fast as possible.
extern LibError trace_run(const char* trace_filename, uint flags);
extern LibError trace_run(const char* trace_filename, uint flags = 0);
#endif // #ifndef TRACE_H__