Stan/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

add cppdoc documentation for compression

Browse Source 
ucsize -> usize etc.

This was SVN commit r5371.
This commit is contained in:
janwas 2007-09-25 10:06:02 +00:00
parent f4adce44bf
commit f997c03028
6 changed files with 98 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
source/lib/res/file/archive.cpp
View File

@ -198,7 +198,7 @@ LibError archive_enum(const Handle ha, const FileCB cb, const uintptr_t user)
{
const ArchiveEntry* ent = &a->ents[i];
s.st_mode = S_IFREG;
s.st_size = (off_t)ent->ucsize;
s.st_size = (off_t)ent->usize;
s.st_mtime = ent->mtime;
const uintptr_t memento = (uintptr_t)ent;
LibError ret = cb(ent->atom_fn, &s, memento, user);
@ -261,7 +261,7 @@ struct ArchiveFile
cassert(sizeof(ArchiveFile) <= FILE_OPAQUE_SIZE);
// convenience function, allows implementation change in File.
// note that size == ucsize isn't foolproof, and adding a flag to
// note that size == usize isn't foolproof, and adding a flag to
// ofs or size is ugly and error-prone.
// no error checking - always called from functions that check af.
static inline bool is_compressed(ArchiveFile* af)
@ -283,7 +283,7 @@ LibError afile_stat(Handle ha, const char* fn, struct stat* s)
ArchiveEntry* ent;
RETURN_ERR(archive_get_file_info(a, fn, 0, ent));
s->st_size = ent->ucsize;
s->st_size = ent->usize;
s->st_mtime = ent->mtime;
return INFO::OK;
}
@ -328,7 +328,7 @@ LibError afile_open(const Handle ha, const char* fn, uintptr_t memento, uint fla
ArchiveEntry* ent;
// don't want File to contain a ArchiveEntry struct -
// its ucsize member must be 'loose' for compatibility with File.
// its usize member must be 'loose' for compatibility with File.
// => need to copy ArchiveEntry fields into File.
RETURN_ERR(archive_get_file_info(a, atom_fn, memento, ent));
@ -344,7 +344,7 @@ LibError afile_open(const Handle ha, const char* fn, uintptr_t memento, uint fla
}
f->flags = flags;
f->size = ent->ucsize;
f->size = ent->usize;
f->atom_fn = atom_fn;
ArchiveFile* af = (ArchiveFile*)f->opaque;
af->ofs = ent->ofs;
@ -471,12 +471,12 @@ LibError afile_io_wait(FileIo* io, u8*& buf, size_t& size)
if(aio->ctx)
{
comp_set_output(aio->ctx, aio->user_buf, aio->max_output_size);
ssize_t ucbytes_output = comp_feed(aio->ctx, raw_buf, raw_size);
const ssize_t ubytes_output = comp_feed(aio->ctx, raw_buf, raw_size);
free(raw_buf);
RETURN_ERR(ucbytes_output);
RETURN_ERR(ubytes_output);
buf = aio->user_buf;
size = ucbytes_output;
size = ubytes_output;
}
else
{
@ -635,11 +635,11 @@ ssize_t afile_read(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb
const size_t csize_max = af->csize - af->last_cofs;
Decompressor d(af->ctx, pbuf, size, cb, cb_ctx);
ssize_t uc_transferred = file_io(&a->f, cofs, csize_max, FILE_BUF_TEMP, decompressor_feed_cb, (uintptr_t)&d);
const ssize_t usize_read = file_io(&a->f, cofs, csize_max, FILE_BUF_TEMP, decompressor_feed_cb, (uintptr_t)&d);
af->last_cofs += (off_t)d.NumCompressedBytesProcessed();
return uc_transferred;
return usize_read;
}

4
source/lib/res/file/archive.h
View File

@ -163,7 +163,7 @@ enum ArchiveFileFlags
struct ArchiveEntry
{
// these are returned by afile_stat:
off_t ucsize;
off_t usize;
time_t mtime;
// used in IO
@ -184,7 +184,7 @@ struct ArchiveEntry
//
// we also need a way to check if a file is compressed (e.g. to fail
// mmap requests if the file is compressed). packing a bit in ofs or
// ucsize is error prone and ugly (1 bit less won't hurt though).
// usize is error prone and ugly (1 bit less won't hurt though).
// any other way will mess up the nice 2^n byte size anyway, so
// might as well store csize.
};

30
source/lib/res/file/archive_builder.cpp
View File

@ -76,10 +76,10 @@ static LibError compress_cb(uintptr_t cb_ctx, const u8* block, size_t size, size
// final decision on whether to store the file as compressed,
// given the observed compressed/uncompressed sizes.
static bool ShouldCompress(size_t ucsize, size_t csize)
static bool ShouldCompress(size_t usize, size_t csize)
{
const float ratio = (float)ucsize / csize;
const ssize_t bytes_saved = (ssize_t)ucsize - (ssize_t)csize;
const float ratio = (float)usize / csize;
const ssize_t bytes_saved = (ssize_t)usize - (ssize_t)csize;
UNUSED2(bytes_saved);
// tiny - store compressed regardless of savings.
@ -87,14 +87,14 @@ static bool ShouldCompress(size_t ucsize, size_t csize)
// - CPU cost is negligible and overlapped with IO anyway;
// - reading from compressed files uses less memory because we
// don't need to allocate space for padding in the final buffer.
if(ucsize < 512)
if(usize < 512)
return true;
// large high-entropy file - store uncompressed.
// rationale:
// - any bigger than this and CPU time becomes a problem: it isn't
// necessarily hidden by IO time anymore.
if(ucsize >= 32*KiB && ratio < 1.02f)
if(usize >= 32*KiB && ratio < 1.02f)
return false;
// TODO: any other cases?
@ -107,33 +107,33 @@ static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx,
{
struct stat s;
RETURN_ERR(vfs_stat(atom_fn, &s));
const size_t ucsize = s.st_size;
const size_t usize = s.st_size;
// skip 0-length files.
// rationale: zip.cpp needs to determine whether a CDFH entry is
// a file or directory (the latter are written by some programs but
// not needed - they'd only pollute the file table).
// it looks like checking for ucsize=csize=0 is the safest way -
// it looks like checking for usize=csize=0 is the safest way -
// relying on file attributes (which are system-dependent!) is
// even less safe.
// we thus skip 0-length files to avoid confusing them with dirs.
if(!ucsize)
if(!usize)
return INFO::SKIPPED;
const bool attempt_compress = !file_type_is_uncompressible(atom_fn);
if(attempt_compress)
{
comp_reset(ctx);
const size_t csizeBound = comp_max_output_size(ctx, ucsize);
const size_t csizeBound = comp_max_output_size(ctx, usize);
RETURN_ERR(comp_alloc_output(ctx, csizeBound));
}
// read file into newly allocated buffer. if attempt_compress, also
// compress the file into another buffer while waiting for IOs.
size_t ucsize_read;
size_t usize_read;
const uint flags = 0;
CompressParams params = { attempt_compress, ctx, 0 };
RETURN_ERR(vfs_load(atom_fn, buf, ucsize_read, flags, compress_cb, (uintptr_t)&params));
debug_assert(ucsize_read == ucsize);
RETURN_ERR(vfs_load(atom_fn, buf, usize_read, flags, compress_cb, (uintptr_t)&params));
debug_assert(usize_read == usize);
// if we compressed the file trial-wise, check results and
// decide whether to store as such or not (based on compression ratio)
@ -149,11 +149,11 @@ static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx,
return ret;
}
shouldCompress = ShouldCompress(ucsize, csize);
shouldCompress = ShouldCompress(usize, csize);
}
// store file info
ent.ucsize = (off_t)ucsize;
ent.usize = (off_t)usize;
ent.mtime = s.st_mtime;
// .. ent.ofs is set by zip_archive_add_file
ent.flags = 0;
@ -168,7 +168,7 @@ static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx,
else
{
ent.method = CM_NONE;
ent.csize = (off_t)ucsize;
ent.csize = (off_t)usize;
file_contents = buf;
}

82
source/lib/res/file/compression.h
View File

@ -18,7 +18,6 @@ namespace ERR
const LibError COMPRESSION_UNKNOWN_METHOD = -110300;
}
enum ContextType
{
CT_COMPRESSION,
@ -43,45 +42,62 @@ extern uintptr_t comp_alloc(ContextType type, CompressionMethod method);
**/
extern size_t comp_max_output_size(uintptr_t ctx, size_t inSize);
// set output buffer. all subsequent comp_feed() calls will write into it.
// should only be called once (*) due to the comp_finish() interface - since
// that allows querying the output buffer, it must not be fragmented.
// * the previous output buffer is wiped out by comp_reset, so
// setting it again (once!) after that is allowed and required.
extern void comp_set_output(uintptr_t ctx, u8* out, size_t out_size);
/**
* set output buffer for subsequent comp_feed() calls.
*
* due to the comp_finish interface, output buffers must be contiguous or
* identical (otherwise IsAllowableOutputBuffer will complain).
**/
extern void comp_set_output(uintptr_t ctx, u8* out, size_t outSize);
// [compression contexts only:] allocate an output buffer big enough to
// hold worst_case_compression_ratio*in_size bytes.
// rationale: this interface is useful because callers cannot
// reliably estimate how much output space is needed.
// raises a warning for decompression contexts because this operation
// does not make sense there:
// - worst-case decompression ratio is quite large - ballpark 1000x;
// - exact uncompressed size is known to caller (via archive file header).
// note: buffer is held until comp_free; it can be reused after a
// comp_reset. this reduces malloc/free calls.
extern LibError comp_alloc_output(uintptr_t ctx, size_t in_size);
/**
* allocate a new output buffer.
*
* @param size [bytes] to allocate.
*
* if a buffer had previously been allocated and is large enough, it is
* reused (this reduces the number of allocations). the buffer is
* automatically freed by comp_free.
**/
extern LibError comp_alloc_output(uintptr_t ctx, size_t inSize);
// 'feed' the given buffer to the compressor/decompressor.
// returns number of output bytes produced (*), or a negative LibError code.
// * 0 is a legitimate return value - this happens if the input buffer is
// small and the codec hasn't produced any output.
// note: the buffer may be overwritten or freed immediately after - we take
// care of copying and queuing any data that remains (e.g. due to
// lack of output buffer space).
extern ssize_t comp_feed(uintptr_t ctx, const u8* in, size_t in_size);
/**
* 'feed' the given buffer to the compressor/decompressor.
*
* @return number of output bytes produced or a negative LibError.
* note that 0 is a legitimate return value - this happens if the input
* buffer is small and the codec hasn't produced any output.
*
* note: after this call returns, the buffer may be overwritten or freed;
* we take care of copying and queuing any data that remains (e.g. due to
* lack of output buffer space).
**/
extern ssize_t comp_feed(uintptr_t ctx, const u8* in, size_t inSize);
// feed any remaining queued input data, finish the compress/decompress and
// pass back the output buffer.
/**
* conclude the compression/decompression operation.
*
* @param out, outSize receive the output buffer. this assumes identical or
* contiguous addresses were passed, which comp_set_output ensures.
* @param checksum
*
* note: this must always be called (even if the output buffer is already
* known) because it feeds any remaining queued input buffers.
**/
extern LibError comp_finish(uintptr_t ctx, u8** out, size_t* out_size, u32* checksum);
// prepare this context for reuse. the effect is similar to freeing this
// context and creating another.
// rationale: this API avoids reallocating a considerable amount of
// memory (ballbark 200KB LZ window plus output buffer).
/**
* clear all previous state and prepare for reuse.
*
* this is as if the object were destroyed and re-created, but more
* efficient since it avoids reallocating a considerable amount of memory
* (about 200KB for LZ).
**/
extern void comp_reset(uintptr_t ctx);
// free this context and all associated memory.
/**
* free this context and all associated memory.
**/
extern void comp_free(uintptr_t ctx);
#endif // #ifndef INCLUDED_COMPRESSION

20
source/lib/res/file/tests/test_compression.h
View File

@ -16,7 +16,7 @@ public:
data[i] = rand() & 0x07;
u8* cdata; size_t csize;
u8 ucdata[data_size];
u8 udata[data_size];
// compress
uintptr_t c = comp_alloc(CT_COMPRESSION, CM_DEFLATE);
@ -35,20 +35,20 @@ public:
uintptr_t d = comp_alloc(CT_DECOMPRESSION, CM_DEFLATE);
{
TS_ASSERT(d != 0);
comp_set_output(d, ucdata, data_size);
const ssize_t ucdata_produced = comp_feed(d, cdata, csize);
TS_ASSERT(ucdata_produced >= 0);
u8* ucdata_final; size_t ucsize_final; u32 checksum;
TS_ASSERT_OK(comp_finish(d, &ucdata_final, &ucsize_final, &checksum));
TS_ASSERT(ucdata_produced <= (ssize_t)ucsize_final); // can't have produced more than total
TS_ASSERT_EQUALS(ucdata_final, ucdata); // output buffer address is same
TS_ASSERT_EQUALS(ucsize_final, data_size); // correct amount of output
comp_set_output(d, udata, data_size);
const ssize_t udata_produced = comp_feed(d, cdata, csize);
TS_ASSERT(udata_produced >= 0);
u8* udata_final; size_t usize_final; u32 checksum;
TS_ASSERT_OK(comp_finish(d, &udata_final, &usize_final, &checksum));
TS_ASSERT(udata_produced <= (ssize_t)usize_final); // can't have produced more than total
TS_ASSERT_EQUALS(udata_final, udata); // output buffer address is same
TS_ASSERT_EQUALS(usize_final, data_size); // correct amount of output
}
comp_free(c);
comp_free(d);
// verify data survived intact
TS_ASSERT_SAME_DATA(data, ucdata, data_size);
TS_ASSERT_SAME_DATA(data, udata, data_size);
}
};

24
source/lib/res/file/zip.cpp
View File

@ -128,7 +128,7 @@ struct LFH
u32 fat_mtime; // last modified time (DOS FAT format)
u32 crc;
u32 csize;
u32 ucsize;
u32 usize;
u16 fn_len;
u16 e_len;
};
@ -154,7 +154,7 @@ struct CDFH
u32 fat_mtime; // last modified time (DOS FAT format)
u32 crc;
u32 csize;
u32 ucsize;
u32 usize;
u16 fn_len;
u16 e_len;
u16 c_len;
@ -203,7 +203,7 @@ static off_t lfh_total_size(const LFH* lfh_le)
static void lfh_assemble(LFH* lfh_le,
CompressionMethod method, time_t mtime, u32 crc,
off_t csize, off_t ucsize, size_t fn_len)
off_t csize, off_t usize, size_t fn_len)
{
const ZipCompressionMethod zip_method = zip_method_for(method);
const u32 fat_mtime = FAT_from_time_t(mtime);
@ -215,20 +215,20 @@ static void lfh_assemble(LFH* lfh_le,
lfh_le->fat_mtime = to_le32(fat_mtime);
lfh_le->crc = to_le32(crc);
lfh_le->csize = to_le32(u32_from_larger(csize));
lfh_le->ucsize = to_le32(u32_from_larger(ucsize));
lfh_le->usize = to_le32(u32_from_larger(usize));
lfh_le->fn_len = to_le16(u16_from_larger(fn_len));
lfh_le->e_len = to_le16(0);
}
static void cdfh_decompose(const CDFH* cdfh_le,
CompressionMethod& method, time_t& mtime, off_t& csize, off_t& ucsize,
CompressionMethod& method, time_t& mtime, off_t& csize, off_t& usize,
const char*& fn, off_t& lfh_ofs, size_t& total_size)
{
const u16 zip_method = read_le16(&cdfh_le->method);
const u32 fat_mtime = read_le32(&cdfh_le->fat_mtime);
csize = (off_t)read_le32(&cdfh_le->csize);
ucsize = (off_t)read_le32(&cdfh_le->ucsize);
usize = (off_t)read_le32(&cdfh_le->usize);
const u16 fn_len = read_le16(&cdfh_le->fn_len);
const u16 e_len = read_le16(&cdfh_le->e_len);
const u16 c_len = read_le16(&cdfh_le->c_len);
@ -249,7 +249,7 @@ static void cdfh_decompose(const CDFH* cdfh_le,
static void cdfh_assemble(CDFH* dst_cdfh_le,
CompressionMethod method, time_t mtime, u32 crc,
size_t csize, size_t ucsize, size_t fn_len, size_t slack, u32 lfh_ofs)
size_t csize, size_t usize, size_t fn_len, size_t slack, u32 lfh_ofs)
{
const ZipCompressionMethod zip_method = zip_method_for(method);
const u32 fat_mtime = FAT_from_time_t(mtime);
@ -261,7 +261,7 @@ static void cdfh_assemble(CDFH* dst_cdfh_le,
dst_cdfh_le->fat_mtime = to_le32(fat_mtime);
dst_cdfh_le->crc = to_le32(crc);
dst_cdfh_le->csize = to_le32(u32_from_larger(csize));
dst_cdfh_le->ucsize = to_le32(u32_from_larger(ucsize));
dst_cdfh_le->usize = to_le32(u32_from_larger(usize));
dst_cdfh_le->fn_len = to_le16(u16_from_larger(fn_len));
dst_cdfh_le->e_len = to_le16(0);
dst_cdfh_le->c_len = to_le16(u16_from_larger(slack));
@ -453,11 +453,11 @@ LibError zip_populate_archive(File* f, Archive* a)
// copy translated fields from CDFH into ArchiveEntry.
ArchiveEntry ae;
cdfh_decompose(cdfh, ae.method, ae.mtime, ae.csize, ae.ucsize, ae.atom_fn, ae.ofs, ofs_to_next_cdfh);
cdfh_decompose(cdfh, ae.method, ae.mtime, ae.csize, ae.usize, ae.atom_fn, ae.ofs, ofs_to_next_cdfh);
ae.flags = ZIP_LFH_FIXUP_NEEDED;
// if file (we don't care about directories):
if(ae.csize && ae.ucsize)
if(ae.csize && ae.usize)
{
ret = archive_add_file(a, &ae);
if(ret != INFO::OK)
@ -587,7 +587,7 @@ LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, const u8*
// write (LFH, filename, file contents) to archive
// .. put LFH and filename into one 'package'
LFH_Package header;
lfh_assemble(&header.lfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->ucsize, fn_len);
lfh_assemble(&header.lfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->usize, fn_len);
strcpy_s(header.fn, ARRAY_SIZE(header.fn), ae->atom_fn);
// .. write that out in 1 IO
const off_t lfh_ofs = za->cur_file_size;
@ -606,7 +606,7 @@ LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, const u8*
if(!p)
WARN_RETURN(ERR::NO_MEM);
const size_t slack = za->cdfhs.da.pos-prev_pos - (CDFH_SIZE+fn_len);
cdfh_assemble(&p->cdfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->ucsize, fn_len, slack, lfh_ofs);
cdfh_assemble(&p->cdfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->usize, fn_len, slack, lfh_ofs);
cpu_memcpy(p->fn, ae->atom_fn, fn_len);
za->cd_entries++;