diff --git a/source/lib/res/file/archive.cpp b/source/lib/res/file/archive.cpp index 48d1ee43d7..c5fb109f04 100644 --- a/source/lib/res/file/archive.cpp +++ b/source/lib/res/file/archive.cpp @@ -198,7 +198,7 @@ LibError archive_enum(const Handle ha, const FileCB cb, const uintptr_t user) { const ArchiveEntry* ent = &a->ents[i]; s.st_mode = S_IFREG; - s.st_size = (off_t)ent->ucsize; + s.st_size = (off_t)ent->usize; s.st_mtime = ent->mtime; const uintptr_t memento = (uintptr_t)ent; LibError ret = cb(ent->atom_fn, &s, memento, user); @@ -261,7 +261,7 @@ struct ArchiveFile cassert(sizeof(ArchiveFile) <= FILE_OPAQUE_SIZE); // convenience function, allows implementation change in File. -// note that size == ucsize isn't foolproof, and adding a flag to +// note that size == usize isn't foolproof, and adding a flag to // ofs or size is ugly and error-prone. // no error checking - always called from functions that check af. static inline bool is_compressed(ArchiveFile* af) @@ -283,7 +283,7 @@ LibError afile_stat(Handle ha, const char* fn, struct stat* s) ArchiveEntry* ent; RETURN_ERR(archive_get_file_info(a, fn, 0, ent)); - s->st_size = ent->ucsize; + s->st_size = ent->usize; s->st_mtime = ent->mtime; return INFO::OK; } @@ -328,7 +328,7 @@ LibError afile_open(const Handle ha, const char* fn, uintptr_t memento, uint fla ArchiveEntry* ent; // don't want File to contain a ArchiveEntry struct - - // its ucsize member must be 'loose' for compatibility with File. + // its usize member must be 'loose' for compatibility with File. // => need to copy ArchiveEntry fields into File. RETURN_ERR(archive_get_file_info(a, atom_fn, memento, ent)); @@ -344,7 +344,7 @@ LibError afile_open(const Handle ha, const char* fn, uintptr_t memento, uint fla } f->flags = flags; - f->size = ent->ucsize; + f->size = ent->usize; f->atom_fn = atom_fn; ArchiveFile* af = (ArchiveFile*)f->opaque; af->ofs = ent->ofs; @@ -471,12 +471,12 @@ LibError afile_io_wait(FileIo* io, u8*& buf, size_t& size) if(aio->ctx) { comp_set_output(aio->ctx, aio->user_buf, aio->max_output_size); - ssize_t ucbytes_output = comp_feed(aio->ctx, raw_buf, raw_size); + const ssize_t ubytes_output = comp_feed(aio->ctx, raw_buf, raw_size); free(raw_buf); - RETURN_ERR(ucbytes_output); + RETURN_ERR(ubytes_output); buf = aio->user_buf; - size = ucbytes_output; + size = ubytes_output; } else { @@ -635,11 +635,11 @@ ssize_t afile_read(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb const size_t csize_max = af->csize - af->last_cofs; Decompressor d(af->ctx, pbuf, size, cb, cb_ctx); - ssize_t uc_transferred = file_io(&a->f, cofs, csize_max, FILE_BUF_TEMP, decompressor_feed_cb, (uintptr_t)&d); + const ssize_t usize_read = file_io(&a->f, cofs, csize_max, FILE_BUF_TEMP, decompressor_feed_cb, (uintptr_t)&d); af->last_cofs += (off_t)d.NumCompressedBytesProcessed(); - return uc_transferred; + return usize_read; } diff --git a/source/lib/res/file/archive.h b/source/lib/res/file/archive.h index 82a15c13d4..1a6c485ebb 100644 --- a/source/lib/res/file/archive.h +++ b/source/lib/res/file/archive.h @@ -163,7 +163,7 @@ enum ArchiveFileFlags struct ArchiveEntry { // these are returned by afile_stat: - off_t ucsize; + off_t usize; time_t mtime; // used in IO @@ -184,7 +184,7 @@ struct ArchiveEntry // // we also need a way to check if a file is compressed (e.g. to fail // mmap requests if the file is compressed). packing a bit in ofs or - // ucsize is error prone and ugly (1 bit less won't hurt though). + // usize is error prone and ugly (1 bit less won't hurt though). // any other way will mess up the nice 2^n byte size anyway, so // might as well store csize. }; diff --git a/source/lib/res/file/archive_builder.cpp b/source/lib/res/file/archive_builder.cpp index dcadd3c86d..8e520116d4 100644 --- a/source/lib/res/file/archive_builder.cpp +++ b/source/lib/res/file/archive_builder.cpp @@ -76,10 +76,10 @@ static LibError compress_cb(uintptr_t cb_ctx, const u8* block, size_t size, size // final decision on whether to store the file as compressed, // given the observed compressed/uncompressed sizes. -static bool ShouldCompress(size_t ucsize, size_t csize) +static bool ShouldCompress(size_t usize, size_t csize) { - const float ratio = (float)ucsize / csize; - const ssize_t bytes_saved = (ssize_t)ucsize - (ssize_t)csize; + const float ratio = (float)usize / csize; + const ssize_t bytes_saved = (ssize_t)usize - (ssize_t)csize; UNUSED2(bytes_saved); // tiny - store compressed regardless of savings. @@ -87,14 +87,14 @@ static bool ShouldCompress(size_t ucsize, size_t csize) // - CPU cost is negligible and overlapped with IO anyway; // - reading from compressed files uses less memory because we // don't need to allocate space for padding in the final buffer. - if(ucsize < 512) + if(usize < 512) return true; // large high-entropy file - store uncompressed. // rationale: // - any bigger than this and CPU time becomes a problem: it isn't // necessarily hidden by IO time anymore. - if(ucsize >= 32*KiB && ratio < 1.02f) + if(usize >= 32*KiB && ratio < 1.02f) return false; // TODO: any other cases? @@ -107,33 +107,33 @@ static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx, { struct stat s; RETURN_ERR(vfs_stat(atom_fn, &s)); - const size_t ucsize = s.st_size; + const size_t usize = s.st_size; // skip 0-length files. // rationale: zip.cpp needs to determine whether a CDFH entry is // a file or directory (the latter are written by some programs but // not needed - they'd only pollute the file table). - // it looks like checking for ucsize=csize=0 is the safest way - + // it looks like checking for usize=csize=0 is the safest way - // relying on file attributes (which are system-dependent!) is // even less safe. // we thus skip 0-length files to avoid confusing them with dirs. - if(!ucsize) + if(!usize) return INFO::SKIPPED; const bool attempt_compress = !file_type_is_uncompressible(atom_fn); if(attempt_compress) { comp_reset(ctx); - const size_t csizeBound = comp_max_output_size(ctx, ucsize); + const size_t csizeBound = comp_max_output_size(ctx, usize); RETURN_ERR(comp_alloc_output(ctx, csizeBound)); } // read file into newly allocated buffer. if attempt_compress, also // compress the file into another buffer while waiting for IOs. - size_t ucsize_read; + size_t usize_read; const uint flags = 0; CompressParams params = { attempt_compress, ctx, 0 }; - RETURN_ERR(vfs_load(atom_fn, buf, ucsize_read, flags, compress_cb, (uintptr_t)¶ms)); - debug_assert(ucsize_read == ucsize); + RETURN_ERR(vfs_load(atom_fn, buf, usize_read, flags, compress_cb, (uintptr_t)¶ms)); + debug_assert(usize_read == usize); // if we compressed the file trial-wise, check results and // decide whether to store as such or not (based on compression ratio) @@ -149,11 +149,11 @@ static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx, return ret; } - shouldCompress = ShouldCompress(ucsize, csize); + shouldCompress = ShouldCompress(usize, csize); } // store file info - ent.ucsize = (off_t)ucsize; + ent.usize = (off_t)usize; ent.mtime = s.st_mtime; // .. ent.ofs is set by zip_archive_add_file ent.flags = 0; @@ -168,7 +168,7 @@ static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx, else { ent.method = CM_NONE; - ent.csize = (off_t)ucsize; + ent.csize = (off_t)usize; file_contents = buf; } diff --git a/source/lib/res/file/compression.h b/source/lib/res/file/compression.h index 4956457999..3f83d8d1e3 100644 --- a/source/lib/res/file/compression.h +++ b/source/lib/res/file/compression.h @@ -18,7 +18,6 @@ namespace ERR const LibError COMPRESSION_UNKNOWN_METHOD = -110300; } - enum ContextType { CT_COMPRESSION, @@ -43,45 +42,62 @@ extern uintptr_t comp_alloc(ContextType type, CompressionMethod method); **/ extern size_t comp_max_output_size(uintptr_t ctx, size_t inSize); -// set output buffer. all subsequent comp_feed() calls will write into it. -// should only be called once (*) due to the comp_finish() interface - since -// that allows querying the output buffer, it must not be fragmented. -// * the previous output buffer is wiped out by comp_reset, so -// setting it again (once!) after that is allowed and required. -extern void comp_set_output(uintptr_t ctx, u8* out, size_t out_size); +/** + * set output buffer for subsequent comp_feed() calls. + * + * due to the comp_finish interface, output buffers must be contiguous or + * identical (otherwise IsAllowableOutputBuffer will complain). + **/ +extern void comp_set_output(uintptr_t ctx, u8* out, size_t outSize); -// [compression contexts only:] allocate an output buffer big enough to -// hold worst_case_compression_ratio*in_size bytes. -// rationale: this interface is useful because callers cannot -// reliably estimate how much output space is needed. -// raises a warning for decompression contexts because this operation -// does not make sense there: -// - worst-case decompression ratio is quite large - ballpark 1000x; -// - exact uncompressed size is known to caller (via archive file header). -// note: buffer is held until comp_free; it can be reused after a -// comp_reset. this reduces malloc/free calls. -extern LibError comp_alloc_output(uintptr_t ctx, size_t in_size); +/** + * allocate a new output buffer. + * + * @param size [bytes] to allocate. + * + * if a buffer had previously been allocated and is large enough, it is + * reused (this reduces the number of allocations). the buffer is + * automatically freed by comp_free. + **/ +extern LibError comp_alloc_output(uintptr_t ctx, size_t inSize); -// 'feed' the given buffer to the compressor/decompressor. -// returns number of output bytes produced (*), or a negative LibError code. -// * 0 is a legitimate return value - this happens if the input buffer is -// small and the codec hasn't produced any output. -// note: the buffer may be overwritten or freed immediately after - we take -// care of copying and queuing any data that remains (e.g. due to -// lack of output buffer space). -extern ssize_t comp_feed(uintptr_t ctx, const u8* in, size_t in_size); +/** + * 'feed' the given buffer to the compressor/decompressor. + * + * @return number of output bytes produced or a negative LibError. + * note that 0 is a legitimate return value - this happens if the input + * buffer is small and the codec hasn't produced any output. + * + * note: after this call returns, the buffer may be overwritten or freed; + * we take care of copying and queuing any data that remains (e.g. due to + * lack of output buffer space). + **/ +extern ssize_t comp_feed(uintptr_t ctx, const u8* in, size_t inSize); -// feed any remaining queued input data, finish the compress/decompress and -// pass back the output buffer. +/** + * conclude the compression/decompression operation. + * + * @param out, outSize receive the output buffer. this assumes identical or + * contiguous addresses were passed, which comp_set_output ensures. + * @param checksum + * + * note: this must always be called (even if the output buffer is already + * known) because it feeds any remaining queued input buffers. + **/ extern LibError comp_finish(uintptr_t ctx, u8** out, size_t* out_size, u32* checksum); -// prepare this context for reuse. the effect is similar to freeing this -// context and creating another. -// rationale: this API avoids reallocating a considerable amount of -// memory (ballbark 200KB LZ window plus output buffer). +/** + * clear all previous state and prepare for reuse. + * + * this is as if the object were destroyed and re-created, but more + * efficient since it avoids reallocating a considerable amount of memory + * (about 200KB for LZ). + **/ extern void comp_reset(uintptr_t ctx); -// free this context and all associated memory. +/** + * free this context and all associated memory. + **/ extern void comp_free(uintptr_t ctx); #endif // #ifndef INCLUDED_COMPRESSION diff --git a/source/lib/res/file/tests/test_compression.h b/source/lib/res/file/tests/test_compression.h index 982ad390bc..7ddf2a363d 100644 --- a/source/lib/res/file/tests/test_compression.h +++ b/source/lib/res/file/tests/test_compression.h @@ -16,7 +16,7 @@ public: data[i] = rand() & 0x07; u8* cdata; size_t csize; - u8 ucdata[data_size]; + u8 udata[data_size]; // compress uintptr_t c = comp_alloc(CT_COMPRESSION, CM_DEFLATE); @@ -35,20 +35,20 @@ public: uintptr_t d = comp_alloc(CT_DECOMPRESSION, CM_DEFLATE); { TS_ASSERT(d != 0); - comp_set_output(d, ucdata, data_size); - const ssize_t ucdata_produced = comp_feed(d, cdata, csize); - TS_ASSERT(ucdata_produced >= 0); - u8* ucdata_final; size_t ucsize_final; u32 checksum; - TS_ASSERT_OK(comp_finish(d, &ucdata_final, &ucsize_final, &checksum)); - TS_ASSERT(ucdata_produced <= (ssize_t)ucsize_final); // can't have produced more than total - TS_ASSERT_EQUALS(ucdata_final, ucdata); // output buffer address is same - TS_ASSERT_EQUALS(ucsize_final, data_size); // correct amount of output + comp_set_output(d, udata, data_size); + const ssize_t udata_produced = comp_feed(d, cdata, csize); + TS_ASSERT(udata_produced >= 0); + u8* udata_final; size_t usize_final; u32 checksum; + TS_ASSERT_OK(comp_finish(d, &udata_final, &usize_final, &checksum)); + TS_ASSERT(udata_produced <= (ssize_t)usize_final); // can't have produced more than total + TS_ASSERT_EQUALS(udata_final, udata); // output buffer address is same + TS_ASSERT_EQUALS(usize_final, data_size); // correct amount of output } comp_free(c); comp_free(d); // verify data survived intact - TS_ASSERT_SAME_DATA(data, ucdata, data_size); + TS_ASSERT_SAME_DATA(data, udata, data_size); } }; diff --git a/source/lib/res/file/zip.cpp b/source/lib/res/file/zip.cpp index 917f6aa9b9..f220aecaa1 100644 --- a/source/lib/res/file/zip.cpp +++ b/source/lib/res/file/zip.cpp @@ -128,7 +128,7 @@ struct LFH u32 fat_mtime; // last modified time (DOS FAT format) u32 crc; u32 csize; - u32 ucsize; + u32 usize; u16 fn_len; u16 e_len; }; @@ -154,7 +154,7 @@ struct CDFH u32 fat_mtime; // last modified time (DOS FAT format) u32 crc; u32 csize; - u32 ucsize; + u32 usize; u16 fn_len; u16 e_len; u16 c_len; @@ -203,7 +203,7 @@ static off_t lfh_total_size(const LFH* lfh_le) static void lfh_assemble(LFH* lfh_le, CompressionMethod method, time_t mtime, u32 crc, - off_t csize, off_t ucsize, size_t fn_len) + off_t csize, off_t usize, size_t fn_len) { const ZipCompressionMethod zip_method = zip_method_for(method); const u32 fat_mtime = FAT_from_time_t(mtime); @@ -215,20 +215,20 @@ static void lfh_assemble(LFH* lfh_le, lfh_le->fat_mtime = to_le32(fat_mtime); lfh_le->crc = to_le32(crc); lfh_le->csize = to_le32(u32_from_larger(csize)); - lfh_le->ucsize = to_le32(u32_from_larger(ucsize)); + lfh_le->usize = to_le32(u32_from_larger(usize)); lfh_le->fn_len = to_le16(u16_from_larger(fn_len)); lfh_le->e_len = to_le16(0); } static void cdfh_decompose(const CDFH* cdfh_le, - CompressionMethod& method, time_t& mtime, off_t& csize, off_t& ucsize, + CompressionMethod& method, time_t& mtime, off_t& csize, off_t& usize, const char*& fn, off_t& lfh_ofs, size_t& total_size) { const u16 zip_method = read_le16(&cdfh_le->method); const u32 fat_mtime = read_le32(&cdfh_le->fat_mtime); csize = (off_t)read_le32(&cdfh_le->csize); - ucsize = (off_t)read_le32(&cdfh_le->ucsize); + usize = (off_t)read_le32(&cdfh_le->usize); const u16 fn_len = read_le16(&cdfh_le->fn_len); const u16 e_len = read_le16(&cdfh_le->e_len); const u16 c_len = read_le16(&cdfh_le->c_len); @@ -249,7 +249,7 @@ static void cdfh_decompose(const CDFH* cdfh_le, static void cdfh_assemble(CDFH* dst_cdfh_le, CompressionMethod method, time_t mtime, u32 crc, - size_t csize, size_t ucsize, size_t fn_len, size_t slack, u32 lfh_ofs) + size_t csize, size_t usize, size_t fn_len, size_t slack, u32 lfh_ofs) { const ZipCompressionMethod zip_method = zip_method_for(method); const u32 fat_mtime = FAT_from_time_t(mtime); @@ -261,7 +261,7 @@ static void cdfh_assemble(CDFH* dst_cdfh_le, dst_cdfh_le->fat_mtime = to_le32(fat_mtime); dst_cdfh_le->crc = to_le32(crc); dst_cdfh_le->csize = to_le32(u32_from_larger(csize)); - dst_cdfh_le->ucsize = to_le32(u32_from_larger(ucsize)); + dst_cdfh_le->usize = to_le32(u32_from_larger(usize)); dst_cdfh_le->fn_len = to_le16(u16_from_larger(fn_len)); dst_cdfh_le->e_len = to_le16(0); dst_cdfh_le->c_len = to_le16(u16_from_larger(slack)); @@ -453,11 +453,11 @@ LibError zip_populate_archive(File* f, Archive* a) // copy translated fields from CDFH into ArchiveEntry. ArchiveEntry ae; - cdfh_decompose(cdfh, ae.method, ae.mtime, ae.csize, ae.ucsize, ae.atom_fn, ae.ofs, ofs_to_next_cdfh); + cdfh_decompose(cdfh, ae.method, ae.mtime, ae.csize, ae.usize, ae.atom_fn, ae.ofs, ofs_to_next_cdfh); ae.flags = ZIP_LFH_FIXUP_NEEDED; // if file (we don't care about directories): - if(ae.csize && ae.ucsize) + if(ae.csize && ae.usize) { ret = archive_add_file(a, &ae); if(ret != INFO::OK) @@ -587,7 +587,7 @@ LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, const u8* // write (LFH, filename, file contents) to archive // .. put LFH and filename into one 'package' LFH_Package header; - lfh_assemble(&header.lfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->ucsize, fn_len); + lfh_assemble(&header.lfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->usize, fn_len); strcpy_s(header.fn, ARRAY_SIZE(header.fn), ae->atom_fn); // .. write that out in 1 IO const off_t lfh_ofs = za->cur_file_size; @@ -606,7 +606,7 @@ LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, const u8* if(!p) WARN_RETURN(ERR::NO_MEM); const size_t slack = za->cdfhs.da.pos-prev_pos - (CDFH_SIZE+fn_len); - cdfh_assemble(&p->cdfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->ucsize, fn_len, slack, lfh_ofs); + cdfh_assemble(&p->cdfh, ae->method, ae->mtime, ae->crc, ae->csize, ae->usize, fn_len, slack, lfh_ofs); cpu_memcpy(p->fn, ae->atom_fn, fn_len); za->cd_entries++;