diff --git a/source/lib/file/archive/archive.h b/source/lib/file/archive/archive.h index 614d045593..892a5fd05b 100644 --- a/source/lib/file/archive/archive.h +++ b/source/lib/file/archive/archive.h @@ -11,7 +11,10 @@ #ifndef INCLUDED_ARCHIVE #define INCLUDED_ARCHIVE -#include "lib/file/filesystem.h" +#include "lib/file/path.h" +#include "lib/file/file_system.h" // FileInfo +#include "lib/file/common/file_loader.h" +#include "lib/file/vfs/vfs_path.h" // rationale: this module doesn't build a directory tree of the entries // within an archive. that task is left to the VFS; here, we are only @@ -23,23 +26,25 @@ namespace ERR const LibError ARCHIVE_UNKNOWN_METHOD = -110401; } -// opaque 'memento' of an archive entry. the instances are stored by -// the IArchiveReader implementation. -struct ArchiveEntry; +struct IArchiveFile : public IFileLoader +{ +}; -struct IArchiveReader : public IFileProvider +typedef shared_ptr PIArchiveFile; + +struct IArchiveReader { virtual ~IArchiveReader(); /** * called for each archive entry. - * @param pathname full pathname of entry (unique pointer) + * @param pathname full pathname of entry; only valid during the callback. **/ - typedef void (*ArchiveEntryCallback)(const char* pathname, const FileInfo& fileInfo, const ArchiveEntry* archiveEntry, uintptr_t cbData); + typedef void (*ArchiveEntryCallback)(const VfsPath& pathname, const FileInfo& fileInfo, PIArchiveFile archiveFile, uintptr_t cbData); virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) = 0; }; -typedef boost::shared_ptr PIArchiveReader; +typedef shared_ptr PIArchiveReader; // note: when creating an archive, any existing file with the given pathname // will be overwritten. @@ -68,9 +73,9 @@ struct IArchiveWriter * precisely because they aren't in archives, and the cache would * thrash anyway, so this is deemed acceptable. **/ - virtual LibError AddFile(const char* pathname) = 0; + virtual LibError AddFile(const Path& pathname) = 0; }; -typedef boost::shared_ptr PIArchiveWriter; +typedef shared_ptr PIArchiveWriter; #endif // #ifndef INCLUDED_ARCHIVE diff --git a/source/lib/file/archive/archive_builder.cpp b/source/lib/file/archive/archive_builder.cpp index 1e4547b6ac..5c304ef1fa 100644 --- a/source/lib/file/archive/archive_builder.cpp +++ b/source/lib/file/archive/archive_builder.cpp @@ -12,6 +12,8 @@ #include "precompiled.h" //#include "vfs_optimizer.h" +#if 0 + #include #include #include @@ -720,7 +722,7 @@ int archive_build_continue(ArchiveBuildState* ab) if(!V_fn) break; - ArchiveEntry ent; const u8* file_contents; IoBuf buf; + IArchiveFile ent; const u8* file_contents; IoBuf buf; if(read_and_compress_file(V_fn, ab->codec, ent, file_contents, buf) == INFO::OK) { (void)ab->archiveBuilder->AddFile(&ent, file_contents); @@ -1092,3 +1094,5 @@ LibError trace_run(const char* osPathname) trace.Entries()[i]->Run(); return INFO::OK; } + +#endif \ No newline at end of file diff --git a/source/lib/file/archive/archive_zip.cpp b/source/lib/file/archive/archive_zip.cpp index bd013469f0..257dc7493b 100644 --- a/source/lib/file/archive/archive_zip.cpp +++ b/source/lib/file/archive/archive_zip.cpp @@ -23,85 +23,47 @@ #include "archive.h" #include "codec_zlib.h" #include "stream.h" -#include "lib/file/filesystem.h" -#include "lib/file/posix/fs_posix.h" -#include "lib/file/posix/io_posix.h" -#include "lib/file/io/io_manager.h" +#include "lib/file/file_system_posix.h" +#include "lib/file/io/io.h" + +static FileSystem_Posix s_posixDirectory; //----------------------------------------------------------------------------- -// Zip file data structures and signatures +// Zip archive definitions //----------------------------------------------------------------------------- +static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2'); +static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4'); +static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6'); + enum ZipMethod { ZIP_METHOD_NONE = 0, ZIP_METHOD_DEFLATE = 8 }; -enum ZipFlags -{ - // indicates ZipEntry.ofs points to a "local file header" instead of - // the file data. a fixup routine is called when reading the file; - // it skips past the LFH and clears this flag. - // this is somewhat of a hack, but vital to archive open performance. - // without it, we'd have to scan through the entire archive file, - // which can take *seconds*. - // (we cannot use the information in CDFH, because its 'extra' field - // has been observed to differ from that of the LFH) - // since we read the LFH right before the rest of the file, the block - // cache will absorb the IO cost. - NEEDS_LFH_FIXUP = 1 -}; - -// all relevant LFH/CDFH fields not covered by FileInfo -struct ZipEntry -{ - ZipEntry() - { - } - - ZipEntry(off_t ofs, off_t csize, u32 checksum, u16 method, u16 flags) - { - this->ofs = ofs; - this->csize = csize; - this->checksum = checksum; - this->method = method; - this->flags = flags; - } - - off_t ofs; - off_t csize; - - u32 checksum; - u16 method; - u16 flags; -}; - - -static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2'); -static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4'); -static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6'); - #pragma pack(push, 1) class LFH { public: - void Init(const FileInfo& fileInfo, const ZipEntry& zipEntry, const char* pathname, size_t pathnameLength) + void Init(const FileInfo& fileInfo, off_t csize, ZipMethod method, u32 checksum, const VfsPath& pathname_) { + const std::string& pathname = pathname_.string(); + m_magic = lfh_magic; m_x1 = to_le16(0); m_flags = to_le16(0); - m_method = to_le16(u16_from_larger(zipEntry.method)); + m_method = to_le16(u16_from_larger(method)); m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime())); - m_crc = to_le32(zipEntry.checksum); - m_csize = to_le32(u32_from_larger(zipEntry.csize)); + m_crc = to_le32(checksum); + m_csize = to_le32(u32_from_larger(csize)); m_usize = to_le32(u32_from_larger(fileInfo.Size())); - m_fn_len = to_le16(u16_from_larger((uint)pathnameLength)); + m_fn_len = to_le16(u16_from_larger(pathname.length())); m_e_len = to_le16(0); - cpu_memcpy((char*)this + sizeof(LFH), pathname, pathnameLength); + cpu_memcpy((char*)this + sizeof(LFH), pathname.c_str(), pathname.length()); } size_t Size() const @@ -133,49 +95,72 @@ cassert(sizeof(LFH) == 30); class CDFH { public: - void Init(const FileInfo& fileInfo, const ZipEntry& zipEntry, const char* pathname, size_t pathnameLength, size_t slack) + void Init(const FileInfo& fileInfo, off_t ofs, off_t csize, ZipMethod method, u32 checksum, const VfsPath& pathname_, size_t slack) { + const std::string& pathname = pathname_.string(); m_magic = cdfh_magic; m_x1 = to_le32(0); m_flags = to_le16(0); - m_method = to_le16(u16_from_larger(zipEntry.method)); + m_method = to_le16(u16_from_larger(method)); m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime())); - m_crc = to_le32(zipEntry.checksum); - m_csize = to_le32(u32_from_larger(zipEntry.csize)); + m_crc = to_le32(checksum); + m_csize = to_le32(u32_from_larger(csize)); m_usize = to_le32(u32_from_larger(fileInfo.Size())); - m_fn_len = to_le16(u16_from_larger((uint)pathnameLength)); + m_fn_len = to_le16(u16_from_larger(pathname.length())); m_e_len = to_le16(0); m_c_len = to_le16(u16_from_larger((uint)slack)); m_x2 = to_le32(0); m_x3 = to_le32(0); - m_lfh_ofs = to_le32(zipEntry.ofs); + m_lfh_ofs = to_le32(ofs); - cpu_memcpy((char*)this + sizeof(CDFH), pathname, pathnameLength); + cpu_memcpy((char*)this + sizeof(CDFH), pathname.c_str(), pathname.length()); } - void Decompose(FileInfo& fileInfo, ZipEntry& zipEntry, const char*& pathname, size_t& cdfhSize) const + VfsPath GetPathname() const { - const u16 zipMethod = read_le16(&m_method); - const u32 fat_mtime = read_le32(&m_fat_mtime); - const u32 crc = read_le32(&m_crc); - const u32 csize = read_le32(&m_csize); - const u32 usize = read_le32(&m_usize); - const u16 fn_len = read_le16(&m_fn_len); - const u16 e_len = read_le16(&m_e_len); - const u16 c_len = read_le16(&m_c_len); - const u32 lfh_ofs = read_le32(&m_lfh_ofs); - - // get 0-terminated copy of pathname + const size_t length = (size_t)read_le16(&m_fn_len); const char* fn = (const char*)this + sizeof(CDFH); // not 0-terminated! - char buf[PATH_MAX]; // path_Pool()->UniqueCopy requires a 0-terminated string - cpu_memcpy(buf, fn, fn_len*sizeof(char)); - buf[fn_len] = '\0'; - pathname = path_Pool()->UniqueCopy(buf); + return VfsPath(std::string(fn, length)); + } - fileInfo = FileInfo(path_name_only(pathname), (off_t)usize, time_t_from_FAT(fat_mtime)); - zipEntry = ZipEntry((off_t)lfh_ofs, (off_t)csize, crc, zipMethod, NEEDS_LFH_FIXUP); + off_t HeaderOffset() const + { + return read_le32(&m_lfh_ofs); + } - cdfhSize = sizeof(CDFH) + fn_len + e_len + c_len; + off_t USize() const + { + return (off_t)read_le32(&m_usize); + } + + off_t CSize() const + { + return (off_t)read_le32(&m_csize); + } + + ZipMethod Method() const + { + return (ZipMethod)read_le16(&m_method); + } + + u32 Checksum() const + { + return read_le32(&m_crc); + } + + time_t MTime() const + { + const u32 fat_mtime = read_le32(&m_fat_mtime); + return time_t_from_FAT(fat_mtime); + } + + size_t Size() const + { + size_t size = sizeof(CDFH); + size += read_le16(&m_fn_len); + size += read_le16(&m_e_len); + size += read_le16(&m_c_len); + return size; } private: @@ -201,7 +186,7 @@ cassert(sizeof(CDFH) == 46); class ECDR { public: - void Init(uint cd_numEntries, off_t cd_ofs, size_t cd_size) + void Init(uint cd_numEntries, off_t cd_ofs, off_t cd_size) { m_magic = ecdr_magic; memset(m_x1, 0, sizeof(m_x1)); @@ -211,11 +196,11 @@ public: m_comment_len = to_le16(0); } - void Decompose(uint& cd_numEntries, off_t& cd_ofs, size_t& cd_size) const + void Decompose(uint& cd_numEntries, off_t& cd_ofs, off_t& cd_size) const { cd_numEntries = (uint)read_le16(&m_cd_numEntries); - cd_ofs = (off_t)read_le32(&m_cd_ofs); - cd_size = (size_t)read_le32(&m_cd_size); + cd_ofs = (off_t)read_le32(&m_cd_ofs); + cd_size = (off_t)read_le32(&m_cd_size); } private: @@ -233,143 +218,17 @@ cassert(sizeof(ECDR) == 22); //----------------------------------------------------------------------------- - -/** - * scan buffer for a Zip file record. - * - * @param start position within buffer - * @param magic signature of record - * @param recordSize size of record (including signature) - * @return pointer to record within buffer or 0 if not found. - **/ -static const u8* zip_FindRecord(const u8* buf, size_t size, const u8* start, u32 magic, size_t recordSize) -{ - // (don't use as the counter - otherwise we can't tell if - // scanning within the buffer was necessary.) - for(const u8* p = start; p <= buf+size-recordSize; p++) - { - // found it - if(*(u32*)p == magic) - { - debug_assert(p == start); // otherwise, the archive is a bit broken - return p; - } - } - - // passed EOF, didn't find it. - // note: do not warn - this happens in the initial ECDR search at - // EOF if the archive contains a comment field. - return 0; -} - - -// search for ECDR in the last bytes of the file. -// if found, fill with a copy of the (little-endian) ECDR and -// return INFO::OK, otherwise IO error or ERR::CORRUPTED. -static LibError zip_ReadECDR(const File_Posix& file, off_t fileSize, IoBuf& buf, size_t maxScanSize, uint& cd_numEntries, off_t& cd_ofs, size_t& cd_size) -{ - // don't scan more than the entire file - const size_t scanSize = std::min(maxScanSize, (size_t)fileSize); - - // read desired chunk of file into memory - const off_t ofs = fileSize - (off_t)scanSize; - RETURN_ERR(io_Read(file, ofs, buf, scanSize)); - - // look for ECDR in buffer - const u8* start = (const u8*)buf.get(); - const ECDR* ecdr = (const ECDR*)zip_FindRecord(start, scanSize, start, ecdr_magic, sizeof(ECDR)); - if(!ecdr) - WARN_RETURN(INFO::CANNOT_HANDLE); - - ecdr->Decompose(cd_numEntries, cd_ofs, cd_size); - return INFO::OK; -} - - -static LibError zip_LocateCentralDirectory(const File_Posix& file, off_t fileSize, off_t& cd_ofs, uint& cd_numEntries, size_t& cd_size) -{ - const size_t maxScanSize = 66000u; // see below - IoBuf buf = io_buf_Allocate(maxScanSize); - - // expected case: ECDR at EOF; no file comment - LibError ret = zip_ReadECDR(file, fileSize, buf, sizeof(ECDR), cd_numEntries, cd_ofs, cd_size); - if(ret == INFO::OK) - return INFO::OK; - // worst case: ECDR precedes 64 KiB of file comment - ret = zip_ReadECDR(file, fileSize, buf, maxScanSize, cd_numEntries, cd_ofs, cd_size); - if(ret == INFO::OK) - return INFO::OK; - - // both ECDR scans failed - this is not a valid Zip file. - RETURN_ERR(io_Read(file, 0, buf, sizeof(LFH))); - // the Zip file has an LFH but lacks an ECDR. this can happen if - // the user hard-exits while an archive is being written. - // notes: - // - return ERR::CORRUPTED so VFS will not include this file. - // - we could work around this by scanning all LFHs, but won't bother - // because it'd be slow. - // - do not warn - the corrupt archive will be deleted on next - // successful archive builder run anyway. - if(zip_FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH))) - return ERR::CORRUPTED; // NOWARN - // totally bogus - else - WARN_RETURN(ERR::ARCHIVE_UNKNOWN_FORMAT); -} - - -//----------------------------------------------------------------------------- -// ArchiveReader_Zip +// ArchiveFile_Zip //----------------------------------------------------------------------------- -static Filesystem_Posix fsPosix; - -class ArchiveReader_Zip : public IArchiveReader +class ArchiveFile_Zip : public IArchiveFile { public: - ArchiveReader_Zip(const char* pathname) + ArchiveFile_Zip(PFile file, off_t ofs, off_t csize, u32 checksum, ZipMethod method) + : m_file(file), m_ofs(ofs) + , m_csize(csize), m_checksum(checksum), m_method((u16)method) + , m_flags(NeedsFixup) { - m_file.Open(pathname, 'r'); - - FileInfo fileInfo; - fsPosix.GetFileInfo(pathname, fileInfo); - m_fileSize = fileInfo.Size(); - debug_assert(m_fileSize >= sizeof(LFH)+sizeof(CDFH)+sizeof(ECDR)); - } - - virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) - { - // locate and read Central Directory - off_t cd_ofs; uint cd_numEntries; size_t cd_size; - RETURN_ERR(zip_LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size)); - IoBuf buf = io_buf_Allocate(cd_size); - RETURN_ERR(io_Read(m_file, cd_ofs, buf, cd_size)); - const u8* cd = (const u8*)buf.get(); - - m_entries.reserve(cd_numEntries); - - // iterate over Central Directory - const u8* pos = cd; - for(uint i = 0; i < cd_numEntries; i++) - { - // scan for next CDFH - CDFH* cdfh = (CDFH*)zip_FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH)); - if(!cdfh) - WARN_RETURN(ERR::CORRUPTED); - - FileInfo fileInfo; - ZipEntry zipEntry; - const char* pathname; - size_t cdfhSize; - cdfh->Decompose(fileInfo, zipEntry, pathname, cdfhSize); - - m_entries.push_back(zipEntry); - cb(pathname, fileInfo, (ArchiveEntry*)&m_entries.back(), cbData); - - pos += cdfhSize; - } - - return INFO::OK; } virtual unsigned Precedence() const @@ -382,18 +241,12 @@ public: return 'A'; } - virtual LibError Load(const char* UNUSED(name), const void* location, u8* fileContents, size_t size) const + virtual LibError Load(const std::string& UNUSED(name), shared_ptr buf, size_t size) const { - ZipEntry* zipEntry = (ZipEntry*)location; + AdjustOffset(); - if((zipEntry->flags & NEEDS_LFH_FIXUP) != 0) - { - FixupEntry(*zipEntry); - zipEntry->flags &= ~NEEDS_LFH_FIXUP; - } - - boost::shared_ptr codec; - switch(zipEntry->method) + PICodec codec; + switch(m_method) { case ZIP_METHOD_NONE: codec = CreateCodec_ZLibNone(); @@ -406,20 +259,32 @@ public: } Stream stream(codec); - stream.SetOutputBuffer(fileContents, size); - RETURN_ERR(io_Read(m_file, zipEntry->ofs, IO_BUF_TEMP, zipEntry->csize, FeedStream, (uintptr_t)&stream)); + stream.SetOutputBuffer(buf.get(), size); + RETURN_ERR(io_Scan(*m_file, m_ofs, m_csize, FeedStream, (uintptr_t)&stream)); RETURN_ERR(stream.Finish()); - debug_assert(zipEntry->checksum == stream.Checksum()); +#if CODEC_COMPUTE_CHECKSUM + debug_assert(m_checksum == stream.Checksum()); +#endif return INFO::OK; } - virtual LibError Store(const char* UNUSED(name), const void* UNUSED(location), const u8* UNUSED(fileContents), size_t UNUSED(size)) const - { - return ERR::NOT_SUPPORTED; - } - private: + enum Flags + { + // indicates m_ofs points to a "local file header" instead of + // the file data. a fixup routine is called when reading the file; + // it skips past the LFH and clears this flag. + // this is somewhat of a hack, but vital to archive open performance. + // without it, we'd have to scan through the entire archive file, + // which can take *seconds*. + // (we cannot use the information in CDFH, because its 'extra' field + // has been observed to differ from that of the LFH) + // since we read the LFH right before the rest of the file, the block + // cache will absorb the IO cost. + NeedsFixup = 1 + }; + struct LFH_Copier { u8* lfh_dst; @@ -433,7 +298,7 @@ private: // rationale: this allows using temp buffers for zip_fixup_lfh, // which avoids involving the file buffer manager and thus // avoids cluttering the trace and cache contents. - static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size, size_t& bytesProcessed) + static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size) { LFH_Copier* p = (LFH_Copier*)cbData; @@ -442,76 +307,198 @@ private: p->lfh_dst += size; p->lfh_bytes_remaining -= size; - bytesProcessed = size; return INFO::CB_CONTINUE; } /** - * fix up zipEntry's offset within the archive. - * called by archive_LoadFile iff NEEDS_LFH_FIXUP is set. - * side effects: adjusts zipEntry.ofs. + * fix up m_ofs (adjust it to point to cdata instead of the LFH). * - * ensures points to the actual file contents; it is initially - * the offset of the LFH. we cannot use CDFH filename and extra field - * lengths to skip past LFH since that may not mirror CDFH (has happened). + * note: we cannot use CDFH filename and extra field lengths to skip + * past LFH since that may not mirror CDFH (has happened). * * this is called at file-open time instead of while mounting to * reduce seeks: since reading the file will typically follow, the * block cache entirely absorbs the IO cost. **/ - void FixupEntry(ZipEntry& zipEntry) const + void AdjustOffset() const { + if(!(m_flags & NeedsFixup)) + return; + m_flags &= ~NeedsFixup; + // performance note: this ends up reading one file block, which is // only in the block cache if the file starts in the same block as a // previously read file (i.e. both are small). LFH lfh; LFH_Copier params = { (u8*)&lfh, sizeof(LFH) }; - THROW_ERR(io_Read(m_file, zipEntry.ofs, IO_BUF_TEMP, sizeof(LFH), lfh_copier_cb, (uintptr_t)¶ms)); - - zipEntry.ofs += (off_t)lfh.Size(); + if(io_Scan(*m_file, m_ofs, sizeof(LFH), lfh_copier_cb, (uintptr_t)¶ms) == INFO::OK) + m_ofs += (off_t)lfh.Size(); } - File_Posix m_file; - off_t m_fileSize; - std::vector m_entries; + PFile m_file; + + // all relevant LFH/CDFH fields not covered by FileInfo + mutable off_t m_ofs; + off_t m_csize; + u32 m_checksum; + u16 m_method; + mutable u16 m_flags; }; -boost::shared_ptr CreateArchiveReader_Zip(const char* archivePathname) +//----------------------------------------------------------------------------- +// ArchiveReader_Zip +//----------------------------------------------------------------------------- + +class ArchiveReader_Zip : public IArchiveReader +{ +public: + ArchiveReader_Zip(const Path& pathname) + : m_file(new File) + { + m_file->Open(pathname, 'r'); + + FileInfo fileInfo; + s_posixDirectory.GetFileInfo(pathname, &fileInfo); + m_fileSize = fileInfo.Size(); + debug_assert(m_fileSize >= sizeof(LFH)+sizeof(CDFH)+sizeof(ECDR)); + } + + virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) + { + // locate and read Central Directory + off_t cd_ofs; uint cd_numEntries; off_t cd_size; + RETURN_ERR(LocateCentralDirectory(*m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size)); + shared_ptr buf = io_Allocate(cd_size, cd_ofs); + u8* cd; + RETURN_ERR(io_Read(*m_file, cd_ofs, buf.get(), cd_size, cd)); + + // iterate over Central Directory + const u8* pos = cd; + for(uint i = 0; i < cd_numEntries; i++) + { + // scan for next CDFH + CDFH* cdfh = (CDFH*)FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH)); + if(!cdfh) + WARN_RETURN(ERR::CORRUPTED); + + const std::string& pathname = cdfh->GetPathname().string(); + const size_t lastSlash = pathname.find_last_of('/'); + if(lastSlash != pathname.length()-1) // we only want files + { + const std::string name(pathname.begin()+lastSlash+1, pathname.end()); + FileInfo fileInfo(name, cdfh->USize(), cdfh->MTime()); + shared_ptr archiveFile(new ArchiveFile_Zip(m_file, cdfh->HeaderOffset(), cdfh->CSize(), cdfh->Checksum(), cdfh->Method())); + cb(pathname, fileInfo, archiveFile, cbData); + } + + pos += cdfh->Size(); + } + + return INFO::OK; + } + +private: + /** + * scan buffer for a Zip file record. + * + * @param start position within buffer + * @param magic signature of record + * @param recordSize size of record (including signature) + * @return pointer to record within buffer or 0 if not found. + **/ + static const u8* FindRecord(const u8* buf, size_t size, const u8* start, u32 magic, size_t recordSize) + { + // (don't use as the counter - otherwise we can't tell if + // scanning within the buffer was necessary.) + for(const u8* p = start; p <= buf+size-recordSize; p++) + { + // found it + if(*(u32*)p == magic) + { + debug_assert(p == start); // otherwise, the archive is a bit broken + return p; + } + } + + // passed EOF, didn't find it. + // note: do not warn - this happens in the initial ECDR search at + // EOF if the archive contains a comment field. + return 0; + } + + // search for ECDR in the last bytes of the file. + // if found, fill with a copy of the (little-endian) ECDR and + // return INFO::OK, otherwise IO error or ERR::CORRUPTED. + static LibError ScanForEcdr(const File& file, off_t fileSize, u8* buf, off_t maxScanSize, uint& cd_numEntries, off_t& cd_ofs, off_t& cd_size) + { + // don't scan more than the entire file + const off_t scanSize = std::min(maxScanSize, fileSize); + + // read desired chunk of file into memory + const off_t ofs = fileSize - scanSize; + u8* data; + RETURN_ERR(io_Read(file, ofs, buf, scanSize, data)); + + // look for ECDR in buffer + const ECDR* ecdr = (const ECDR*)FindRecord(data, scanSize, data, ecdr_magic, sizeof(ECDR)); + if(!ecdr) + return INFO::CANNOT_HANDLE; + + ecdr->Decompose(cd_numEntries, cd_ofs, cd_size); + return INFO::OK; + } + + static LibError LocateCentralDirectory(const File& file, off_t fileSize, off_t& cd_ofs, uint& cd_numEntries, off_t& cd_size) + { + const off_t maxScanSize = 66000u; // see below + shared_ptr buf = io_Allocate(maxScanSize, ~0); // assume worst-case for alignment + + // expected case: ECDR at EOF; no file comment + LibError ret = ScanForEcdr(file, fileSize, const_cast(buf.get()), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size); + if(ret == INFO::OK) + return INFO::OK; + // worst case: ECDR precedes 64 KiB of file comment + ret = ScanForEcdr(file, fileSize, const_cast(buf.get()), maxScanSize, cd_numEntries, cd_ofs, cd_size); + if(ret == INFO::OK) + return INFO::OK; + + // both ECDR scans failed - this is not a valid Zip file. + RETURN_ERR(io_ReadAligned(file, 0, const_cast(buf.get()), sizeof(LFH))); + // the Zip file has an LFH but lacks an ECDR. this can happen if + // the user hard-exits while an archive is being written. + // notes: + // - return ERR::CORRUPTED so VFS will not include this file. + // - we could work around this by scanning all LFHs, but won't bother + // because it'd be slow. + // - do not warn - the corrupt archive will be deleted on next + // successful archive builder run anyway. + if(FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH))) + return ERR::CORRUPTED; // NOWARN + // totally bogus + else + WARN_RETURN(ERR::ARCHIVE_UNKNOWN_FORMAT); + } + + PFile m_file; + off_t m_fileSize; +}; + +PIArchiveReader CreateArchiveReader_Zip(const Path& archivePathname) { return PIArchiveReader(new ArchiveReader_Zip(archivePathname)); } //----------------------------------------------------------------------------- -// ArchiveWriterZip +// ArchiveWriter_Zip //----------------------------------------------------------------------------- -static bool IsFileTypeIncompressible(const char* extension) -{ - // file extensions that we don't want to compress - static const char* incompressibleExtensions[] = - { - "zip", "rar", - "jpg", "jpeg", "png", - "ogg", "mp3" - }; - - for(uint i = 0; i < ARRAY_SIZE(incompressibleExtensions); i++) - { - if(!strcasecmp(extension, incompressibleExtensions[i])) - return true; - } - - return false; -} - - class ArchiveWriter_Zip : public IArchiveWriter { public: - ArchiveWriter_Zip(const char* archivePathname) - : m_fileSize(0) + ArchiveWriter_Zip(const Path& archivePathname) + : m_fileSize(0), m_unalignedWriter(m_file, 0) , m_numEntries(0) { THROW_ERR(m_file.Open(archivePathname, 'w')); @@ -520,24 +507,24 @@ public: ~ArchiveWriter_Zip() { - const size_t cd_size = m_cdfhPool.da.pos; - // append an ECDR to the CDFH list (this allows us to // write out both to the archive file in one burst) + const off_t cd_size = (off_t)m_cdfhPool.da.pos; ECDR* ecdr = (ECDR*)pool_alloc(&m_cdfhPool, sizeof(ECDR)); - if(ecdr) - { - ecdr->Init(m_numEntries, m_fileSize, cd_size); - io_Write(m_file, m_fileSize, m_cdfhPool.da.base, cd_size+sizeof(ECDR)); - } + if(!ecdr) + throw std::bad_alloc(); + ecdr->Init(m_numEntries, m_fileSize, cd_size); + + m_unalignedWriter.Append(m_cdfhPool.da.base, cd_size+sizeof(ECDR)); + m_unalignedWriter.Flush(); (void)pool_destroy(&m_cdfhPool); } - LibError AddFile(const char* pathname) + LibError AddFile(const Path& pathname) { FileInfo fileInfo; - RETURN_ERR(fsPosix.GetFileInfo(pathname, fileInfo)); + RETURN_ERR(s_posixDirectory.GetFileInfo(pathname, &fileInfo)); const off_t usize = fileInfo.Size(); // skip 0-length files. // rationale: zip.cpp needs to determine whether a CDFH entry is @@ -550,27 +537,29 @@ public: if(!usize) return INFO::SKIPPED; - File_Posix file; + File file; RETURN_ERR(file.Open(pathname, 'r')); + const size_t pathnameLength = pathname.string().length(); + const VfsPath vfsPathname(pathname.string()); + // choose method and the corresponding codec - ZipMethod zipMethod; + ZipMethod method; PICodec codec; if(IsFileTypeIncompressible(pathname)) { - zipMethod = ZIP_METHOD_NONE; + method = ZIP_METHOD_NONE; codec = CreateCodec_ZLibNone(); } else { - zipMethod = ZIP_METHOD_DEFLATE; + method = ZIP_METHOD_DEFLATE; codec = CreateCompressor_ZLibDeflate(); } // allocate memory - const size_t pathnameLength = strlen(pathname); - const size_t csizeMax = codec.get()->MaxOutputSize(usize); - IoBuf buf = io_buf_Allocate(sizeof(LFH) + pathnameLength + csizeMax); + const size_t csizeMax = codec->MaxOutputSize(usize); + shared_ptr buf = io_Allocate(sizeof(LFH) + pathnameLength + csizeMax); // read and compress file contents size_t csize; u32 checksum; @@ -578,49 +567,68 @@ public: u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength; Stream stream(codec); stream.SetOutputBuffer(cdata, csizeMax); - RETURN_ERR(io_Read(file, 0, IO_BUF_TEMP, usize, FeedStream, (uintptr_t)&stream)); + RETURN_ERR(io_Scan(file, 0, usize, FeedStream, (uintptr_t)&stream)); RETURN_ERR(stream.Finish()); csize = stream.OutSize(); checksum = stream.Checksum(); } - ZipEntry zipEntry(m_fileSize, (off_t)csize, checksum, zipMethod, 0); - // build LFH { LFH* lfh = (LFH*)buf.get(); - lfh->Init(fileInfo, zipEntry, pathname, pathnameLength); + lfh->Init(fileInfo, (off_t)csize, method, checksum, vfsPathname); } - // write to LFH, pathname and cdata to file - const size_t packageSize = sizeof(LFH) + pathnameLength + csize; - RETURN_ERR(io_Write(m_file, m_fileSize, buf, packageSize)); - m_fileSize += (off_t)packageSize; - // append a CDFH to the central directory (in memory) - // .. note: pool_alloc may round size up for padding purposes. - const size_t prev_pos = m_cdfhPool.da.pos; + const off_t ofs = m_fileSize; + const size_t prev_pos = m_cdfhPool.da.pos; // (required to determine padding size) const size_t cdfhSize = sizeof(CDFH) + pathnameLength; CDFH* cdfh = (CDFH*)pool_alloc(&m_cdfhPool, cdfhSize); if(!cdfh) WARN_RETURN(ERR::NO_MEM); const size_t slack = m_cdfhPool.da.pos - prev_pos - cdfhSize; - cdfh->Init(fileInfo, zipEntry, pathname, pathnameLength, slack); - + cdfh->Init(fileInfo, ofs, (off_t)csize, method, checksum, vfsPathname, slack); m_numEntries++; + // write LFH, pathname and cdata to file + const size_t packageSize = sizeof(LFH) + pathnameLength + csize; + RETURN_ERR(m_unalignedWriter.Append(buf.get(), (off_t)packageSize)); + m_fileSize += (off_t)packageSize; + return INFO::OK; } - +#include private: - File_Posix m_file; + static bool IsFileTypeIncompressible(const Path& pathname) + { + const char* extension = path_extension(pathname.string().c_str()); + + // file extensions that we don't want to compress + static const char* incompressibleExtensions[] = + { + "zip", "rar", + "jpg", "jpeg", "png", + "ogg", "mp3" + }; + + for(uint i = 0; i < ARRAY_SIZE(incompressibleExtensions); i++) + { + if(!strcasecmp(extension, incompressibleExtensions[i])) + return true; + } + + return false; + } + + File m_file; off_t m_fileSize; + UnalignedWriter m_unalignedWriter; Pool m_cdfhPool; uint m_numEntries; }; -PIArchiveWriter CreateArchiveWriter_Zip(const char* archivePathname) +PIArchiveWriter CreateArchiveWriter_Zip(const Path& archivePathname) { return PIArchiveWriter(new ArchiveWriter_Zip(archivePathname)); } diff --git a/source/lib/file/archive/archive_zip.h b/source/lib/file/archive/archive_zip.h index 9fd0c9a602..ffd1c641fa 100644 --- a/source/lib/file/archive/archive_zip.h +++ b/source/lib/file/archive/archive_zip.h @@ -13,7 +13,7 @@ #include "archive.h" -PIArchiveReader CreateArchiveReader_Zip(const char* archivePathname); -PIArchiveWriter CreateArchiveWriter_Zip(const char* archivePathname); +PIArchiveReader CreateArchiveReader_Zip(const Path& archivePathname); +PIArchiveWriter CreateArchiveWriter_Zip(const Path& archivePathname); #endif // #ifndef INCLUDED_ARCHIVE_ZIP diff --git a/source/lib/file/archive/codec.h b/source/lib/file/archive/codec.h index 33605cf3a4..27dbd95c98 100644 --- a/source/lib/file/archive/codec.h +++ b/source/lib/file/archive/codec.h @@ -15,6 +15,8 @@ // besides ZLib. it also simplifies the interface for user code and // does error checking, etc. +#define CODEC_COMPUTE_CHECKSUM 1 + struct ICodec { public: @@ -67,6 +69,6 @@ public: virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const = 0; }; -typedef boost::shared_ptr PICodec; +typedef shared_ptr PICodec; #endif // #ifndef INCLUDED_CODEC diff --git a/source/lib/file/archive/codec_zlib.cpp b/source/lib/file/archive/codec_zlib.cpp index 9c62eb8985..4ee7013971 100644 --- a/source/lib/file/archive/codec_zlib.cpp +++ b/source/lib/file/archive/codec_zlib.cpp @@ -12,13 +12,24 @@ class Codec_ZLib : public ICodec public: u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const { +#if CODEC_COMPUTE_CHECKSUM return (u32)crc32(checksum, in, (uInt)inSize); +#else + UNUSED2(checksum); + UNUSED2(in); + UNUSED2(inSize); + return 0; +#endif } protected: u32 InitializeChecksum() { +#if CODEC_COMPUTE_CHECKSUM return crc32(0, 0, 0); +#else + return 0; +#endif } }; @@ -48,12 +59,12 @@ public: return INFO::OK; } - virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed) + virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) { const size_t transferSize = std::min(inSize, outSize); cpu_memcpy(out, in, transferSize); - inConsumed = outConsumed = transferSize; - m_checksum = UpdateChecksum(m_checksum, in, inSize); + inConsumed = outProduced = transferSize; + m_checksum = UpdateChecksum(m_checksum, out, outProduced); return INFO::OK; } @@ -105,7 +116,7 @@ protected: typedef int ZEXPORT (*ZLibFunc)(z_streamp strm, int flush); - LibError Process(ZLibFunc func, int flush, const u8* in, const size_t inSize, u8* out, const size_t outSize, size_t& inConsumed, size_t& outConsumed) + LibError Process(ZLibFunc func, int flush, const u8* in, const size_t inSize, u8* out, const size_t outSize, size_t& inConsumed, size_t& outProduced) { m_zs.next_in = (Byte*)in; m_zs.avail_in = (uInt)inSize; @@ -123,7 +134,7 @@ protected: debug_assert(inSize >= m_zs.avail_in && outSize >= m_zs.avail_out); inConsumed = inSize - m_zs.avail_in; - outConsumed = outSize - m_zs.avail_out; + outProduced = outSize - m_zs.avail_out; return LibError_from_zlib(ret); } @@ -182,10 +193,10 @@ public: return LibError_from_zlib(ret); } - virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed) + virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) { m_checksum = UpdateChecksum(m_checksum, in, inSize); - return CodecZLibStream::Process(deflate, 0, in, inSize, out, outSize, inConsumed, outConsumed); + return CodecZLibStream::Process(deflate, 0, in, inSize, out, outSize, inConsumed, outProduced); } virtual LibError Finish(u32& checksum) @@ -227,8 +238,7 @@ public: // so callers should use that when allocating the output buffer. debug_assert(inSize < 1*MiB); - // http://www.zlib.org/zlib_tech.html - return inSize*1032; + return inSize*1032; // see http://www.zlib.org/zlib_tech.html } virtual LibError Reset() @@ -238,10 +248,10 @@ public: return LibError_from_zlib(ret); } - virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed) + virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) { - const LibError ret = CodecZLibStream::Process(inflate, Z_SYNC_FLUSH, in, inSize, out, outSize, inConsumed, outConsumed); - m_checksum = UpdateChecksum(m_checksum, in, inSize); + const LibError ret = CodecZLibStream::Process(inflate, Z_SYNC_FLUSH, in, inSize, out, outSize, inConsumed, outProduced); + m_checksum = UpdateChecksum(m_checksum, out, outProduced); return ret; } diff --git a/source/lib/file/archive/stream.cpp b/source/lib/file/archive/stream.cpp index 992456fdda..fab7f64a1b 100644 --- a/source/lib/file/archive/stream.cpp +++ b/source/lib/file/archive/stream.cpp @@ -46,7 +46,7 @@ void OutputBufferManager::AllocateBuffer(size_t size) // than the address space of 32-bit systems). // no buffer or the previous one wasn't big enough: reallocate - if(!m_mem.get() || m_capacity < size) + if(!m_mem || m_capacity < size) { m_mem.reset((u8*)page_aligned_alloc(size), PageAlignedDeleter(size)); m_capacity = size; @@ -97,31 +97,30 @@ void Stream::SetOutputBuffer(u8* out, size_t outSize) } -LibError Stream::Feed(const u8* in, size_t inSize, size_t& bytesProcessed) +LibError Stream::Feed(const u8* in, size_t inSize) { if(m_outProduced == m_outputBufferManager.Size()) // output buffer full; must not call Process return INFO::OK; size_t inConsumed, outProduced; - u8* const out = m_outputBufferManager.Buffer(); + u8* const out = m_outputBufferManager.Buffer() + m_outProduced; const size_t outSize = m_outputBufferManager.Size() - m_outProduced; - RETURN_ERR(m_codec.get()->Process(in, inSize, out, outSize, inConsumed, outProduced)); + RETURN_ERR(m_codec->Process(in, inSize, out, outSize, inConsumed, outProduced)); m_inConsumed += inConsumed; m_outProduced += outProduced; - bytesProcessed = outProduced; return INFO::CB_CONTINUE; } LibError Stream::Finish() { - return m_codec.get()->Finish(m_checksum); + return m_codec->Finish(m_checksum); } -LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize, size_t& bytesProcessed) +LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize) { Stream& stream = *(Stream*)cbData; - return stream.Feed(in, inSize, bytesProcessed); + return stream.Feed(in, inSize); } diff --git a/source/lib/file/archive/stream.h b/source/lib/file/archive/stream.h index d3312fc9ba..5e98e8426e 100644 --- a/source/lib/file/archive/stream.h +++ b/source/lib/file/archive/stream.h @@ -51,7 +51,7 @@ private: u8* m_buffer; size_t m_size; - boost::shared_array m_mem; + shared_ptr m_mem; // size of m_mem. allows reusing previously allocated buffers // (user-specified buffers can't be reused because we have no control // over their lifetime) @@ -70,12 +70,8 @@ public: /** * 'feed' the codec with a data block. - * - * @param bytesProcessed receives the number of output bytes produced. - * it can legitimately be 0 - this happens if the input buffer is small - * and the codec hasn't produced any output. **/ - LibError Feed(const u8* in, size_t inSize, size_t& bytesProcessed); + LibError Feed(const u8* in, size_t inSize); LibError Finish(); @@ -98,6 +94,6 @@ private: u32 m_checksum; }; -extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize, size_t& bytesProcessed); +extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize); #endif // #ifndef INCLUDED_STREAM diff --git a/source/lib/file/common/file_loader.cpp b/source/lib/file/common/file_loader.cpp new file mode 100644 index 0000000000..867cd4e8d1 --- /dev/null +++ b/source/lib/file/common/file_loader.cpp @@ -0,0 +1,6 @@ +#include "precompiled.h" +#include "file_loader.h" + +/*virtual*/ IFileLoader::~IFileLoader() +{ +} diff --git a/source/lib/file/common/file_loader.h b/source/lib/file/common/file_loader.h new file mode 100644 index 0000000000..0276c0a4fb --- /dev/null +++ b/source/lib/file/common/file_loader.h @@ -0,0 +1,16 @@ +#ifndef INCLUDED_FILE_LOADER +#define INCLUDED_FILE_LOADER + +struct IFileLoader +{ + virtual ~IFileLoader(); + + virtual unsigned Precedence() const = 0; + virtual char LocationCode() const = 0; + + virtual LibError Load(const std::string& name, shared_ptr buf, size_t size) const = 0; +}; + +typedef shared_ptr PIFileLoader; + +#endif // #ifndef INCLUDED_FILE_LOADER diff --git a/source/lib/file/file_stats.cpp b/source/lib/file/common/file_stats.cpp similarity index 88% rename from source/lib/file/file_stats.cpp rename to source/lib/file/common/file_stats.cpp index 4f6b61b6dc..2758a0731c 100644 --- a/source/lib/file/file_stats.cpp +++ b/source/lib/file/common/file_stats.cpp @@ -16,9 +16,6 @@ #include "lib/timer.h" -typedef std::set AtomFnSet; -typedef std::pair PairIB; - // vfs static uint vfs_files; static size_t vfs_size_total; @@ -28,8 +25,6 @@ static double vfs_init_elapsed_time; static uint unique_names; static size_t unique_name_len_total; static uint open_files_cur, open_files_max; // total = opened_files.size() -static double opened_file_size_total; -static AtomFnSet opened_files; // file_buf static uint extant_bufs_cur, extant_bufs_max, extant_bufs_total; @@ -46,7 +41,6 @@ static uint io_seeks; // file_cache static uint cache_count[2]; static double cache_size_total[2]; -static AtomFnSet ever_cached_files; static uint conflict_misses; static double conflict_miss_size_total; static uint block_cache_count[2]; @@ -115,15 +109,12 @@ void stats_unique_name(size_t name_len) } -void stats_open(const char* atom_fn, size_t file_size) +void stats_open() { open_files_cur++; open_files_max = std::max(open_files_max, open_files_cur); - PairIB ret = opened_files.insert(atom_fn); - // hadn't been opened yet - if(ret.second) - opened_file_size_total += file_size; + // could also use a set to determine unique files that have been opened } void stats_close() @@ -171,12 +162,14 @@ void stats_io_user_request(size_t user_size) ScopedIoMonitor::ScopedIoMonitor() { + m_startTime = 0.0; timer_start(&m_startTime); } ScopedIoMonitor::~ScopedIoMonitor() { // note: we can only bill IOs that have succeeded :S + timer_reset(&m_startTime); } void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, char mode, size_t size) @@ -214,10 +207,11 @@ void stats_cb_finish() // file_cache // -void stats_cache(CacheRet cr, size_t size, const char* atom_fn) +void stats_cache(CacheRet cr, size_t size) { debug_assert(cr == CR_HIT || cr == CR_MISS); +#if 0 if(cr == CR_MISS) { PairIB ret = ever_cached_files.insert(atom_fn); @@ -227,6 +221,7 @@ void stats_cache(CacheRet cr, size_t size, const char* atom_fn) conflict_misses++; } } +#endif cache_count[cr]++; cache_size_total[cr] += size; @@ -284,10 +279,8 @@ void file_stats_dump() debug_printf( "\nfile:\n" "Total names: %u (%u KB)\n" - "Accessed files: %u (%g MB) -- %u%% of data set\n" "Max. concurrent: %u; leaked: %u.\n", unique_names, unique_name_len_total/1000, - opened_files.size(), opened_file_size_total/MB, percent(opened_files.size(), (size_t)vfs_files), open_files_max, open_files_cur ); diff --git a/source/lib/file/file_stats.h b/source/lib/file/common/file_stats.h similarity index 87% rename from source/lib/file/file_stats.h rename to source/lib/file/common/file_stats.h index b5531b3467..259253196b 100644 --- a/source/lib/file/file_stats.h +++ b/source/lib/file/common/file_stats.h @@ -18,7 +18,7 @@ enum FileIOImplentation { FI_LOWIO, FI_AIO, FI_BCACHE, FI_MAX_IDX }; enum FileOp { FO_READ, FO_WRITE }; enum CacheRet { CR_HIT, CR_MISS }; -#include "io/block_cache.h" // BlockId +#include "lib/file/io/block_cache.h" // BlockId #if FILE_STATS_ENABLED @@ -32,7 +32,7 @@ extern void stats_vfs_init_finish(); // currently not called because string_pool is now in lib/allocators extern void stats_unique_name(size_t name_len); -extern void stats_open(const char* atom_fn, size_t file_size); +extern void stats_open(); extern void stats_close(); // file_buf @@ -64,7 +64,7 @@ extern void stats_cb_start(); extern void stats_cb_finish(); // file_cache -extern void stats_cache(CacheRet cr, size_t size, const char* atom_fn); +extern void stats_cache(CacheRet cr, size_t size); extern void stats_block_cache(CacheRet cr); // archive builder @@ -79,7 +79,7 @@ extern void file_stats_dump(); #define stats_vfs_init_start() #define stats_vfs_init_finish() #define stats_unique_name(name_len) -#define stats_open(atom_fn, file_size) +#define stats_open() #define stats_close() #define stats_buf_alloc(size, alignedSize) #define stats_buf_free() @@ -95,7 +95,7 @@ public: #define stats_io_check_seek(blockId) #define stats_cb_start() #define stats_cb_finish() -#define stats_cache(cr, size, atom_fn) +#define stats_cache(cr, size) #define stats_block_cache(cr) #define stats_ab_connection(already_exists) #define file_stats_dump() diff --git a/source/lib/file/common/real_directory.cpp b/source/lib/file/common/real_directory.cpp new file mode 100644 index 0000000000..51eb35fac9 --- /dev/null +++ b/source/lib/file/common/real_directory.cpp @@ -0,0 +1,66 @@ +#include "precompiled.h" +#include "real_directory.h" + +#include "lib/path_util.h" +#include "lib/file/io/io.h" + + +RealDirectory::RealDirectory(const Path& path, unsigned priority, unsigned flags) + : m_path(path), m_priority(priority), m_flags(flags) +{ +} + + +/*virtual*/ unsigned RealDirectory::Precedence() const +{ + return 1u; +} + + +/*virtual*/ char RealDirectory::LocationCode() const +{ + return 'F'; +} + + +/*virtual*/ LibError RealDirectory::Load(const std::string& name, shared_ptr buf, size_t size) const +{ + File file; + RETURN_ERR(file.Open(m_path/name, 'r')); + + RETURN_ERR(io_ReadAligned(file, 0, buf.get(), size)); + return INFO::OK; +} + + +LibError RealDirectory::Store(const std::string& name, shared_ptr fileContents, size_t size) +{ + const Path pathname(m_path/name); + + { + File file; + RETURN_ERR(file.Open(pathname, 'w')); + RETURN_ERR(io_WriteAligned(file, 0, fileContents.get(), size)); + } + + // io_WriteAligned pads the file; we need to truncate it to the actual + // length. ftruncate can't be used because Windows' FILE_FLAG_NO_BUFFERING + // only allows resizing to sector boundaries, so the file must first + // be closed. + truncate(pathname.external_file_string().c_str(), (off_t)size); + + return INFO::OK; +} + + +void RealDirectory::Watch() +{ + //m_watch = CreateWatch(Path().external_file_string().c_str()); +} + + +PRealDirectory CreateRealSubdirectory(PRealDirectory realDirectory, const std::string& subdirectoryName) +{ + const Path path(realDirectory->GetPath()/subdirectoryName); + return PRealDirectory(new RealDirectory(path, realDirectory->Priority(), realDirectory->Flags())); +} diff --git a/source/lib/file/common/real_directory.h b/source/lib/file/common/real_directory.h new file mode 100644 index 0000000000..a6a813e698 --- /dev/null +++ b/source/lib/file/common/real_directory.h @@ -0,0 +1,58 @@ +#ifndef INCLUDED_REAL_DIRECTORY +#define INCLUDED_REAL_DIRECTORY + +#include "file_loader.h" +#include "lib/file/path.h" + +class RealDirectory : public IFileLoader +{ +public: + RealDirectory(const Path& path, unsigned priority, unsigned flags); + + const Path& GetPath() const + { + return m_path; + } + + unsigned Priority() const + { + return m_priority; + } + + unsigned Flags() const + { + return m_flags; + } + + // IFileLoader + virtual unsigned Precedence() const; + virtual char LocationCode() const; + virtual LibError Load(const std::string& name, shared_ptr buf, size_t size) const; + + LibError Store(const std::string& name, shared_ptr fileContents, size_t size); + + void Watch(); + +private: + RealDirectory(const RealDirectory& rhs); // noncopyable due to const members + RealDirectory& operator=(const RealDirectory& rhs); + + // note: paths are relative to the root directory, so storing the + // entire path instead of just the portion relative to the mount point + // is not all too wasteful. + const Path m_path; + + const unsigned m_priority; + + const unsigned m_flags; + + // note: watches are needed in each directory because some APIs + // (e.g. FAM) cannot watch entire trees with one call. + void* m_watch; +}; + +typedef shared_ptr PRealDirectory; + +extern PRealDirectory CreateRealSubdirectory(PRealDirectory realDirectory, const std::string& subdirectoryName); + +#endif // #ifndef INCLUDED_REAL_DIRECTORY diff --git a/source/lib/file/trace.cpp b/source/lib/file/common/trace.cpp similarity index 85% rename from source/lib/file/trace.cpp rename to source/lib/file/common/trace.cpp index fe4a1255b0..a965c4054f 100644 --- a/source/lib/file/trace.cpp +++ b/source/lib/file/common/trace.cpp @@ -12,7 +12,6 @@ #include "trace.h" #include "lib/allocators/pool.h" -#include "lib/path_util.h" // path_Pool #include "lib/timer.h" // get_time #include "lib/nommgr.h" // placement new @@ -27,7 +26,7 @@ TraceEntry::TraceEntry(EAction action, const char* pathname, size_t size) : m_timestamp(get_time()) , m_action(action) -, m_pathname(path_Pool()->UniqueCopy(pathname)) +, m_pathname(strdup(pathname)) , m_size(size) { } @@ -44,7 +43,13 @@ TraceEntry::TraceEntry(const char* text) debug_assert(fieldsRead == 4); debug_assert(action == 'L' || action == 'S'); m_action = (EAction)action; - m_pathname = path_Pool()->UniqueCopy(pathname); + m_pathname = strdup(pathname); +} + + +TraceEntry::~TraceEntry() +{ + SAFE_FREE(m_pathname); } @@ -73,12 +78,12 @@ public: { } - virtual LibError Load(const char* UNUSED(vfsPathname)) + virtual LibError Load(const char* UNUSED(pathname)) { return INFO::OK; } - virtual LibError Store(const char* UNUSED(vfsPathname)) const + virtual LibError Store(const char* UNUSED(pathname)) const { return INFO::OK; } @@ -107,6 +112,9 @@ public: virtual ~Trace() { + TraceEntry* entries = (TraceEntry*)m_pool.da.base; + for(TraceEntry* entry = entries; entry < entries+NumEntries(); entry++) + entry->~TraceEntry(); (void)pool_destroy(&m_pool); } diff --git a/source/lib/file/trace.h b/source/lib/file/common/trace.h similarity index 82% rename from source/lib/file/trace.h rename to source/lib/file/common/trace.h index 1fb9adad75..83b8d82471 100644 --- a/source/lib/file/trace.h +++ b/source/lib/file/common/trace.h @@ -31,6 +31,7 @@ public: TraceEntry(EAction action, const char* pathname, size_t size); TraceEntry(const char* textualRepresentation); + ~TraceEntry(); EAction Action() const { @@ -75,8 +76,8 @@ struct ITrace { virtual ~ITrace(); - virtual void NotifyLoad(const char* pathname, size_t size); - virtual void NotifyStore(const char* pathname, size_t size); + virtual void NotifyLoad(const char* pathname, size_t size) = 0; + virtual void NotifyStore(const char* pathname, size_t size) = 0; /** * store all entries into a file. @@ -87,7 +88,7 @@ struct ITrace * because storing filename strings in a binary format would be a * bit awkward. **/ - virtual LibError Store(const char* osPathname) const; + virtual LibError Store(const char* osPathname) const = 0; /** * load entries from file. @@ -96,13 +97,13 @@ struct ITrace * * replaces any existing entries. **/ - virtual LibError Load(const char* osPathname); + virtual LibError Load(const char* osPathname) = 0; - virtual const TraceEntry* Entries() const; - virtual size_t NumEntries() const; + virtual const TraceEntry* Entries() const = 0; + virtual size_t NumEntries() const = 0; }; -typedef boost::shared_ptr PITrace; +typedef shared_ptr PITrace; extern PITrace CreateDummyTrace(size_t maxSize); extern PITrace CreateTrace(size_t maxSize); diff --git a/source/lib/file/dir_util.cpp b/source/lib/file/dir_util.cpp deleted file mode 100644 index c2564e1ec1..0000000000 --- a/source/lib/file/dir_util.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/** - * ========================================================================= - * File : dir_util.cpp - * Project : 0 A.D. - * Description : helper functions for directory access - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "dir_util.h" - -#include -#include - -#include "lib/path_util.h" -#include "lib/regex.h" - - -bool dir_FileExists(IFilesystem* fs, const char* pathname) -{ - FileInfo fileInfo; - if(fs->GetFileInfo(pathname, fileInfo) < 0) - return false; - return true; -} - - -struct FileInfoNameLess : public std::binary_function -{ - bool operator()(const FileInfo& fileInfo1, const FileInfo& fileInfo2) const - { - return strcmp(fileInfo1.Name(), fileInfo2.Name()) < 0; - } -}; - -void dir_SortFiles(FileInfos& files) -{ - std::sort(files.begin(), files.end(), FileInfoNameLess()); -} - - -struct NameLess : public std::binary_function -{ - bool operator()(const char* name1, const char* name2) const - { - return strcmp(name1, name2) < 0; - } -}; - -void dir_SortDirectories(Directories& directories) -{ - std::sort(directories.begin(), directories.end(), NameLess()); -} - - -LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, DirCallback cb, uintptr_t cbData, const char* pattern, uint flags) -{ - debug_assert((flags & ~(VFS_DIR_RECURSIVE)) == 0); - - // (declare here to avoid reallocations) - FileInfos files; Directories subdirectories; - - // (a FIFO queue is more efficient than recursion because it uses less - // stack space and avoids seeks due to breadth-first traversal.) - std::queue dir_queue; - dir_queue.push(path_Pool()->UniqueCopy(dirPath)); - while(!dir_queue.empty()) - { - // get directory path - PathPackage pp; - (void)path_package_set_dir(&pp, dir_queue.front()); - dir_queue.pop(); - - RETURN_ERR(fs->GetDirectoryEntries(pp.path, &files, &subdirectories)); - - for(size_t i = 0; i < files.size(); i++) - { - const FileInfo fileInfo = files[i]; - if(!match_wildcard(fileInfo.Name(), pattern)) - continue; - - // build complete path (FileInfo only stores name) - (void)path_package_append_file(&pp, fileInfo.Name()); - cb(pp.path, fileInfo, cbData); - } - - if((flags & VFS_DIR_RECURSIVE) == 0) - break; - - for(size_t i = 0; i < subdirectories.size(); i++) - { - (void)path_package_append_file(&pp, subdirectories[i]); - dir_queue.push(path_Pool()->UniqueCopy(pp.path)); - } - } - - return INFO::OK; -} - - -void dir_NextNumberedFilename(IFilesystem* fs, const char* pathnameFmt, unsigned& nextNumber, char* nextPathname) -{ - // (first call only:) scan directory and set nextNumber according to - // highest matching filename found. this avoids filling "holes" in - // the number series due to deleted files, which could be confusing. - // example: add 1st and 2nd; [exit] delete 1st; [restart] - // add 3rd -> without this measure it would get number 1, not 3. - if(nextNumber == 0) - { - const char* path; const char* nameFmt; - path_split(pathnameFmt, &path, &nameFmt); - - unsigned maxNumber = 0; - FileInfos files; - fs->GetDirectoryEntries(path, &files, 0); - for(size_t i = 0; i < files.size(); i++) - { - unsigned number; - if(sscanf(files[i].Name(), nameFmt, &number) == 1) - maxNumber = std::max(number, maxNumber); - } - - nextNumber = maxNumber+1; - } - - // now increment number until that file doesn't yet exist. - // this is fairly slow, but typically only happens once due - // to scan loop above. (we still need to provide for looping since - // someone may have added files in the meantime) - // we don't bother with binary search - this isn't a bottleneck. - do - snprintf(nextPathname, PATH_MAX, pathnameFmt, nextNumber++); - while(dir_FileExists(fs, nextPathname)); -} diff --git a/source/lib/res/res.cpp b/source/lib/file/file_system.cpp similarity index 55% rename from source/lib/res/res.cpp rename to source/lib/file/file_system.cpp index d153ceec9d..c6ef706add 100644 --- a/source/lib/res/res.cpp +++ b/source/lib/file/file_system.cpp @@ -1,6 +1,6 @@ /** * ========================================================================= - * File : res.cpp + * File : file_system.cpp * Project : 0 A.D. * Description : * ========================================================================= @@ -9,8 +9,4 @@ // license: GPL; see lib/license.txt #include "precompiled.h" -#include "res.h" - - -ERROR_ASSOCIATE(ERR::RES_UNKNOWN_FORMAT, "Unknown file format", -1); -ERROR_ASSOCIATE(ERR::RES_INCOMPLETE_HEADER, "File header not completely read", -1); +#include "file_system.h" diff --git a/source/lib/file/file_system.h b/source/lib/file/file_system.h new file mode 100644 index 0000000000..2442abd4e4 --- /dev/null +++ b/source/lib/file/file_system.h @@ -0,0 +1,50 @@ +/** + * ========================================================================= + * File : file_system.h + * Project : 0 A.D. + * Description : + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#ifndef INCLUDED_FILE_SYSTEM +#define INCLUDED_FILE_SYSTEM + +class FileInfo +{ +public: + FileInfo() + { + } + + FileInfo(const std::string& name, off_t size, time_t mtime) + : m_name(name), m_size(size), m_mtime(mtime) + { + } + + const std::string& Name() const + { + return m_name; + } + + off_t Size() const + { + return m_size; + } + + time_t MTime() const + { + return m_mtime; + } + +private: + std::string m_name; + off_t m_size; + time_t m_mtime; +}; + +typedef std::vector FileInfos; +typedef std::vector DirectoryNames; + +#endif // #ifndef INCLUDED_FILE_SYSTEM diff --git a/source/lib/file/file_system_posix.cpp b/source/lib/file/file_system_posix.cpp new file mode 100644 index 0000000000..236f36c8d5 --- /dev/null +++ b/source/lib/file/file_system_posix.cpp @@ -0,0 +1,165 @@ +/** + * ========================================================================= + * File : directory_posix.cpp + * Project : 0 A.D. + * Description : file layer on top of POSIX. avoids the need for + * : absolute paths and provides fast I/O. + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#include "precompiled.h" +#include "file_system_posix.h" + +#include +#include +#include + +#include "lib/path_util.h" +#include "lib/file/path.h" +#include "lib/posix/posix_filesystem.h" + + +struct DirDeleter +{ + void operator()(DIR* osDir) const + { + const int ret = closedir(osDir); + debug_assert(ret == 0); + } +}; + +// is name "." or ".."? +static bool IsDummyDirectory(const char* name) +{ + if(name[0] != '.') + return false; + return (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); +} + +/*virtual*/ LibError FileSystem_Posix::GetDirectoryEntries(const Path& path, FileInfos* files, DirectoryNames* subdirectoryNames) const +{ + // open directory + errno = 0; + DIR* pDir = opendir(path.external_file_string().c_str()); + if(!pDir) + return LibError_from_errno(); + shared_ptr osDir(pDir, DirDeleter()); + + // (we need absolute paths below; using this instead of path_append avoids + // a strlen call for each entry.) + PathPackage pp; + (void)path_package_set_dir(&pp, path.external_directory_string().c_str()); + + for(;;) + { + errno = 0; + struct dirent* osEnt = readdir(osDir.get()); + if(!osEnt) + { + // no error, just no more entries to return + if(!errno) + return INFO::OK; + return LibError_from_errno(); + } + + const char* name = osEnt->d_name; + RETURN_ERR(path_component_validate(name)); + + // get file information (mode, size, mtime) + struct stat s; +#if OS_WIN + // .. wposix readdir has enough information to return dirent + // status directly (much faster than calling stat). + RETURN_ERR(readdir_stat_np(osDir.get(), &s)); +#else + // .. call regular stat(). + errno = 0; + path_package_append_file(&pp, name); + if(stat(pp->path, &s) != 0) + return LibError_from_errno(); +#endif + + if(files && S_ISREG(s.st_mode)) + files->push_back(FileInfo(name, s.st_size, s.st_mtime)); + else if(subdirectoryNames && S_ISDIR(s.st_mode) && !IsDummyDirectory(name)) + subdirectoryNames->push_back(name); + } +} + + +LibError FileSystem_Posix::GetFileInfo(const Path& pathname, FileInfo* pfileInfo) const +{ + char osPathname[PATH_MAX]; + path_copy(osPathname, pathname.external_directory_string().c_str()); + + // if path ends in slash, remove it (required by stat) + char* last_char = osPathname+strlen(osPathname)-1; + if(path_is_dir_sep(*last_char)) + *last_char = '\0'; + + errno = 0; + struct stat s; + memset(&s, 0, sizeof(s)); + if(stat(osPathname, &s) != 0) + return LibError_from_errno(); + + const char* name = path_name_only(osPathname); + *pfileInfo = FileInfo(name, s.st_size, s.st_mtime); + return INFO::OK; +} + + +LibError FileSystem_Posix::DeleteFile(const Path& pathname) +{ + errno = 0; + if(unlink(pathname.external_file_string().c_str()) != 0) + return LibError_from_errno(); + + return INFO::OK; +} + + +LibError FileSystem_Posix::CreateDirectory(const Path& path) +{ + errno = 0; + if(mkdir(path.external_directory_string().c_str(), S_IRWXO|S_IRWXU|S_IRWXG) != 0) + return LibError_from_errno(); + + return INFO::OK; +} + + +LibError FileSystem_Posix::DeleteDirectory(const Path& path) +{ + // note: we have to recursively empty the directory before it can + // be deleted (required by Windows and POSIX rmdir()). + + const char* osPath = path.external_directory_string().c_str(); + + PathPackage pp; + RETURN_ERR(path_package_set_dir(&pp, osPath)); + + FileInfos files; DirectoryNames subdirectoryNames; + RETURN_ERR(GetDirectoryEntries(path, &files, &subdirectoryNames)); + + // delete files + for(size_t i = 0; i < files.size(); i++) + { + RETURN_ERR(path_package_append_file(&pp, files[i].Name().c_str())); + errno = 0; + if(unlink(pp.path) != 0) + return LibError_from_errno(); + } + + // recurse over subdirectoryNames + for(size_t i = 0; i < subdirectoryNames.size(); i++) + RETURN_ERR(DeleteDirectory(path/subdirectoryNames[i])); + + errno = 0; + if(rmdir(osPath) != 0) + return LibError_from_errno(); + + return INFO::OK; +} diff --git a/source/lib/file/file_system_posix.h b/source/lib/file/file_system_posix.h new file mode 100644 index 0000000000..635a4ef731 --- /dev/null +++ b/source/lib/file/file_system_posix.h @@ -0,0 +1,28 @@ +/** + * ========================================================================= + * File : file_system_posix.h + * Project : 0 A.D. + * Description : file layer on top of POSIX. avoids the need for + * : absolute paths and provides fast I/O. + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#ifndef INCLUDED_FILE_SYSTEM_POSIX +#define INCLUDED_FILE_SYSTEM_POSIX + +#include "lib/file/path.h" +#include "lib/file/file_system.h" + +struct FileSystem_Posix +{ + virtual LibError GetFileInfo(const Path& pathname, FileInfo* fileInfo) const; + virtual LibError GetDirectoryEntries(const Path& path, FileInfos* files, DirectoryNames* subdirectoryNames) const; + + LibError DeleteFile(const Path& pathname); + LibError CreateDirectory(const Path& dirPath); + LibError DeleteDirectory(const Path& dirPath); +}; + +#endif // #ifndef INCLUDED_FILE_SYSTEM_POSIX diff --git a/source/lib/file/file_system_util.cpp b/source/lib/file/file_system_util.cpp new file mode 100644 index 0000000000..562b67ad50 --- /dev/null +++ b/source/lib/file/file_system_util.cpp @@ -0,0 +1,138 @@ +/** + * ========================================================================= + * File : file_system_util.cpp + * Project : 0 A.D. + * Description : helper functions for directory access + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#include "precompiled.h" +#include "file_system_util.h" + +#include +#include + +#include "lib/path_util.h" +#include "lib/regex.h" + +LibError fs_GetPathnames(PIVFS fs, const VfsPath& path, const char* filter, VfsPaths& pathnames) +{ + std::vector files; + RETURN_ERR(fs->GetDirectoryEntries(path, &files, 0)); + + pathnames.clear(); + pathnames.reserve(files.size()); + + for(size_t i = 0; i < files.size(); i++) + { + if(match_wildcard(files[i].Name().c_str(), filter)) + pathnames.push_back(path/files[i].Name()); + } + + return INFO::OK; +} + + +struct FileInfoNameLess : public std::binary_function +{ + bool operator()(const FileInfo& fileInfo1, const FileInfo& fileInfo2) const + { + return strcasecmp(fileInfo1.Name().c_str(), fileInfo2.Name().c_str()) < 0; + } +}; + +void fs_SortFiles(FileInfos& files) +{ + std::sort(files.begin(), files.end(), FileInfoNameLess()); +} + + +struct NameLess : public std::binary_function +{ + bool operator()(const std::string& name1, const std::string& name2) const + { + return strcasecmp(name1.c_str(), name2.c_str()) < 0; + } +}; + +void fs_SortDirectories(DirectoryNames& directories) +{ + std::sort(directories.begin(), directories.end(), NameLess()); +} + + +LibError fs_ForEachFile(PIVFS fs, const VfsPath& path, FileCallback cb, uintptr_t cbData, const char* pattern, unsigned flags) +{ + debug_assert(IsDirectory(path)); + + // (declare here to avoid reallocations) + FileInfos files; DirectoryNames subdirectoryNames; + + // (a FIFO queue is more efficient than recursion because it uses less + // stack space and avoids seeks due to breadth-first traversal.) + std::queue pendingDirectories; + pendingDirectories.push(path); + while(!pendingDirectories.empty()) + { + const VfsPath& path = pendingDirectories.front(); + + RETURN_ERR(fs->GetDirectoryEntries(path/"/", &files, &subdirectoryNames)); + + for(size_t i = 0; i < files.size(); i++) + { + const FileInfo fileInfo = files[i]; + if(!match_wildcard(fileInfo.Name().c_str(), pattern)) + continue; + + const VfsPath pathname(path/fileInfo.Name()); // (FileInfo only stores the name) + cb(pathname, fileInfo, cbData); + } + + if(!(flags & DIR_RECURSIVE)) + break; + + for(size_t i = 0; i < subdirectoryNames.size(); i++) + pendingDirectories.push(path/subdirectoryNames[i]); + pendingDirectories.pop(); + } + + return INFO::OK; +} + + +void fs_NextNumberedFilename(PIVFS fs, const char* pathnameFmt, unsigned& nextNumber, char* nextPathname) +{ + // (first call only:) scan directory and set nextNumber according to + // highest matching filename found. this avoids filling "holes" in + // the number series due to deleted files, which could be confusing. + // example: add 1st and 2nd; [exit] delete 1st; [restart] + // add 3rd -> without this measure it would get number 1, not 3. + if(nextNumber == 0) + { + char path[PATH_MAX]; char nameFmt[PATH_MAX]; + path_split(pathnameFmt, path, nameFmt); + + unsigned maxNumber = 0; + FileInfos files; + fs->GetDirectoryEntries(path, &files, 0); + for(size_t i = 0; i < files.size(); i++) + { + unsigned number; + if(sscanf(files[i].Name().c_str(), nameFmt, &number) == 1) + maxNumber = std::max(number, maxNumber); + } + + nextNumber = maxNumber+1; + } + + // now increment number until that file doesn't yet exist. + // this is fairly slow, but typically only happens once due + // to scan loop above. (we still need to provide for looping since + // someone may have added files in the meantime) + // we don't bother with binary search - this isn't a bottleneck. + do + snprintf(nextPathname, PATH_MAX, pathnameFmt, nextNumber++); + while(fs->GetFileInfo(nextPathname, 0) == INFO::OK); +} diff --git a/source/lib/file/dir_util.h b/source/lib/file/file_system_util.h similarity index 62% rename from source/lib/file/dir_util.h rename to source/lib/file/file_system_util.h index 083fdd090f..ea3f148ed0 100644 --- a/source/lib/file/dir_util.h +++ b/source/lib/file/file_system_util.h @@ -1,6 +1,6 @@ /** * ========================================================================= - * File : dir_util.h + * File : file_system_util.h * Project : 0 A.D. * Description : helper functions for directory access * ========================================================================= @@ -8,15 +8,16 @@ // license: GPL; see lib/license.txt -#ifndef INCLUDED_DIR_UTIL -#define INCLUDED_DIR_UTIL +#ifndef INCLUDED_FILE_SYSTEM_UTIL +#define INCLUDED_FILE_SYSTEM_UTIL -#include "filesystem.h" +#include "lib/file/vfs/vfs.h" -extern bool dir_FileExists(IFilesystem* fs, const char* pathname); +extern void fs_SortFiles(FileInfos& files); +extern void fs_SortDirectories(DirectoryNames& directories); + +extern LibError fs_GetPathnames(PIVFS fs, const VfsPath& path, const char* filter, VfsPaths& pathnames); -extern void dir_SortFiles(FileInfos& files); -extern void dir_SortDirectories(Directories& directories); /** * called for files in a directory. @@ -30,16 +31,15 @@ extern void dir_SortDirectories(Directories& directories); * CAVEAT: pathname and fileInfo are only valid until the function * returns! **/ -typedef LibError (*DirCallback)(const char* pathname, const FileInfo& fileInfo, const uintptr_t cbData); +typedef LibError (*FileCallback)(const VfsPath& pathname, const FileInfo& fileInfo, const uintptr_t cbData); enum DirFlags { - /// include files in subdirectories. - VFS_DIR_RECURSIVE = 1 + DIR_RECURSIVE = 1 }; /** - * call back for each file in a directory (tree) + * call back for each file in a directory tree * * @param cb see DirCallback * @param pattern that file names must match. '*' and '&' wildcards @@ -47,7 +47,7 @@ enum DirFlags * @param flags see DirFlags * @param LibError **/ -extern LibError dir_ForEachFile(IFilesystem* fs, const char* path, DirCallback cb, uintptr_t cbData, const char* pattern = 0, uint flags = 0); +extern LibError fs_ForEachFile(PIVFS fs, const VfsPath& path, FileCallback cb, uintptr_t cbData, const char* pattern = 0, uint flags = 0); /** @@ -62,6 +62,6 @@ extern LibError dir_ForEachFile(IFilesystem* fs, const char* path, DirCallback c * if 0, numbers corresponding to existing files are skipped. * @param receives the output; must hold at least PATH_MAX characters. **/ -extern void dir_NextNumberedFilename(IFilesystem* fs, const char* pathnameFmt, unsigned& nextNumber, char* nextPathname); +extern void fs_NextNumberedFilename(PIVFS fs, const char* pathnameFmt, unsigned& nextNumber, char* nextPathname); -#endif // #ifndef INCLUDED_DIR_UTIL +#endif // #ifndef INCLUDED_FILE_SYSTEM_UTIL diff --git a/source/lib/file/filesystem.cpp b/source/lib/file/filesystem.cpp deleted file mode 100644 index 9a096d1dab..0000000000 --- a/source/lib/file/filesystem.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "precompiled.h" -#include "filesystem.h" - -IFilesystem::~IFilesystem() -{ -} - -IFileProvider::~IFileProvider() -{ -} diff --git a/source/lib/file/filesystem.h b/source/lib/file/filesystem.h deleted file mode 100644 index 1e4c105e93..0000000000 --- a/source/lib/file/filesystem.h +++ /dev/null @@ -1,91 +0,0 @@ -/** - * ========================================================================= - * File : filesystem.h - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_FILESYSTEM -#define INCLUDED_FILESYSTEM - - -/** - * information describing file system entries (i.e. files or directories) - * - * note: don't be extravagant with memory - dir_ForEachSortedEntry allocates - * one instance of this per directory entry. - **/ -class FileInfo -{ -public: - FileInfo() - { - } - - FileInfo(const char* name, off_t size, time_t mtime) - : m_name(name), m_size(size), m_mtime(mtime) - { - } - - const char* Name() const - { - return m_name; - } - - off_t Size() const - { - return m_size; - } - - time_t MTime() const - { - return m_mtime; - } - -private: - /** - * name of the entry; does not include a path. - * the underlying storage is guaranteed to remain valid and must not - * be freed/modified. - **/ - const char* m_name; - off_t m_size; - time_t m_mtime; -}; - -typedef std::vector FileInfos; - -typedef std::vector Directories; - -// -struct IFileProvider -{ - virtual ~IFileProvider(); - - virtual unsigned Precedence() const = 0; - virtual char LocationCode() const = 0; - - virtual LibError Load(const char* name, const void* location, u8* fileContents, size_t size) const = 0; - virtual LibError Store(const char* name, const void* location, const u8* fileContents, size_t size) const = 0; -}; - -typedef boost::shared_ptr PIFileProvider; - - -struct IFilesystem -{ - virtual ~IFilesystem(); - - virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const = 0; - - // note: this interface avoids having to lock a directory while an - // iterator is extant. - // (don't split this into 2 functions because POSIX can't implement - // that efficiently) - virtual LibError GetDirectoryEntries(const char* path, FileInfos* files, Directories* subdirectories) const = 0; -}; - -#endif // #ifndef INCLUDED_FILESYSTEM diff --git a/source/lib/file/io/block_cache.cpp b/source/lib/file/io/block_cache.cpp index ef96f5fe0c..4f7cda759b 100644 --- a/source/lib/file/io/block_cache.cpp +++ b/source/lib/file/io/block_cache.cpp @@ -11,28 +11,33 @@ #include "precompiled.h" #include "block_cache.h" -#include "lib/file/file_stats.h" +#include "lib/file/common/file_stats.h" #include "lib/lockfree.h" #include "lib/allocators/pool.h" +#include "lib/fnv_hash.h" +#include "io_internal.h" //----------------------------------------------------------------------------- BlockId::BlockId() - : m_atom_fn(0), m_blockIndex(~0u) + : m_id(0) { } -BlockId::BlockId(const char* atom_fn, off_t ofs) +BlockId::BlockId(const Path& pathname, off_t ofs) { - debug_assert(ofs <= (u64)BLOCK_SIZE * 0xFFFFFFFF); // ensure value fits in m_blockIndex - m_atom_fn = atom_fn; // unique (by definition) - m_blockIndex = (u32)(ofs / BLOCK_SIZE); + m_id = fnv_hash64(pathname.string().c_str(), pathname.string().length()); + const size_t indexBits = 16; + m_id <<= indexBits; + const off_t blockIndex = ofs / BLOCK_SIZE; + debug_assert(blockIndex < (1ul << indexBits)); + m_id |= blockIndex; } bool BlockId::operator==(const BlockId& rhs) const { - return m_atom_fn == rhs.m_atom_fn && m_blockIndex == rhs.m_blockIndex; + return m_id == rhs.m_id; } bool BlockId::operator!=(const BlockId& rhs) const @@ -45,76 +50,18 @@ bool BlockId::operator!=(const BlockId& rhs) const struct Block { - Block() + Block(BlockId id, shared_ptr buf) { - buf = io_buf_Allocate(BLOCK_SIZE); + this->id = id; + this->buf = buf; } - IoBuf buf; - LF_ReferenceCounter refs; -}; + // block is "valid" and can satisfy Retrieve() requests if a + // (non-default-constructed) ID has been assigned. + BlockId id; -class BlockManager -{ -public: - BlockManager(size_t numBlocks) - : m_ids(numBlocks), m_blocks(numBlocks), m_oldestIndex(0) - { - } - - // (linear search is ok since we only expect to manage a few blocks) - Block* Find(const BlockId& id) - { - for(size_t i = 0; i < m_ids.size(); i++) - { - if(m_ids[i] == id) - return &m_blocks[i]; - } - - return 0; - } - - Block* AcquireOldestAvailableBlock(BlockId id) - { - for(size_t i = 0; i < m_ids.size(); i++) - { - // (m_blocks are evicted in FIFO order.) - Block& block = m_blocks[m_oldestIndex % m_blocks.size()]; - cpu_AtomicAdd(&m_oldestIndex, +1); - - if(block.refs.AcquireExclusiveAccess()) - { - m_ids[i] = id; - return █ - } - - // the oldest item is currently locked, so keep looking. - // - // to see when this can happen, consider IO depth = 4. let the - // Block at m_blocks[oldest_block] contain data that an IO wants. - // the 2nd and 3rd m_blocks are not in cache and happen to be taken - // from near the end of m_blocks[]. attempting to issue block #4 - // fails because its buffer would want the first slot - // (which is locked since its IO is still pending). - } - - DEBUG_WARN_ERR(ERR::LIMIT); // all m_blocks are locked - return 0; - } - - void InvalidateAll() - { - // note: don't check whether any references are held etc. because - // this should only be called at the end of the (test) program. - - for(size_t i = 0; i < m_blocks.size(); i++) - m_ids[i] = BlockId(); - } - -private: - std::vector m_ids; - std::vector m_blocks; - volatile intptr_t m_oldestIndex; + // this block is "in use" if use_count != 1. + shared_ptr buf; }; @@ -123,97 +70,75 @@ private: class BlockCache::Impl { public: - Impl(size_t cacheSize) - : m_blockManager(cacheSize / BLOCK_SIZE) + Impl(size_t numBlocks) + : m_maxBlocks(numBlocks) { } - IoBuf Reserve(BlockId id) + void Add(BlockId id, shared_ptr buf) { - debug_assert(!m_blockManager.Find(id)); // must not already be extant - - Block* block = m_blockManager.AcquireOldestAvailableBlock(id); + if(m_blocks.size() > m_maxBlocks) + { +#if CONFIG_READ_ONLY_CACHE + mprotect((void*)m_blocks.front().buf.get(), BLOCK_SIZE, PROT_READ); +#endif + m_blocks.pop_front(); // evict oldest block + } #if CONFIG_READ_ONLY_CACHE - mprotect((void*)block->buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ); + mprotect((void*)buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ); #endif - - return block->buf; + m_blocks.push_back(Block(id, buf)); } - void MarkComplete(BlockId id) + bool Retrieve(BlockId id, shared_ptr& buf) { - Block* block = m_blockManager.Find(id); - debug_assert(block); // ( cannot have been evicted because it is locked) + // (linear search is ok since we only expect to manage a few blocks) + for(size_t i = 0; i < m_blocks.size(); i++) + { + Block& block = m_blocks[i]; + if(block.id == id) + { + buf = block.buf; + return true; + } + } - block->refs.RelinquishExclusiveAccess(); - -#if CONFIG_READ_ONLY_CACHE - mprotect((void*)block->buf.get(), BLOCK_SIZE, PROT_READ); -#endif - } - - bool Retrieve(BlockId id, IoBuf& buf) - { - Block* block = m_blockManager.Find(id); - if(!block) // not found - return false; - - if(!block->refs.AddReference()) // contents are not yet valid (can happen due to multithreaded IOs) - return false; - - buf = block->buf; - return true; - } - - void Release(BlockId id) - { - Block* block = m_blockManager.Find(id); - // ( ought not yet have been evicted because it is still referenced; - // if not found, Release was called too often) - debug_assert(block); - - block->refs.Release(); + return false; } void InvalidateAll() { - m_blockManager.InvalidateAll(); + // note: don't check whether any references are held etc. because + // this should only be called at the end of the (test) program. + m_blocks.clear(); } private: - BlockManager m_blockManager; + size_t m_maxBlocks; + typedef std::deque Blocks; + Blocks m_blocks; }; //----------------------------------------------------------------------------- -BlockCache::BlockCache(size_t cacheSize) - : impl(new Impl(cacheSize)) +BlockCache::BlockCache(size_t numBlocks) + : impl(new Impl(numBlocks)) { } -IoBuf BlockCache::Reserve(BlockId id) +void BlockCache::Add(BlockId id, shared_ptr buf) { - return impl.get()->Reserve(id); + impl->Add(id, buf); } -void BlockCache::MarkComplete(BlockId id) +bool BlockCache::Retrieve(BlockId id, shared_ptr& buf) { - impl.get()->Reserve(id); -} - -bool BlockCache::Retrieve(BlockId id, IoBuf& buf) -{ - return impl.get()->Retrieve(id, buf); -} - -void BlockCache::Release(BlockId id) -{ - impl.get()->Release(id); + return impl->Retrieve(id, buf); } void BlockCache::InvalidateAll() { - return impl.get()->InvalidateAll(); + return impl->InvalidateAll(); } diff --git a/source/lib/file/io/block_cache.h b/source/lib/file/io/block_cache.h index 8e021f9ed5..fb084d34fd 100644 --- a/source/lib/file/io/block_cache.h +++ b/source/lib/file/io/block_cache.h @@ -11,18 +11,7 @@ #ifndef INCLUDED_BLOCK_CACHE #define INCLUDED_BLOCK_CACHE -#include "io_buf.h" - -/** - * block := power-of-two sized chunk of a file. - * all transfers are expanded to naturally aligned, whole blocks - * (this makes caching parts of files feasible; it is also much faster - * for some aio implementations, e.g. wposix). - * - * measurements show this value to yield best read throughput. - **/ -static const size_t BLOCK_SIZE = 32*KiB; - +#include "lib/file/path.h" /** * ID that uniquely identifies a block within a file @@ -31,13 +20,12 @@ class BlockId { public: BlockId(); - BlockId(const char* atom_fn, off_t ofs); + BlockId(const Path& pathname, off_t ofs); bool operator==(const BlockId& rhs) const; bool operator!=(const BlockId& rhs) const; private: - const char* m_atom_fn; - u32 m_blockIndex; + u64 m_id; }; @@ -46,67 +34,47 @@ private: * absorbs the overhead of rounding up archive IOs to the nearest block * boundaries by keeping the last few blocks in memory. * - * the interface is quite similar to FileCache; see the note there. + * the interface is somewhat similar to FileCache; see the note there. + * + * not thread-safe (each thread is intended to have its own cache). **/ class BlockCache { public: /** - * @param cacheSize total size [bytes] that the cache is to manage. - * the default value is enough to support temp buffers and - * absorb the cost of unaligned reads from a few archives. + * @param numBlocks (the default value is enough to support temp buffers + * and absorb the cost of unaligned reads from archives.) **/ - BlockCache(size_t cacheSize = 16 * BLOCK_SIZE); + BlockCache(size_t numBlocks = 16); /** - * Reserve a block for use as an IO buffer. + * Add a block to the cache. * - * @return suitably aligned memory; never fails. + * @param id key that will be used to Retrieve the block. * - * no further operations with the same id are allowed to succeed - * until MarkComplete has been called. + * call this when the block's IO has completed; its data will + * satisfy subsequent Retrieve calls for the same id. + * if CONFIG_READ_ONLY_CACHE, the memory is made read-only. **/ - IoBuf Reserve(BlockId id); + void Add(BlockId id, shared_ptr buf); /** - * Indicate that IO into the block has completed. + * Attempt to retrieve a block's contents. * - * this allows the cache to satisfy subsequent Retrieve() calls by - * returning this block; if CONFIG_READ_ONLY_CACHE, the block is - * made read-only. if need be and no references are currently attached - * to it, the memory can also be commandeered by Reserve(). + * @return whether the block is in cache. + * + * if successful, a shared pointer to the contents is returned. + * they remain valid until all references are removed and the block + * is evicted. **/ - void MarkComplete(BlockId id); - - /** - * Attempt to retrieve a block the file cache. - * - * @return false if not in cache or its IO is still pending, - * otherwise true. - * - * if successful, a reference is added to the block and its - * buffer is returned. - **/ - bool Retrieve(BlockId id, IoBuf& buf); - - /** - * Indicate the block contents are no longer needed. - * - * this decreases the reference count; the memory can only be reused - * if it reaches 0. the block remains in cache until evicted by a - * subsequent Reserve() call. - * - * note: fails (raises a warning) if called for a buffer that is - * currently between Reserve and MarkComplete operations. - **/ - void Release(BlockId id); + bool Retrieve(BlockId id, shared_ptr& buf); /** * Invalidate the contents of the cache. * * this effectively discards the contents of existing blocks - * (more specifically: prevents them from satisfying Retrieve() calls - * until a subsequent Reserve/MarkComplete of that block). + * (more specifically: prevents them from satisfying Retrieve calls + * until a subsequent Add with the same id). * * useful for self-tests: multiple independent IO tests run in the same * process and must not influence each other via the cache. @@ -115,7 +83,7 @@ public: private: class Impl; - boost::shared_ptr impl; + shared_ptr impl; }; #endif // #ifndef INCLUDED_BLOCK_CACHE diff --git a/source/lib/file/io/io.cpp b/source/lib/file/io/io.cpp new file mode 100644 index 0000000000..c414cef6a2 --- /dev/null +++ b/source/lib/file/io/io.cpp @@ -0,0 +1,530 @@ +/** + * ========================================================================= + * File : io.cpp + * Project : 0 A.D. + * Description : + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#include "precompiled.h" +#include "io.h" + +#include "lib/allocators/allocators.h" // AllocatorChecker +#include "lib/bits.h" // IsAligned, round_up +#include "lib/sysdep/cpu.h" // cpu_memcpy +#include "lib/file/common/file_stats.h" +#include "lib/file/path.h" +#include "block_cache.h" +#include "io_internal.h" + + +ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES); +ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO); +ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1); + + +// the underlying aio implementation likes buffer and offset to be +// sector-aligned; if not, the transfer goes through an align buffer, +// and requires an extra cpu_memcpy. +// +// if the user specifies an unaligned buffer, there's not much we can +// do - we can't assume the buffer contains padding. therefore, +// callers should let us allocate the buffer if possible. +// +// if ofs misalign = buffer, only the first and last blocks will need +// to be copied by aio, since we read up to the next block boundary. +// otherwise, everything will have to be copied; at least we split +// the read into blocks, so aio's buffer won't have to cover the +// whole file. + +// we don't do any caching or alignment here - this is just a thin +// AIO wrapper. rationale: +// - aligning the transfer isn't possible here since we have no control +// over the buffer, i.e. we cannot read more data than requested. +// instead, this is done in io_manager. +// - transfer sizes here are arbitrary (i.e. not block-aligned); +// that means the cache would have to handle this or also split them up +// into blocks, which would duplicate the abovementioned work. +// - if caching here, we'd also have to handle "forwarding" (i.e. +// desired block has been issued but isn't yet complete). again, it +// is easier to let the synchronous io_manager handle this. +// - finally, io_manager knows more about whether the block should be cached +// (e.g. whether another block request will follow), but we don't +// currently make use of this. +// +// disadvantages: +// - streamed data will always be read from disk. that's not a problem, +// because such data (e.g. music, long speech) is unlikely to be used +// again soon. +// - prefetching (issuing the next few blocks from archive/file during +// idle time to satisfy potential future IOs) requires extra buffers; +// this is a bit more complicated than just using the cache as storage. + + +//----------------------------------------------------------------------------- +// allocator +//----------------------------------------------------------------------------- + +#ifndef NDEBUG +static AllocatorChecker allocatorChecker; +#endif + +class IoDeleter +{ +public: + IoDeleter(size_t paddedSize) + : m_paddedSize(paddedSize) + { + } + + void operator()(u8* mem) + { + debug_assert(m_paddedSize != 0); +#ifndef NDEBUG + allocatorChecker.notify_free(mem, m_paddedSize); +#endif + page_aligned_free(mem, m_paddedSize); + m_paddedSize = 0; + } + +private: + size_t m_paddedSize; +}; + + +shared_ptr io_Allocate(size_t size, off_t ofs) +{ + debug_assert(size != 0); + + const size_t misalignment = ofs % BLOCK_SIZE; + const size_t paddedSize = (size_t)round_up(size+misalignment, BLOCK_SIZE); + + u8* mem = (u8*)page_aligned_alloc(paddedSize); + if(!mem) + throw std::bad_alloc(); + +#ifndef NDEBUG + allocatorChecker.notify_alloc(mem, paddedSize); +#endif + + return shared_ptr(mem, IoDeleter(paddedSize)); +} + + +//----------------------------------------------------------------------------- +// File +//----------------------------------------------------------------------------- + +File::File() +{ +} + + +File::~File() +{ + Close(); +} + + +LibError File::Open(const Path& pathname, char mode) +{ + debug_assert(mode == 'w' || mode == 'r'); + + m_pathname = pathname; + m_mode = mode; + + int oflag = (mode == 'r')? O_RDONLY : O_WRONLY|O_CREAT|O_TRUNC; +#if OS_WIN + oflag |= O_BINARY_NP; +#endif + m_fd = open(m_pathname.external_file_string().c_str(), oflag, S_IRWXO|S_IRWXU|S_IRWXG); + if(m_fd < 0) + WARN_RETURN(ERR::FILE_ACCESS); + + stats_open(); + return INFO::OK; +} + + +void File::Close() +{ + m_mode = '\0'; + + close(m_fd); + m_fd = 0; +} + + +LibError File::Issue(aiocb& req, off_t alignedOfs, u8* alignedBuf, size_t alignedSize) const +{ + memset(&req, 0, sizeof(req)); + req.aio_lio_opcode = (m_mode == 'w')? LIO_WRITE : LIO_READ; + req.aio_buf = (volatile void*)alignedBuf; + req.aio_fildes = m_fd; + req.aio_offset = alignedOfs; + req.aio_nbytes = alignedSize; + struct sigevent* sig = 0; // no notification signal + aiocb* const reqs = &req; + if(lio_listio(LIO_NOWAIT, &reqs, 1, sig) != 0) + return LibError_from_errno(); + return INFO::OK; +} + + +/*static*/ LibError File::WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize) +{ + // wait for transfer to complete. + while(aio_error(&req) == EINPROGRESS) + { + aiocb* const reqs = &req; + aio_suspend(&reqs, 1, (timespec*)0); // wait indefinitely + } + + const ssize_t bytesTransferred = aio_return(&req); + if(bytesTransferred == -1) // transfer failed + WARN_RETURN(ERR::IO); + + alignedBuf = (u8*)req.aio_buf; // cast from volatile void* + alignedSize = bytesTransferred; + return INFO::OK; +} + + +LibError File::IO(off_t ofs, u8* buf, size_t size) const +{ + ScopedIoMonitor monitor; + + lseek(m_fd, ofs, SEEK_SET); + + errno = 0; + const ssize_t ret = (m_mode == 'w')? write(m_fd, buf, size) : read(m_fd, buf, size); + if(ret < 0) + return LibError_from_errno(); + + const size_t totalTransferred = (size_t)ret; + if(totalTransferred != size) + WARN_RETURN(ERR::IO); + + monitor.NotifyOfSuccess(FI_LOWIO, m_mode, totalTransferred); + return INFO::OK; +} + + +LibError File::Write(off_t ofs, const u8* buf, size_t size) const +{ + return IO(ofs, const_cast(buf), size); +} + + +LibError File::Read(off_t ofs, u8* buf, size_t size) const +{ + return IO(ofs, buf, size); +} + + +//----------------------------------------------------------------------------- +// BlockIo +//----------------------------------------------------------------------------- + +class BlockIo +{ +public: + LibError Issue(const File& file, off_t alignedOfs, u8* alignedBuf) + { + m_blockId = BlockId(file.Pathname(), alignedOfs); + if(file.Mode() == 'r' && s_blockCache.Retrieve(m_blockId, m_cachedBlock)) + { + stats_block_cache(CR_HIT); + + // copy from cache into user buffer + if(alignedBuf) + { + cpu_memcpy(alignedBuf, m_cachedBlock.get(), BLOCK_SIZE); + m_alignedBuf = alignedBuf; + } + // return cached block + else + { + m_alignedBuf = const_cast(m_cachedBlock.get()); + } + + return INFO::OK; + } + else + { + stats_block_cache(CR_MISS); + stats_io_check_seek(m_blockId); + + // transfer directly to/from user buffer + if(alignedBuf) + { + m_alignedBuf = alignedBuf; + } + // transfer into newly allocated temporary block + else + { + m_tempBlock = io_Allocate(BLOCK_SIZE); + m_alignedBuf = const_cast(m_tempBlock.get()); + } + + return file.Issue(m_req, alignedOfs, m_alignedBuf, BLOCK_SIZE); + } + } + + LibError WaitUntilComplete(const u8*& block, size_t& blockSize) + { + if(m_cachedBlock) + { + block = m_alignedBuf; + blockSize = BLOCK_SIZE; + return INFO::OK; + } + + RETURN_ERR(File::WaitUntilComplete(m_req, const_cast(block), blockSize)); + + if(m_tempBlock) + s_blockCache.Add(m_blockId, m_tempBlock); + + return INFO::OK; + } + +private: + static BlockCache s_blockCache; + + BlockId m_blockId; + + // the address that WaitUntilComplete will return + // (cached or temporary block, or user buffer) + u8* m_alignedBuf; + + shared_ptr m_cachedBlock; + shared_ptr m_tempBlock; + + aiocb m_req; +}; + +BlockCache BlockIo::s_blockCache; + + +//----------------------------------------------------------------------------- +// IoSplitter +//----------------------------------------------------------------------------- + +class IoSplitter : boost::noncopyable +{ +public: + IoSplitter(off_t ofs, u8* alignedBuf, off_t size) + : m_ofs(ofs), m_alignedBuf(alignedBuf), m_size(size) + , m_totalIssued(0), m_totalTransferred(0) + { + m_misalignment = ofs % BLOCK_SIZE; + m_alignedOfs = ofs - (off_t)m_misalignment; + m_alignedSize = round_up(m_misalignment+size, BLOCK_SIZE); + } + + LibError Run(const File& file, IoCallback cb = 0, uintptr_t cbData = 0) + { + ScopedIoMonitor monitor; + + // (issue even if cache hit because blocks must be processed in order) + std::deque pendingIos; + for(;;) + { + while(pendingIos.size() < ioDepth && m_totalIssued < m_alignedSize) + { + pendingIos.push_back(BlockIo()); + const off_t alignedOfs = m_alignedOfs + m_totalIssued; + u8* const alignedBuf = m_alignedBuf? m_alignedBuf+m_totalIssued : 0; + RETURN_ERR(pendingIos.back().Issue(file, alignedOfs, alignedBuf)); + m_totalIssued += BLOCK_SIZE; + } + + if(pendingIos.empty()) + break; + + Process(pendingIos.front(), cb, cbData); + pendingIos.pop_front(); + } + + debug_assert(m_totalIssued >= m_totalTransferred && m_totalTransferred >= m_size); + + monitor.NotifyOfSuccess(FI_AIO, file.Mode(), m_totalTransferred); + return INFO::OK; + } + + size_t Misalignment() const + { + return m_misalignment; + } + +private: + LibError Process(BlockIo& blockIo, IoCallback cb, uintptr_t cbData) const + { + const u8* block; size_t blockSize; + RETURN_ERR(blockIo.WaitUntilComplete(block, blockSize)); + + // first block: skip past alignment + if(m_totalTransferred == 0) + { + block += m_misalignment; + blockSize -= m_misalignment; + } + + // last block: don't include trailing padding + if(m_totalTransferred + (off_t)blockSize > m_size) + blockSize = m_size - m_totalTransferred; + + m_totalTransferred += (off_t)blockSize; + + if(cb) + { + stats_cb_start(); + LibError ret = cb(cbData, block, blockSize); + stats_cb_finish(); + CHECK_ERR(ret); + } + + return INFO::OK; + } + + off_t m_ofs; + u8* m_alignedBuf; + off_t m_size; + + size_t m_misalignment; + off_t m_alignedOfs; + off_t m_alignedSize; + + // (useful, raw data: possibly compressed, but doesn't count padding) + mutable off_t m_totalIssued; + mutable off_t m_totalTransferred; +}; + + +LibError io_Scan(const File& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData) +{ + u8* alignedBuf = 0; // use temporary block buffers + IoSplitter splitter(ofs, alignedBuf, size); + return splitter.Run(file, cb, cbData); +} + + +LibError io_Read(const File& file, off_t ofs, u8* alignedBuf, size_t size, u8*& data) +{ + IoSplitter splitter(ofs, alignedBuf, (off_t)size); + RETURN_ERR(splitter.Run(file)); + data = alignedBuf + splitter.Misalignment(); + return INFO::OK; +} + + +LibError io_WriteAligned(const File& file, off_t alignedOfs, const u8* alignedData, size_t size) +{ + debug_assert(IsAligned(alignedOfs, BLOCK_SIZE)); + debug_assert(IsAligned(alignedData, BLOCK_SIZE)); + + IoSplitter splitter(alignedOfs, const_cast(alignedData), (off_t)size); + return splitter.Run(file); +} + + +LibError io_ReadAligned(const File& file, off_t alignedOfs, u8* alignedBuf, size_t size) +{ + debug_assert(IsAligned(alignedOfs, BLOCK_SIZE)); + debug_assert(IsAligned(alignedBuf, BLOCK_SIZE)); + + IoSplitter splitter(alignedOfs, alignedBuf, (off_t)size); + return splitter.Run(file); +} + + +//----------------------------------------------------------------------------- +// UnalignedWriter +//----------------------------------------------------------------------------- + +class UnalignedWriter::Impl : boost::noncopyable +{ +public: + Impl(const File& file, off_t ofs) + : m_file(file), m_alignedBuf(io_Allocate(BLOCK_SIZE)) + { + const size_t misalignment = (size_t)ofs % BLOCK_SIZE; + m_alignedOfs = ofs - (off_t)misalignment; + if(misalignment) + io_ReadAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE); + m_bytesUsed = misalignment; + } + + ~Impl() + { + Flush(); + } + + LibError Append(const u8* data, size_t size) const + { + while(size != 0) + { + // optimization: write directly from the input buffer, if possible + const size_t alignedSize = (size / BLOCK_SIZE) * BLOCK_SIZE; + if(m_bytesUsed == 0 && IsAligned(data, BLOCK_SIZE) && alignedSize != 0) + { + RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, data, alignedSize)); + m_alignedOfs += (off_t)alignedSize; + data += alignedSize; + size -= alignedSize; + } + + const size_t chunkSize = std::min(size, BLOCK_SIZE-m_bytesUsed); + cpu_memcpy(m_alignedBuf.get()+m_bytesUsed, data, chunkSize); + m_bytesUsed += chunkSize; + data += chunkSize; + size -= chunkSize; + + if(m_bytesUsed == BLOCK_SIZE) + RETURN_ERR(WriteBlock()); + } + + return INFO::OK; + } + + void Flush() const + { + if(m_bytesUsed) + { + memset(m_alignedBuf.get()+m_bytesUsed, 0, BLOCK_SIZE-m_bytesUsed); + (void)WriteBlock(); + } + } + +private: + LibError WriteBlock() const + { + RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE)); + m_alignedOfs += BLOCK_SIZE; + m_bytesUsed = 0; + return INFO::OK; + } + + const File& m_file; + mutable off_t m_alignedOfs; + shared_ptr m_alignedBuf; + mutable size_t m_bytesUsed; +}; + + +UnalignedWriter::UnalignedWriter(const File& file, off_t ofs) +: impl(new Impl(file, ofs)) +{ +} + +LibError UnalignedWriter::Append(const u8* data, size_t size) const +{ + return impl->Append(data, size); +} + +void UnalignedWriter::Flush() const +{ + impl->Flush(); +} diff --git a/source/lib/file/io/io.h b/source/lib/file/io/io.h new file mode 100644 index 0000000000..c050480d3b --- /dev/null +++ b/source/lib/file/io/io.h @@ -0,0 +1,107 @@ +/** + * ========================================================================= + * File : io.h + * Project : 0 A.D. + * Description : + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#ifndef INCLUDED_IO +#define INCLUDED_IO + +#include "lib/file/path.h" + +namespace ERR +{ + const LibError FILE_ACCESS = -110200; + const LibError IO = -110201; + const LibError IO_EOF = -110202; +} + + +/** + * simple POSIX file wrapper. + **/ +class LIB_API File +{ +public: + File(); + ~File(); + + LibError Open(const Path& pathname, char mode); + void Close(); + + const Path& Pathname() const + { + return m_pathname; + } + + char Mode() const + { + return m_mode; + } + + LibError Issue(aiocb& req, off_t alignedOfs, u8* alignedBuf, size_t alignedSize) const; + static LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize); + + LibError Read(off_t ofs, u8* buf, size_t size) const; + LibError Write(off_t ofs, const u8* buf, size_t size) const; + +private: + LibError IO(off_t ofs, u8* buf, size_t size) const; + + Path m_pathname; + char m_mode; + int m_fd; +}; + +typedef shared_ptr PFile; + + +// memory will be allocated from the heap, not the (limited) file cache. +// this makes sense for write buffers that are never used again, +// because we avoid having to displace some other cached items. +shared_ptr io_Allocate(size_t size, off_t ofs = 0); + + +/** + * called after a block IO has completed. + * + * @return INFO::CB_CONTINUE to continue; any other value will cause the + * IO splitter to abort immediately and return that. + * + * this is useful for interleaving e.g. decompression with IOs. + **/ +typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t blockSize); + +LIB_API LibError io_Scan(const File& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData); + +LIB_API LibError io_Read(const File& file, off_t ofs, u8* alignedBuf, size_t size, u8*& data); + +LIB_API LibError io_WriteAligned(const File& file, off_t alignedOfs, const u8* alignedData, size_t size); +LIB_API LibError io_ReadAligned(const File& file, off_t alignedOfs, u8* alignedBuf, size_t size); + +class LIB_API UnalignedWriter +{ +public: + UnalignedWriter(const File& file, off_t ofs); + + /** + * add data to the align buffer, writing it out to disk if full. + **/ + LibError Append(const u8* data, size_t size) const; + + /** + * zero-initialize any remaining space in the align buffer and write + * it to the file. this is called by the destructor. + **/ + void Flush() const; + +private: + class Impl; + shared_ptr impl; +}; + +#endif // #ifndef INCLUDED_IO diff --git a/source/lib/file/io/io_buf.cpp b/source/lib/file/io/io_buf.cpp deleted file mode 100644 index 17f9f5cdb1..0000000000 --- a/source/lib/file/io/io_buf.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "precompiled.h" -#include "io_buf.h" - -#include "lib/allocators/allocators.h" // AllocatorChecker -#include "lib/bits.h" // round_up -#include "block_cache.h" // BLOCK_SIZE - - -#ifndef NDEBUG -static AllocatorChecker allocatorChecker; -#endif - - -class IoBufDeleter -{ -public: - IoBufDeleter(size_t paddedSize) - : m_paddedSize(paddedSize) - { - debug_assert(m_paddedSize != 0); - } - - void operator()(const u8* mem) - { - debug_assert(m_paddedSize != 0); -#ifndef NDEBUG - allocatorChecker.notify_free((void*)mem, m_paddedSize); -#endif - page_aligned_free((void*)mem, m_paddedSize); - m_paddedSize = 0; - } - -private: - size_t m_paddedSize; -}; - - -IoBuf io_buf_Allocate(size_t size) -{ - const size_t paddedSize = (size_t)round_up(size, BLOCK_SIZE); - const u8* mem = (const u8*)page_aligned_alloc(paddedSize); - if(!mem) - throw std::bad_alloc(); - -#ifndef NDEBUG - allocatorChecker.notify_alloc((void*)mem, paddedSize); -#endif - - return IoBuf(mem, IoBufDeleter(paddedSize)); -} diff --git a/source/lib/file/io/io_buf.h b/source/lib/file/io/io_buf.h deleted file mode 100644 index 20066f6791..0000000000 --- a/source/lib/file/io/io_buf.h +++ /dev/null @@ -1,22 +0,0 @@ -/** - * ========================================================================= - * File : io_buf.h - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_IO_BUF -#define INCLUDED_IO_BUF - -typedef boost::shared_ptr IoBuf; -const IoBuf IO_BUF_TEMP((const u8*)0); - -// memory will be allocated from the heap, not the (limited) file cache. -// this makes sense for write buffers that are never used again, -// because we avoid having to displace some other cached items. -extern IoBuf io_buf_Allocate(size_t size); - -#endif // #ifndef INCLUDED_IO_BUF diff --git a/source/lib/file/io/io_internal.h b/source/lib/file/io/io_internal.h new file mode 100644 index 0000000000..75d06a5d84 --- /dev/null +++ b/source/lib/file/io/io_internal.h @@ -0,0 +1,11 @@ +/** + * block := power-of-two sized chunk of a file. + * all transfers are expanded to naturally aligned, whole blocks. + * (this makes caching parts of files feasible; it is also much faster + * for some aio implementations, e.g. wposix.) + * (blocks are also thereby page-aligned, which allows write-protecting + * file buffers without worrying about their boundaries.) + **/ +static const size_t BLOCK_SIZE = 64*KiB; + +static const unsigned ioDepth = 8; diff --git a/source/lib/file/io/io_manager.cpp b/source/lib/file/io/io_manager.cpp deleted file mode 100644 index 5016a86205..0000000000 --- a/source/lib/file/io/io_manager.cpp +++ /dev/null @@ -1,362 +0,0 @@ -/** - * ========================================================================= - * File : - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "io_manager.h" - -#include "../posix/io_posix.h" -#include "../file_stats.h" -#include "block_cache.h" - - -// the underlying aio implementation likes buffer and offset to be -// sector-aligned; if not, the transfer goes through an align buffer, -// and requires an extra cpu_memcpy. -// -// if the user specifies an unaligned buffer, there's not much we can -// do - we can't assume the buffer contains padding. therefore, -// callers should let us allocate the buffer if possible. -// -// if ofs misalign = buffer, only the first and last blocks will need -// to be copied by aio, since we read up to the next block boundary. -// otherwise, everything will have to be copied; at least we split -// the read into blocks, so aio's buffer won't have to cover the -// whole file. - - -// note: cutting off at EOF is necessary to avoid transfer errors, -// but makes size no longer sector-aligned, which would force -// waio to realign (slow). we want to pad back to sector boundaries -// afterwards (to avoid realignment), but that is not possible here -// since we have no control over the buffer (there might not be -// enough room in it). hence, do cut-off in IOManager. -// -// example: 200-byte file. IOManager issues (large) blocks; -// that ends up way beyond EOF, so ReadFile fails. -// limiting size to 200 bytes works, but causes waio to pad the -// transfer and use align buffer (slow). -// rounding up to 512 bytes avoids realignment and does not fail -// (apparently since NTFS files are sector-padded anyway?) - - - -//----------------------------------------------------------------------------- - -class BlockIo -{ -public: - BlockIo() - : m_blockId(), m_cachedBlock(0), m_tempBlock(0), m_posixIo() - { - } - - LibError Issue(File_Posix& file, off_t ofs, IoBuf buf, size_t size) - { - m_blockId = BlockId(file.Pathname(), ofs); - - // block already available in cache? - if(s_blockCache.Retrieve(m_blockId, m_cachedBlock)) - { - stats_block_cache(CR_HIT); - return INFO::OK; - } - - stats_block_cache(CR_MISS); - stats_io_check_seek(m_blockId); - - // use a temporary block if not writing to a preallocated buffer. - if(!buf) - buf = m_tempBlock = s_blockCache.Reserve(m_blockId); - - return m_posixIo.Issue(file, ofs, buf, size); - } - - LibError WaitUntilComplete(const u8*& block, size_t& blockSize) - { - block = m_cachedBlock.get(); - if(block) - { - blockSize = BLOCK_SIZE; - return INFO::OK; - } - - return m_posixIo.WaitUntilComplete(block, blockSize); - } - - void Discard() - { - if(m_cachedBlock) - { - s_blockCache.Release(m_blockId); - m_cachedBlock = 0; - return; - } - - if(m_tempBlock) - { - s_blockCache.MarkComplete(m_blockId); - m_tempBlock = 0; - } - } - -private: - static BlockCache s_blockCache; - - BlockId m_blockId; - IoBuf m_cachedBlock; - - IoBuf m_tempBlock; - - Io_Posix m_posixIo; -}; - - -//----------------------------------------------------------------------------- - -class IOManager : boost::noncopyable -{ -public: - IOManager(File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0) - : m_file(file) - , start_ofs(ofs), user_size(size) - , m_cb(cb), m_cbData(cbData) - , m_totalIssued(0), m_totalTransferred(0), m_totalProcessed(0) - , err(INFO::CB_CONTINUE) - { - } - - - // now we read the file in 64 KiB chunks, N-buffered. - // if reading from Zip, inflate while reading the next block. - LibError run() - { - ScopedIoMonitor monitor; - - aio(); - - if(err != INFO::CB_CONTINUE && err != INFO::OK) - return (ssize_t)err; - - debug_assert(m_totalIssued >= m_totalTransferred && m_totalTransferred >= user_size); - - monitor.NotifyOfSuccess(FI_AIO, m_file.Mode(), m_totalTransferred); - return m_totalProcessed; - } - - -private: - // helper routine used by functions that call back to a IoCallback. - // - // bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE } - // note: don't abort if = 0: zip callback may not actually - // output anything if passed very little data. - static LibError InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed) - { - if(cb) - { - stats_cb_start(); - LibError ret = cb(cbData, block, size, bytesProcessed); - stats_cb_finish(); - - // failed - reset byte count in case callback didn't - if(ret != INFO::OK && ret != INFO::CB_CONTINUE) - bytesProcessed = 0; - - CHECK_ERR(ret); // user might not have raised a warning; make sure - return ret; - } - // no callback to process data: raw = actual - else - { - bytesProcessed = size; - return INFO::CB_CONTINUE; - } - } - - void wait(BlockIo& blockIo, u8*& block, size_t& blockSize) - { - LibError ret = blockIo.WaitUntilComplete(block, blockSize); - if(ret < 0) - err = ret; - - // first time; skip past padding - if(m_totalTransferred == 0) - { - block = (u8*)block + ofs_misalign; - blockSize -= ofs_misalign; - } - - // last time: don't include trailing padding - if(m_totalTransferred + blockSize > user_size) - blockSize = user_size - m_totalTransferred; - - // we have useable data from a previous temp buffer, - // but it needs to be copied into the user's buffer - if(blockIo.m_cachedBlock && pbuf != IO_BUF_TEMP) - cpu_memcpy((char*)*pbuf+ofs_misalign+m_totalTransferred, block, blockSize); - - m_totalTransferred += blockSize; - } - - - // align and pad the IO to BLOCK_SIZE - // (reduces work for AIO implementation). - LibError prepare() - { - ofs_misalign = 0; - size = user_size; - - if(!is_write && !no_aio) - { - // note: we go to the trouble of aligning the first block (instead of - // just reading up to the next block and letting aio realign it), - // so that it can be taken from the cache. - // this is not possible if we don't allocate the buffer because - // extra space must be added for the padding. - - ofs_misalign = start_ofs % BLOCK_SIZE; - start_ofs -= (off_t)ofs_misalign; - size = round_up(ofs_misalign + user_size, BLOCK_SIZE); - - // but cut off at EOF (necessary to prevent IO error). - const off_t bytes_left = f->size - start_ofs; - if(bytes_left < 0) - WARN_RETURN(ERR::IO_EOF); - size = std::min(size, (size_t)bytes_left); - - // and round back up to sector size. - // see rationale in file_io_issue. - const size_t AIO_SECTOR_SIZE = 512; - size = round_up(size, AIO_SECTOR_SIZE); - } - - RETURN_ERR(file_io_get_buf(pbuf, size, f->atom_fn, f->flags, cb)); - - // see if actual transfer count matches requested size. - // note: most callers clamp to EOF but round back up to sector size - // (see explanation in file_io_issue). - ////debug_assert(bytes_transferred >= (ssize_t)(m_aiocb.aio_nbytes-AIO_SECTOR_SIZE)); - - - return INFO::OK; - } - - - void aio() - { - RETURN_ERR(prepare()); - -again: - { - // data remaining to transfer, and no error: - // start transferring next block. - if(m_totalIssued < size && err == INFO::CB_CONTINUE && queue.size() < MAX_PENDING_IOS) - { - queue.push_back(BlockIo()); - BlockIo& blockIo = queue.back(); - - const off_t ofs = start_ofs+(off_t)m_totalIssued; - // for both reads and writes, do not issue beyond end of file/data - const size_t issue_size = std::min(BLOCK_SIZE, size - m_totalIssued); - // try to grab whole blocks (so we can put them in the cache). - // any excess data (can only be within first or last) is - // discarded in wait(). - - if(pbuf == IO_BUF_TEMP) - buf = 0; - else - buf = (char*)*pbuf + m_totalIssued; - - LibError ret = blockIo.Issue(); - // transfer failed - loop will now terminate after - // waiting for all pending transfers to complete. - if(ret != INFO::OK) - err = ret; - - m_totalIssued += issue_size; - - goto again; - } - - // IO pending: wait for it to complete, and process it. - if(!queue.empty()) - { - BlockIo& blockIo = queue.front(); - u8* block; size_t blockSize; - wait(blockIo, block, blockSize); - - - if(err == INFO::CB_CONTINUE) - { - size_t bytesProcessed; - LibError ret = io_InvokeCallback(block, blockSize, m_cb, m_cbData, bytesProcessed); - if(ret == INFO::CB_CONTINUE || ret == INFO::OK) - m_totalProcessed += bytesProcessed; - // processing failed - loop will now terminate after - // waiting for all pending transfers to complete. - else - err = ret; - } - - blockIo.Discard(); - - - queue.pop_front(); - goto again; - } - } - // (all issued OR error) AND no pending transfers - done. - - - // we allocated the memory: skip any leading padding - if(not_temp && !is_write) - { - IoBuf org_buf = *pbuf; - *pbuf = (u8*)org_buf + ofs_misalign; - if(ofs_misalign || size != user_size) - assert(0); // TODO": no longer supported, rule this out - } - - } - - File_Posix& m_file; - bool m_isWrite; - -off_t start_ofs; -size_t user_size; - IoCallback m_cb; - uintptr_t m_cbData; - - // (useful, raw data: possibly compressed, but doesn't count padding) - size_t m_totalIssued; - size_t m_totalTransferred; - // if callback, sum of what it reports; otherwise, = m_totalTransferred - // this is what we'll return. - size_t m_totalProcessed; - - // stop issuing and processing as soon as this changes - LibError err; - -IoBuf* pbuf; -size_t ofs_misalign; -size_t size; - - static const uint MAX_PENDING_IOS = 4; - //RingBuf queue; - std::deque queue; -}; - -LibError io(File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb, uintptr_t cbData) -{ - debug_printf("IO| size=%d\n", size); - - - return ERR::NOT_IMPLEMENTED; -} \ No newline at end of file diff --git a/source/lib/file/io/io_manager.h b/source/lib/file/io/io_manager.h deleted file mode 100644 index d18a162632..0000000000 --- a/source/lib/file/io/io_manager.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * ========================================================================= - * File : io_manager.h - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_IO_MANAGER -#define INCLUDED_IO_MANAGER - -#include "lib/file/posix/io_posix.h" // File_Posix, IoBuf - -// called by file_io after a block IO has completed. -// bytesProcessed must be set; file_io will return the sum of these values. -// example: when reading compressed data and decompressing in the callback, -// indicate #bytes decompressed. -// return value: INFO::CB_CONTINUE to continue calling; anything else: -// abort immediately and return that. -// note: in situations where the entire IO is not split into blocks -// (e.g. when reading from cache or not using AIO), this is still called but -// for the entire IO. we do not split into fake blocks because it is -// advantageous (e.g. for decompressors) to have all data at once, if available -// anyway. -typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t size, size_t& bytesProcessed); - -extern LibError io_Read(const File_Posix& file, off_t ofs, u8* buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0); -extern LibError io_Read(const File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0); - -extern LibError io_Write(File_Posix& file, off_t ofs, const u8* data, size_t size, IoCallback cb = 0, uintptr_t cbData = 0); -extern LibError io_Write(File_Posix& file, off_t ofs, IoBuf data, size_t size, IoCallback cb = 0, uintptr_t cbData = 0); - -#endif // #ifndef INCLUDED_IO_MANAGER diff --git a/source/lib/file/io/write_buffer.h b/source/lib/file/io/write_buffer.h new file mode 100644 index 0000000000..dbce7e5ad7 --- /dev/null +++ b/source/lib/file/io/write_buffer.h @@ -0,0 +1,47 @@ +#ifndef INCLUDED_WRITE_BUFFER +#define INCLUDED_WRITE_BUFFER + +class LIB_API WriteBuffer +{ +public: + WriteBuffer() + : m_capacity(4096), m_data(io_Allocate(m_capacity)), m_size(0) + { + } + + void Append(const void* data, size_t size) + { + while(m_size + size > m_capacity) + m_capacity *= 2; + shared_ptr newData = io_Allocate(m_capacity); + cpu_memcpy(newData.get(), m_data.get(), m_size); + m_data = newData; + + cpu_memcpy(m_data.get() + m_size, data, size); + m_size += size; + } + + void Overwrite(const void* data, size_t size, size_t offset) + { + debug_assert(offset+size < m_size); + cpu_memcpy(m_data.get()+offset, data, size); + } + + shared_ptr Data() const + { + return m_data; + } + + size_t Size() const + { + return m_size; + } + +private: + size_t m_capacity; // must come first (init order) + + shared_ptr m_data; + size_t m_size; +}; + +#endif // #ifndef INCLUDED_WRITE_BUFFER diff --git a/source/lib/file/path.cpp b/source/lib/file/path.cpp index 54ddce956e..86ec90d9ee 100644 --- a/source/lib/file/path.cpp +++ b/source/lib/file/path.cpp @@ -22,10 +22,6 @@ ERROR_ASSOCIATE(ERR::PATH_ROOT_DIR_ALREADY_SET, "Attempting to set FS root dir m ERROR_ASSOCIATE(ERR::PATH_NOT_IN_ROOT_DIR, "Accessing a file that's outside of the root dir", -1); -// set by path_SetRoot -static char osRootPath[PATH_MAX]; -static size_t osRootPathLength; - // security check: only allow path_SetRoot once so that malicious code // cannot circumvent the VFS checks that disallow access to anything above // the current directory (set here). @@ -33,14 +29,33 @@ static size_t osRootPathLength; // are likely bogus. // we provide for resetting this from the self-test to allow clean // re-init of the individual tests. -static bool isRootDirEstablished; +static bool s_isRootPathEstablished; + +static std::string s_rootPath; -LibError path_SetRoot(const char* argv0, const char* rel_path) +/*static*/ PathTraits::external_string_type PathTraits::to_external(const Path&, const PathTraits::internal_string_type& src) { - if(isRootDirEstablished) + std::string absolutePath = s_rootPath + src; + std::replace(absolutePath.begin(), absolutePath.end(), '/', SYS_DIR_SEP); + return absolutePath; +} + +/*static*/ PathTraits::internal_string_type PathTraits::to_internal(const PathTraits::external_string_type& src) +{ + if(s_rootPath.compare(0, s_rootPath.length(), src) != 0) + DEBUG_WARN_ERR(ERR::PATH_NOT_IN_ROOT_DIR); + std::string relativePath = src.substr(s_rootPath.length(), src.length()-s_rootPath.length()); + std::replace(relativePath.begin(), relativePath.end(), SYS_DIR_SEP, '/'); + return relativePath; +} + + +LibError path_SetRoot(const char* argv0, const char* relativePath) +{ + if(s_isRootPathEstablished) WARN_RETURN(ERR::PATH_ROOT_DIR_ALREADY_SET); - isRootDirEstablished = true; + s_isRootPathEstablished = true; // get full path to executable char osPathname[PATH_MAX]; @@ -62,21 +77,17 @@ LibError path_SetRoot(const char* argv0, const char* rel_path) char* name = (char*)path_name_only(osPathname); *name = '\0'; - strcat_s(osPathname, ARRAY_SIZE(osRootPath), rel_path); + strcat_s(osPathname, PATH_MAX, relativePath); // get actual root dir - previous osPathname may include ".." // (slight optimization, speeds up path lookup) errno = 0; + char osRootPath[PATH_MAX]; if(!realpath(osPathname, osRootPath)) return LibError_from_errno(); - // .. append SYS_DIR_SEP to simplify code that uses osRootPath - osRootPathLength = strlen(osRootPath)+1; // +1 for trailing SYS_DIR_SEP - debug_assert((osRootPathLength+1) < sizeof(osRootPath)); // Just checking - osRootPath[osRootPathLength-1] = SYS_DIR_SEP; - // You might think that osRootPath is already 0-terminated, since it's - // static - but that might not be true after calling path_ResetRootDir! - osRootPath[osRootPathLength] = 0; + s_rootPath = osRootPath; + s_rootPath.append(1, SYS_DIR_SEP); // simplifies to_external return INFO::OK; } @@ -84,47 +95,7 @@ LibError path_SetRoot(const char* argv0, const char* rel_path) void path_ResetRootDir() { - // see comment at isRootDirEstablished. - debug_assert(isRootDirEstablished); - osRootPath[0] = '\0'; - osRootPathLength = 0; - isRootDirEstablished = false; -} - - -// (this assumes SYS_DIR_SEP is a single character) -static void ConvertSlashCharacters(char* dst, const char* src, char from, char to) -{ - for(size_t len = 0; len < PATH_MAX; len++) - { - char c = *src++; - if(c == from) - c = to; - *dst++ = c; - - // end of string - done - if(c == '\0') - return; - } - - DEBUG_WARN_ERR(ERR::PATH_LENGTH); -} - - -void path_MakeAbsolute(const char* path, char* osPath) -{ - debug_assert(path != osPath); // doesn't work in-place - - strcpy_s(osPath, PATH_MAX, osRootPath); - ConvertSlashCharacters(osPath+osRootPathLength, path, '/', SYS_DIR_SEP); -} - - -void path_MakeRelative(const char* osPath, char* path) -{ - debug_assert(path != osPath); // doesn't work in-place - - if(strncmp(osPath, osRootPath, osRootPathLength) != 0) - DEBUG_WARN_ERR(ERR::PATH_NOT_IN_ROOT_DIR); - ConvertSlashCharacters(path, osPath+osRootPathLength, SYS_DIR_SEP, '/'); + debug_assert(s_isRootPathEstablished); // see comment at s_isRootPathEstablished. + s_rootPath.clear(); + s_isRootPathEstablished = false; } diff --git a/source/lib/file/path.h b/source/lib/file/path.h index ae133c38c2..847698b0da 100644 --- a/source/lib/file/path.h +++ b/source/lib/file/path.h @@ -20,6 +20,19 @@ #ifndef INCLUDED_PATH #define INCLUDED_PATH +struct PathTraits; +typedef fs::basic_path Path; + +struct PathTraits +{ + typedef std::string internal_string_type; + typedef std::string external_string_type; + + static external_string_type to_external(const Path&, const internal_string_type& src); + static internal_string_type to_internal(const external_string_type& src); +}; + + namespace ERR { const LibError PATH_ROOT_DIR_ALREADY_SET = -110200; @@ -33,7 +46,7 @@ namespace ERR * of the executable in case sys_get_executable_path fails). note that * the current directory cannot be used because it's not set when * starting via batch file. - * @param rel_path root directory relative to the executable's directory. + * @param relativePath root directory relative to the executable's directory. * the value is considered trusted since it will typically be hard-coded. * * example: executable in "$install_dir/system"; desired root dir is @@ -41,7 +54,7 @@ namespace ERR * * can only be called once unless path_ResetRootDir is called. **/ -extern LibError path_SetRoot(const char* argv0, const char* rel_path); +LIB_API LibError path_SetRoot(const char* argv0, const char* relativePath); /** * reset root directory that was previously established via path_SetRoot. @@ -50,21 +63,8 @@ extern LibError path_SetRoot(const char* argv0, const char* rel_path); * path_SetRoot is called twice; it is provided for the * legitimate application of a self-test setUp()/tearDown(). **/ -extern void path_ResetRootDir(); +LIB_API void path_ResetRootDir(); - -/** - * return the absolute OS path for a given relative portable path. - * - * this is useful for external libraries that require the real filename. - **/ -extern void path_MakeAbsolute(const char* path, char* osPath); - -/** - * return the relative portable path for a given absolute OS path. - * - * this is useful when receiving paths from external libraries (e.g. FAM). - **/ -extern void path_MakeRelative(const char* osPath, char* path); +// note: path_MakeAbsolute has been replaced by Path::external_directory_string. #endif // #ifndef INCLUDED_PATH diff --git a/source/lib/file/posix/fs_posix.cpp b/source/lib/file/posix/fs_posix.cpp deleted file mode 100644 index a091f7855e..0000000000 --- a/source/lib/file/posix/fs_posix.cpp +++ /dev/null @@ -1,214 +0,0 @@ -/** - * ========================================================================= - * File : fs_posix.cpp - * Project : 0 A.D. - * Description : file layer on top of POSIX. avoids the need for - * : absolute paths and provides fast I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "fs_posix.h" - -#include -#include -#include - -#include "lib/path_util.h" -#include "lib/file/path.h" -#include "lib/posix/posix_filesystem.h" - - -struct DirDeleter -{ - void operator()(DIR* osDir) const - { - const int ret = closedir(osDir); - debug_assert(ret == 0); - } -}; - -// is name "." or ".."? -static bool IsDummyDirectory(const char* name) -{ - if(name[0] != '.') - return false; - return (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); -} - -/*virtual*/ LibError Filesystem_Posix::GetDirectoryEntries(const char* path, FileInfos* files, Directories* subdirectories) const -{ - // open directory - char osPath[PATH_MAX]; - path_MakeAbsolute(path, osPath); - errno = 0; - boost::shared_ptr osDir(opendir(osPath), DirDeleter()); - if(!osDir.get()) - return LibError_from_errno(); - - // (we need absolute paths below; using this instead of path_append avoids - // a strlen call for each entry.) - PathPackage pp; - (void)path_package_set_dir(&pp, osPath); - - for(;;) - { - errno = 0; - struct dirent* osEnt = readdir(osDir.get()); - if(!osEnt) - { - // no error, just no more entries to return - if(!errno) - return INFO::OK; - return LibError_from_errno(); - } - - const char* name = path_Pool()->UniqueCopy(osEnt->d_name); - RETURN_ERR(path_component_validate(name)); - - // get file information (mode, size, mtime) - struct stat s; -#if OS_WIN - // .. wposix readdir has enough information to return dirent - // status directly (much faster than calling stat). - RETURN_ERR(readdir_stat_np(osDir.get(), &s)); -#else - // .. call regular stat(). - errno = 0; - path_package_append_file(&pp, name); - if(stat(pp->path, &s) != 0) - return LibError_from_errno(); -#endif - - if(files && S_ISREG(s.st_mode)) - files->push_back(FileInfo(name, s.st_size, s.st_mtime)); - else if(subdirectories && S_ISDIR(s.st_mode) && !IsDummyDirectory(name)) - subdirectories->push_back(name); - } -} - - -LibError Filesystem_Posix::GetFileInfo(const char* pathname, FileInfo& fileInfo) const -{ - char osPathname[PATH_MAX]; - path_MakeAbsolute(pathname, osPathname); - - // if path ends in slash, remove it (required by stat) - char* last_char = osPathname+strlen(osPathname)-1; - if(path_is_dir_sep(*last_char)) - *last_char = '\0'; - - errno = 0; - struct stat s; - memset(&s, 0, sizeof(s)); - if(stat(osPathname, &s) != 0) - return LibError_from_errno(); - - const char* name = path_Pool()->UniqueCopy(path_name_only(osPathname)); - fileInfo = FileInfo(name, s.st_size, s.st_mtime); - return INFO::OK; -} - - -LibError Filesystem_Posix::DeleteFile(const char* pathname) -{ - char osPathname[PATH_MAX+1]; - path_MakeAbsolute(pathname, osPathname); - - errno = 0; - if(unlink(osPathname) != 0) - return LibError_from_errno(); - - return INFO::OK; -} - - -LibError Filesystem_Posix::CreateDirectory(const char* path) -{ - char osPath[PATH_MAX]; - path_MakeAbsolute(path, osPath); - - errno = 0; - struct stat s; - if(stat(osPath, &s) != 0) - return LibError_from_errno(); - - errno = 0; - if(mkdir(osPath, S_IRWXO|S_IRWXU|S_IRWXG) != 0) - return LibError_from_errno(); - - return INFO::OK; -} - - -LibError Filesystem_Posix::DeleteDirectory(const char* path) -{ - // note: we have to recursively empty the directory before it can - // be deleted (required by Windows and POSIX rmdir()). - - char osPath[PATH_MAX]; - path_MakeAbsolute(path, osPath); - PathPackage pp; - RETURN_ERR(path_package_set_dir(&pp, osPath)); - - FileInfos files; Directories subdirectories; - RETURN_ERR(GetDirectoryEntries(path, &files, &subdirectories)); - - // delete files - for(size_t i = 0; i < files.size(); i++) - { - RETURN_ERR(path_package_append_file(&pp, files[i].Name())); - errno = 0; - if(unlink(pp.path) != 0) - return LibError_from_errno(); - } - - // recurse over subdirectories - for(size_t i = 0; i < subdirectories.size(); i++) - { - char subdirectoryPath[PATH_MAX]; - path_append(subdirectoryPath, path, subdirectories[i]); - RETURN_ERR(DeleteDirectory(subdirectoryPath)); - } - - errno = 0; - if(rmdir(osPath) != 0) - return LibError_from_errno(); - - return INFO::OK; -} - - -//----------------------------------------------------------------------------- - -/*virtual*/ unsigned FileProvider_Posix::Precedence() const -{ - return 1u; -} - -/*virtual*/ char FileProvider_Posix::LocationCode() const -{ - return 'F'; -} - -/*virtual*/ LibError FileProvider_Posix::LoadFile(const char* name, const void* location, u8* fileContents, size_t size) const -{ - const char* path = (const char*)location; - const char* pathname = path_append2(path, name); - File_Posix file; - RETURN_ERR(file.Open(pathname, 'r')); - RETURN_ERR(io_Read(file, 0, buf, size)); - return INFO::OK; -} - -/*virtual*/ LibError FileProvider_Posix::StoreFile(const char* name, const void* location, const u8* fileContents, size_t size) const -{ - const char* path = (const char*)location; - const char* pathname = path_append2(path, name); - File_Posix file; - RETURN_ERR(file.Open(pathname, 'r')); - RETURN_ERR(io_Write(file, 0, buf, size)); - return INFO::OK; -} diff --git a/source/lib/file/posix/fs_posix.h b/source/lib/file/posix/fs_posix.h deleted file mode 100644 index 9f5e42b8ec..0000000000 --- a/source/lib/file/posix/fs_posix.h +++ /dev/null @@ -1,37 +0,0 @@ -/** - * ========================================================================= - * File : fs_posix.h - * Project : 0 A.D. - * Description : file layer on top of POSIX. avoids the need for - * : absolute paths and provides fast I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_FS_POSIX -#define INCLUDED_FS_POSIX - -#include "lib/file/filesystem.h" - -struct FileProvider_Posix : public IFileProvider -{ - virtual unsigned Precedence() const; - virtual char LocationCode() const; - - virtual LibError Load(const char* name, const void* location, u8* fileContents, size_t size) const; - virtual LibError Store(const char* name, const void* location, const u8* fileContents, size_t size) const; -}; - - -struct Filesystem_Posix : public IFilesystem -{ - virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const; - virtual LibError GetDirectoryEntries(const char* path, FileInfos* files, Directories* subdirectories) const; - - LibError DeleteFile(const char* pathname); - LibError CreateDirectory(const char* dirPath); - LibError DeleteDirectory(const char* dirPath); -}; - -#endif // #ifndef INCLUDED_FS_POSIX diff --git a/source/lib/file/posix/io_posix.cpp b/source/lib/file/posix/io_posix.cpp deleted file mode 100644 index e24ee213a5..0000000000 --- a/source/lib/file/posix/io_posix.cpp +++ /dev/null @@ -1,205 +0,0 @@ -/** - * ========================================================================= - * File : io_posix.cpp - * Project : 0 A.D. - * Description : lightweight POSIX aio wrapper - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "io_posix.h" - -#include "lib/file/filesystem.h" -#include "lib/file/path.h" -#include "lib/file/file_stats.h" -#include "lib/posix/posix_aio.h" - - -ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES); -ERROR_ASSOCIATE(ERR::DIR_NOT_FOUND, "Directory not found", ENOENT); -ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO); -ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1); - - -//----------------------------------------------------------------------------- - -File_Posix::File_Posix() -{ -} - - -File_Posix::~File_Posix() -{ - Close(); -} - - -LibError File_Posix::Open(const char* P_pathname, char mode) -{ - debug_assert(mode == 'w' || mode == 'r'); - - m_pathname = path_Pool()->UniqueCopy(P_pathname); - m_mode = mode; - - char N_pathname[PATH_MAX]; - (void)path_MakeAbsolute(P_pathname, N_pathname); - - int oflag = (mode == 'r')? O_RDONLY : O_WRONLY|O_CREAT|O_TRUNC; - m_fd = open(N_pathname, oflag, S_IRWXO|S_IRWXU|S_IRWXG); - if(m_fd < 0) - WARN_RETURN(ERR::FILE_ACCESS); - - stats_open(m_pathname, Size()); - - return INFO::OK; -} - - -void File_Posix::Close() -{ - close(m_fd); - m_fd = 0; -} - - -LibError File_Posix::Validate() const -{ - if(path_Pool()->UniqueCopy(m_pathname) != m_pathname) - WARN_RETURN(ERR::_1); - if((m_mode != 'w' && m_mode != 'r')) - WARN_RETURN(ERR::_2); - // >= 0x100 is not necessarily bogus, but suspicious. - if(!(3 <= m_fd && m_fd < 0x100)) - WARN_RETURN(ERR::_3); - - return INFO::OK; -} - - -//----------------------------------------------------------------------------- - -class Io_Posix::Impl -{ -public: - Impl() - { - memset(&m_aiocb, 0, sizeof(m_aiocb)); - } - - LibError Issue(File_Posix& file, off_t ofs, IoBuf buf, size_t size) - { - debug_printf("FILE| Issue ofs=0x%X size=0x%X\n", ofs, size); - - m_aiocb.aio_lio_opcode = (file.Mode() == 'w')? LIO_WRITE : LIO_READ; - m_aiocb.aio_buf = (volatile void*)buf; - m_aiocb.aio_fildes = file.Handle(); - m_aiocb.aio_offset = ofs; - m_aiocb.aio_nbytes = size; - struct sigevent* sig = 0; // no notification signal - if(lio_listio(LIO_NOWAIT, (aiocb**)&m_aiocb, 1, sig) != 0) - return LibError_from_errno(); - - return INFO::OK; - } - - LibError Validate() const - { - if(debug_is_pointer_bogus((void*)m_aiocb.aio_buf)) - WARN_RETURN(ERR::_2); - const int opcode = m_aiocb.aio_lio_opcode; - if(opcode != LIO_WRITE && opcode != LIO_READ && opcode != LIO_NOP) - WARN_RETURN(ERR::_3); - // all other aiocb fields have no invariants we could check. - return INFO::OK; - } - - LibError Status() const - { - debug_assert(Validate() == INFO::OK); - - errno = 0; - int ret = aio_error(&m_aiocb); - if(ret == EINPROGRESS) - return INFO::IO_PENDING; - if(ret == 0) - return INFO::IO_COMPLETE; - - return LibError_from_errno(); - } - - LibError WaitUntilComplete(IoBuf& buf, size_t& size) - { - debug_printf("FILE| Wait io=%p\n", this); - debug_assert(Validate() == INFO::OK); - - // wait for transfer to complete. - while(aio_error(&m_aiocb) == EINPROGRESS) - aio_suspend((aiocb**)&m_aiocb, 1, (timespec*)0); // wait indefinitely - - // query number of bytes transferred (-1 if the transfer failed) - const ssize_t bytes_transferred = aio_return(&m_aiocb); - debug_printf("FILE| bytes_transferred=%d aio_nbytes=%u\n", bytes_transferred, m_aiocb.aio_nbytes); - - buf = (IoBuf)m_aiocb.aio_buf; // cast from volatile void* - size = bytes_transferred; - return INFO::OK; - } - -private: - aiocb m_aiocb; -}; - - -//----------------------------------------------------------------------------- - -Io_Posix::Io_Posix() -: impl(new Impl) -{ -} - -Io_Posix::~Io_Posix() -{ -} - -LibError Io_Posix::Issue(File_Posix& file, off_t ofs, IoBuf buf, size_t size) -{ - return impl.get()->Issue(file, ofs, buf, size); -} - -LibError Io_Posix::Status() const -{ - return impl.get()->Status(); -} - -LibError Io_Posix::WaitUntilComplete(IoBuf& buf, size_t& size) -{ - return impl.get()->WaitUntilComplete(buf, size); -} - - -//----------------------------------------------------------------------------- - -LibError io_posix_Synchronous(File_Posix& file, off_t ofs, IoBuf buf, size_t size) -{ - const int fd = file.Handle(); - const bool isWrite = (file.Mode() == 'w'); - - ScopedIoMonitor monitor; - - lseek(fd, ofs, SEEK_SET); - - errno = 0; - void* dst = (void*)buf; - const ssize_t ret = isWrite? write(fd, dst, size) : read(fd, dst, size); - if(ret < 0) - return LibError_from_errno(); - - const size_t totalTransferred = (size_t)ret; - if(totalTransferred != size) - WARN_RETURN(ERR::IO); - - monitor.NotifyOfSuccess(FI_LOWIO, isWrite? FO_WRITE : FO_READ, totalTransferred); - return INFO::OK; -} diff --git a/source/lib/file/posix/io_posix.h b/source/lib/file/posix/io_posix.h deleted file mode 100644 index 7513ed4153..0000000000 --- a/source/lib/file/posix/io_posix.h +++ /dev/null @@ -1,137 +0,0 @@ -/** - * ========================================================================= - * File : io_posix.h - * Project : 0 A.D. - * Description : lightweight POSIX aio wrapper - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_IO_POSIX -#define INCLUDED_IO_POSIX - -#include "lib/file/io/io_buf.h" - -// rationale for using aio instead of mmap: -// - parallelism: instead of just waiting for the transfer to complete, -// other work can be done in the meantime. -// example: decompressing from a Zip archive is practically free because -// we inflate one block while reading the next. -// - throughput: with aio, the drive always has something to do, as opposed -// to read requests triggered by the OS for mapped files, which come -// in smaller chunks. this leads to much higher transfer rates. -// - memory: when used with VFS, aio makes better use of a file cache. -// data is generally compressed in an archive. a cache should store the -// decompressed and decoded (e.g. BGR->RGB) data; mmap would keep the -// original compressed data in memory, which doesn't help. -// we bypass the OS file cache via aio; high-level code will take care -// of caching the decoded file contents. - - -// however, aio is only used internally in file_io. this simplifies the -// interface by preventing the need for multiple implementations (archive, -// vfs, etc.) and avoiding the need for automatic free (since aio won't -// be used directly by client code). -// this also affects streaming of sounds, which is currently indeed -// implemented via aio from archives. however, it doesn't appear to be used -// (even music files are loaded at once) and really ought to be done via -// thread, so we could disable it. - - - -// we don't do any caching or alignment here - this is just a thin AIO wrapper. -// rationale: -// - aligning the transfer isn't possible here since we have no control -// over the buffer, i.e. we cannot read more data than requested. -// instead, this is done in io_manager. -// - transfer sizes here are arbitrary (i.e. not block-aligned); -// that means the cache would have to handle this or also split them up -// into blocks, which would duplicate the abovementioned work. -// - if caching here, we'd also have to handle "forwarding" (i.e. -// desired block has been issued but isn't yet complete). again, it -// is easier to let the synchronous io_manager handle this. -// - finally, io_manager knows more about whether the block should be cached -// (e.g. whether another block request will follow), but we don't -// currently make use of this. -// -// disadvantages: -// - streamed data will always be read from disk. no problem, because -// such data (e.g. music, long speech) is unlikely to be used again soon. -// - prefetching (issuing the next few blocks from archive/file during -// idle time to satisfy potential future IOs) requires extra buffers; -// this is a bit more complicated than just using the cache as storage. - -namespace ERR -{ - const LibError FILE_ACCESS = -110200; - const LibError IO = -110201; - const LibError IO_EOF = -110202; -} - -namespace INFO -{ - const LibError IO_PENDING = +110203; - const LibError IO_COMPLETE = +110204; -} - - -class File_Posix -{ - friend class Io_Posix; - -public: - File_Posix(); - ~File_Posix(); - - LibError Open(const char* pathname, char mode); - void Close(); - - const char* Pathname() const - { - return m_pathname; - } - - char Mode() const - { - return m_mode; - } - - int Handle() const - { - return m_fd; - } - - LibError Validate() const; - -private: - const char* m_pathname; - char m_mode; - int m_fd; -}; - - -class Io_Posix -{ -public: - Io_Posix(); - ~Io_Posix(); - - // no attempt is made at aligning or padding the transfer. - LibError Issue(File_Posix& file, off_t ofs, IoBuf buf, size_t size); - - // check if the IO has completed. - LibError Status() const; - - // passes back the buffer and its size. - LibError WaitUntilComplete(IoBuf& buf, size_t& size); - -private: - class Impl; - boost::shared_ptr impl; -}; - - -extern LibError io_posix_Synchronous(File_Posix& file, off_t ofs, IoBuf buf, size_t size); - -#endif // #ifndef INCLUDED_IO_POSIX diff --git a/source/lib/file/vfs/file_cache.cpp b/source/lib/file/vfs/file_cache.cpp index 630b942d25..167a0f66ea 100644 --- a/source/lib/file/vfs/file_cache.cpp +++ b/source/lib/file/vfs/file_cache.cpp @@ -11,7 +11,8 @@ #include "precompiled.h" #include "file_cache.h" -#include "lib/file/file_stats.h" +#include "lib/file/common/file_stats.h" +#include "lib/file/io/io_internal.h" // sectorSize #include "lib/cache_adt.h" // Cache #include "lib/bits.h" // round_up #include "lib/allocators/allocators.h" @@ -45,30 +46,29 @@ references: "Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson) */ +// shared_ptrs must own a reference to their allocator to ensure it's extant when +// they are freed. it is stored in the shared_ptr deleter. class Allocator; +typedef shared_ptr PAllocator; -class Deleter +class FileCacheDeleter { public: - Deleter(size_t size, Allocator* allocator) + FileCacheDeleter(size_t size, PAllocator allocator) : m_size(size), m_allocator(allocator) { } - // (this must come after Allocator because it calls Deallocate()) - void operator()(const u8* data) const; + // (this uses Allocator and must come after its definition) + void operator()(u8* mem) const; private: size_t m_size; - Allocator* m_allocator; + PAllocator m_allocator; }; -// >= sys_max_sector_size or else waio will have to realign. -// chosen as exactly 1 page: this allows write-protecting file buffers -// without worrying about their (non-page-aligned) borders. -// internal fragmentation is considerable but acceptable. -static const size_t alignment = mem_PageSize(); +// adds statistics and AllocatorChecker to a HeaderlessAllocator class Allocator { public: @@ -77,32 +77,32 @@ public: { } - FileCacheData Allocate(size_t size) + shared_ptr Allocate(size_t size, PAllocator pthis) { - const size_t alignedSize = round_up(size, alignment); + const size_t alignedSize = round_up(size, BLOCK_SIZE); stats_buf_alloc(size, alignedSize); - const u8* data = (const u8*)m_allocator.Allocate(alignedSize); + u8* mem = (u8*)m_allocator.Allocate(alignedSize); #ifndef NDEBUG - m_checker.notify_alloc((void*)data, alignedSize); + m_checker.notify_alloc(mem, alignedSize); #endif - return FileCacheData(data, Deleter(size, this)); + return shared_ptr(mem, FileCacheDeleter(size, pthis)); } - void Deallocate(const u8* data, size_t size) + void Deallocate(u8* mem, size_t size) { - void* const p = (void*)data; - const size_t alignedSize = round_up(size, alignment); + const size_t alignedSize = round_up(size, BLOCK_SIZE); - // (re)allow writes. it would be nice to un-map the buffer, but this is - // not possible because HeaderlessAllocator needs to affix boundary tags. - (void)mprotect(p, size, PROT_READ|PROT_WRITE); + // (re)allow writes in case the buffer was made read-only. it would + // be nice to unmap the buffer, but this is not possible because + // HeaderlessAllocator needs to affix boundary tags. + (void)mprotect(mem, size, PROT_READ|PROT_WRITE); #ifndef NDEBUG - m_checker.notify_free(p, alignedSize); + m_checker.notify_free(mem, alignedSize); #endif - m_allocator.Deallocate(p, alignedSize); + m_allocator.Deallocate(mem, alignedSize); stats_buf_free(); } @@ -116,9 +116,9 @@ private: }; -void Deleter::operator()(const u8* data) const +void FileCacheDeleter::operator()(u8* mem) const { - m_allocator->Deallocate(data, m_size); + m_allocator->Deallocate(mem, m_size); } @@ -135,12 +135,12 @@ void Deleter::operator()(const u8* data) const class FileCache::Impl { public: - Impl(size_t size) - : m_allocator(size) + Impl(size_t maxSize) + : m_allocator(new Allocator(maxSize)) { } - FileCacheData Reserve(size_t size) + shared_ptr Reserve(size_t size) { // (this probably indicates a bug; caching 0-length files would // have no benefit, anyway) @@ -150,13 +150,13 @@ public: // of space in a full cache) for(;;) { - FileCacheData data = m_allocator.Allocate(size); - if(data.get()) + shared_ptr data = m_allocator->Allocate(size, m_allocator); + if(data) return data; // remove least valuable entry from cache (if users are holding // references, the contents won't actually be deallocated) - FileCacheData discardedData; size_t discardedSize; + shared_ptr discardedData; size_t discardedSize; bool removed = m_cache.remove_least_valuable(&discardedData, &discardedSize); // only false if cache is empty, which can't be the case because // allocation failed. @@ -164,40 +164,37 @@ public: } } - void Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost) + void Add(const VfsPath& pathname, shared_ptr data, size_t size, uint cost) { // zero-copy cache => all users share the contents => must not // allow changes. this will be reverted when deallocating. (void)mprotect((void*)data.get(), size, PROT_READ); - m_cache.add(vfsPathname, data, size, cost); + m_cache.add(pathname, data, size, cost); } - bool Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size) + bool Retrieve(const VfsPath& pathname, shared_ptr& data, size_t& size) { // (note: don't call stats_cache because we don't know the file size // in case of a cache miss; doing so is left to the caller.) stats_buf_ref(); - return m_cache.retrieve(vfsPathname, data, &size); + return m_cache.retrieve(pathname, data, &size); } - void Remove(const char* vfsPathname) + void Remove(const VfsPath& pathname) { - m_cache.remove(vfsPathname); + m_cache.remove(pathname); // note: we could check if someone is still holding a reference // to the contents, but that currently doesn't matter. } private: - // HACK: due to vfsPathname, we are assured that strings are equal iff their - // addresses match. however, Cache's STL (hash_)map stupidly assumes that - // const char* keys are "strings". to avoid this behavior, we specify the - // key as const void*. - static Cache m_cache; + typedef Cache< VfsPath, shared_ptr > Cache; + Cache m_cache; - Allocator m_allocator; + PAllocator m_allocator; }; @@ -208,22 +205,22 @@ FileCache::FileCache(size_t size) { } -FileCacheData FileCache::Reserve(size_t size) +shared_ptr FileCache::Reserve(size_t size) { - return impl.get()->Reserve(size); + return impl->Reserve(size); } -void FileCache::Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost) +void FileCache::Add(const VfsPath& pathname, shared_ptr data, size_t size, uint cost) { - impl.get()->Add(vfsPathname, data, size, cost); + impl->Add(pathname, data, size, cost); } -void FileCache::Remove(const char* vfsPathname) +void FileCache::Remove(const VfsPath& pathname) { - impl.get()->Remove(vfsPathname); + impl->Remove(pathname); } -bool FileCache::Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size) +bool FileCache::Retrieve(const VfsPath& pathname, shared_ptr& data, size_t& size) { - return impl.get()->Retrieve(vfsPathname, data, size); + return impl->Retrieve(pathname, data, size); } diff --git a/source/lib/file/vfs/file_cache.h b/source/lib/file/vfs/file_cache.h index 48b65b7942..63ab384a87 100644 --- a/source/lib/file/vfs/file_cache.h +++ b/source/lib/file/vfs/file_cache.h @@ -11,7 +11,7 @@ #ifndef INCLUDED_FILE_CACHE #define INCLUDED_FILE_CACHE -typedef boost::shared_ptr FileCacheData; +#include "vfs_path.h" /** * cache of file contents with support for zero-copy IO. @@ -26,8 +26,8 @@ typedef boost::shared_ptr FileCacheData; * process it, and only then start reading the next file. * * rationale: this is rather similar to BlockCache; however, the differences - * (Reserve's size parameter, Add vs. MarkComplete, different eviction - * policies) are enough to warrant separate implementations. + * (Reserve's size parameter, eviction policies) are enough to warrant + * separate implementations. **/ class FileCache { @@ -47,7 +47,7 @@ public: * * it is expected that this data will be Add()-ed once its IO completes. **/ - FileCacheData Reserve(size_t size); + shared_ptr Reserve(size_t size); /** * Add a file's contents to the cache. @@ -57,11 +57,11 @@ public: * read-only. if need be and no references are currently attached to it, * the memory can also be commandeered by Reserve(). * - * @param vfsPathname key that will be used to Retrieve file contents. + * @param pathname key that will be used to Retrieve file contents. * @param cost is the expected cost of retrieving the file again and * influences how/when it is evicted from the cache. **/ - void Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost = 1); + void Add(const VfsPath& pathname, shared_ptr data, size_t size, uint cost = 1); /** * Remove a file's contents from the cache (if it exists). @@ -72,7 +72,7 @@ public: * this would typically be called in response to a notification that a * file has changed. **/ - void Remove(const char* vfsPathname); + void Remove(const VfsPath& pathname); /** * Attempt to retrieve a file's contents from the file cache. @@ -80,11 +80,11 @@ public: * @return whether the contents were successfully retrieved; if so, * data references the read-only file contents. **/ - bool Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size); + bool Retrieve(const VfsPath& pathname, shared_ptr& data, size_t& size); private: class Impl; - boost::shared_ptr impl; + shared_ptr impl; }; #endif // #ifndef INCLUDED_FILE_CACHE diff --git a/source/lib/file/vfs/real_directory.cpp b/source/lib/file/vfs/real_directory.cpp deleted file mode 100644 index 1f66db3ca1..0000000000 --- a/source/lib/file/vfs/real_directory.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// skip version control directories - this avoids cluttering the -// VFS with hundreds of irrelevant files. -static const char* const svnName = path_Pool()->UniqueCopy(".svn"); - - -//----------------------------------------------------------------------------- -// ArchiveEnumerator -//----------------------------------------------------------------------------- - -class ArchiveEnumerator -{ -public: - ArchiveEnumerator(VfsDirectory* directory) - : m_startDirectory(directory), m_previousPath(0), m_previousDirectory(0) - { - } - -private: - LibError Next(const char* pathname, const ArchiveEntry& archiveEntry) - { - vfs_opt_notify_archived_file(pathname); - - const char* name = path_name_only(pathname); - const char* path = path_dir_only2(pathname); - - // into which directory should the file be inserted? - VfsDirectory* directory = m_previousDirectory; - if(path != m_previousPath) - { - // (we have to create missing subdirectories because archivers - // don't always place directory entries before their files) - VfsFile* file; - TraverseAndCreate(path, m_startDirectory, directory, file); - debug_assert(file == 0); // should not be a file on the path - - m_previousPath = path; - m_previousDirectory = directory; - } - - const FileInfo fileInfo(name, archiveEntry.usize, archiveEntry.mtime); - directory->AddFile(fileInfo, pri, ); - - return INFO::CB_CONTINUE; - } - - static LibError Callback(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData) - { - ArchiveEnumerator* archiveEnumerator = (ArchiveEnumerator*)cbData; - return ArchiveEnumerator->Next(pathname, archiveEntry); - } - - VfsDirectory* m_startDirectory; - - // optimization: looking up each full path is rather slow, so we - // cache the previous directory and use it if the path string - // addresses match. - const char* m_previousPath; - VfsDirectory* m_previousDirectory; -}; - -static LibError AddArchiveFiles(VfsDirectory* directory, PIArchiveReader archiveReader) -{ - ArchiveEnumerator archiveEnumerator(directory); - RETURN_ERR(archiveReader->ReadEntries(&Callback, (uintptr_t)&archiveEnumerator)); -} - - -/** - * search for archives and add their contents into VFS. - * - * @param directory already populated VFS directory into which to - * add the archives' contents. - * - * note: we only support archives in the mount point and not its - * subdirectories. this simplifies Populate() and avoids having to - * check the extension of every single VFS file. - **/ -LibError AddArchives(VfsDirectory* directory) -{ - std::vector files; - directory->GetEntries(&files, 0); - for(size_t i = 0; i < files.size(); i++) - { - FileInfo& fileInfo = files[i]; - const char* extension = path_extension(fileInfo.Name()); - const char* pathname = path_append2(m_path, fileInfo.Name()); - PIArchiveReader archiveReader; - if(strcasecmp(extension, "zip") == 0) - archiveReader = CreateArchiveReader_Zip(pathname); - else - continue; // not a (supported) archive file - - AddArchiveFiles(archiveReader, directory); - m_archiveReaders.push_back(archiveReader); - } - - return INFO::OK; -} - -LibError RealDirectory::CreateFile(const char* name) -{ - const char* pathname = path_append2(pathname, m_path, name); - File_Posix file; - RETURN_ERR(file.Open(pathname, 'w')); - RETURN_ERR(io_Write(file, 0, buf, size)); -} - - - RETURN_ERR(AddArchives(directory, m_archiveReaders)); \ No newline at end of file diff --git a/source/lib/file/vfs/real_directory.h b/source/lib/file/vfs/real_directory.h deleted file mode 100644 index c312fdef2a..0000000000 --- a/source/lib/file/vfs/real_directory.h +++ /dev/null @@ -1,34 +0,0 @@ - -class RealDirectory -{ -public: - RealDirectory(const char* path, unsigned priority, unsigned flags) - : m_path(path), m_priority(priority), m_flags(flags) - { - } - - const char* Path() const - { - return m_path; - } - - unsigned Priority() const - { - return m_priority; - } - - unsigned Flags() const - { - return m_flags; - } - -private: - // note: paths are relative to the root directory, so storing the - // entire path instead of just the portion relative to the mount point - // is not all too wasteful. - const char* m_path; - - unsigned m_priority; - - unsigned m_flags; -}; diff --git a/source/lib/file/vfs/vfs.cpp b/source/lib/file/vfs/vfs.cpp index f01e97f0cf..b7878d4b2f 100644 --- a/source/lib/file/vfs/vfs.cpp +++ b/source/lib/file/vfs/vfs.cpp @@ -12,82 +12,76 @@ #include "vfs.h" #include "lib/path_util.h" -#include "lib/file/file_stats.h" -#include "lib/file/trace.h" +#include "lib/file/common/file_stats.h" +#include "lib/file/common/trace.h" #include "lib/file/archive/archive.h" -#include "lib/file/posix/fs_posix.h" +#include "lib/file/io/io.h" #include "vfs_tree.h" #include "vfs_lookup.h" -#include "vfs_populate.h" #include "file_cache.h" -PIFileProvider provider; - -class Filesystem_VFS::Impl : public IFilesystem +class VFS : public IVFS { public: - Impl() - : m_fileCache(ChooseCacheSize()), m_trace(CreateTrace(4*MiB)) + VFS() + : m_fileCache(ChooseCacheSize()) + , m_trace(CreateTrace(4*MiB)) { } - LibError Mount(const char* vfsPath, const char* path, uint flags /* = 0 */, uint priority /* = 0 */) + virtual LibError Mount(const VfsPath& mountPoint, const char* path, uint flags /* = 0 */, uint priority /* = 0 */) { - // make sure caller didn't forget the required trailing '/'. - debug_assert(path_IsDirectory(vfsPath)); + debug_assert(IsDirectory(mountPoint)); + debug_assert(strcmp(path, ".") != 0); // "./" isn't supported on Windows. + // note: mounting subdirectoryNames is now allowed. - // note: we no longer need to check if mounting a subdirectory - - // the new RealDirectory scheme doesn't care. - - // disallow "." because "./" isn't supported on Windows. - // "./" and "/." are caught by CHECK_PATH. - if(!strcmp(path, ".")) - WARN_RETURN(ERR::PATH_NON_CANONICAL); - - VfsDirectory* directory = vfs_LookupDirectory(vfsPath, &m_rootDirectory, VFS_LOOKUP_CREATE); - directory->Attach(RealDirectory(path, priority, flags)); + VfsDirectory* vfsDirectory; + CHECK_ERR(vfs_Lookup(mountPoint, &m_rootDirectory, vfsDirectory, 0, VFS_LOOKUP_ADD|VFS_LOOKUP_CREATE)); + PRealDirectory realDirectory(new RealDirectory(std::string(path), priority, flags)); + vfsDirectory->Attach(realDirectory); return INFO::OK; } - virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const + virtual LibError GetFileInfo(const VfsPath& pathname, FileInfo* pfileInfo) const { - VfsFile* file = vfs_LookupFile(vfsPathname, &m_rootDirectory); - if(!file) - return ERR::VFS_FILE_NOT_FOUND; // NOWARN - file->GetFileInfo(fileInfo); + VfsDirectory* vfsDirectory; VfsFile* vfsFile; + LibError ret = vfs_Lookup(pathname, &m_rootDirectory, vfsDirectory, &vfsFile); + if(!pfileInfo) // just indicate if the file exists without raising warnings. + return ret; + CHECK_ERR(ret); + vfsFile->GetFileInfo(pfileInfo); return INFO::OK; } - virtual LibError GetDirectoryEntries(const char* vfsPath, FileInfos* files, Directories* subdirectories) const + virtual LibError GetDirectoryEntries(const VfsPath& path, FileInfos* files, DirectoryNames* subdirectoryNames) const { - VfsDirectory* directory = vfs_LookupDirectory(vfsPath, &m_rootDirectory); - if(!directory) - WARN_RETURN(ERR::VFS_DIR_NOT_FOUND); - directory->GetEntries(files, subdirectories); + debug_assert(IsDirectory(path)); + VfsDirectory* vfsDirectory; + CHECK_ERR(vfs_Lookup(path, &m_rootDirectory, vfsDirectory, 0)); + vfsDirectory->GetEntries(files, subdirectoryNames); return INFO::OK; } // note: only allowing either reads or writes simplifies file cache // coherency (need only invalidate when closing a FILE_WRITE file). - LibError CreateFile(const char* vfsPathname, const u8* buf, size_t size) + virtual LibError CreateFile(const VfsPath& pathname, shared_ptr fileContents, size_t size) { - VfsDirectory* directory = vfs_LookupDirectory(vfsPathname, &m_rootDirectory); - if(!directory) - WARN_RETURN(ERR::VFS_DIR_NOT_FOUND); - const char* name = path_name_only(vfsPathname); + VfsDirectory* vfsDirectory; + CHECK_ERR(vfs_Lookup(pathname, &m_rootDirectory, vfsDirectory, 0, VFS_LOOKUP_ADD|VFS_LOOKUP_CREATE)); + + PRealDirectory realDirectory = vfsDirectory->AssociatedDirectory(); + const std::string& name = pathname.leaf(); + RETURN_ERR(realDirectory->Store(name, fileContents, size)); + + const VfsFile file(FileInfo(name, (off_t)size, time(0)), realDirectory->Priority(), realDirectory); + vfsDirectory->AddFile(file); - const RealDirectory& realDirectory = directory->AttachedDirectories().back(); - const char* location = realDirectory.Path(); - const VfsFile file(FileInfo(name, (off_t)size, time(0)), realDirectory.Priority(), provider, location); - file.Store(buf, size); - directory->AddFile(file); - // wipe out any cached blocks. this is necessary to cover the (rare) case // of file cache contents predating the file write. - m_fileCache.Remove(vfsPathname); + m_fileCache.Remove(pathname); - m_trace.get()->NotifyStore(vfsPathname, size); + m_trace->NotifyStore(pathname.string().c_str(), size); return INFO::OK; } @@ -100,58 +94,60 @@ public: // the callback mechanism is useful for user progress notification or // processing data while waiting for the next I/O to complete // (quasi-parallel, without the complexity of threads). - LibError LoadFile(const char* vfsPathname, FileContents& contents, size_t& size) + virtual LibError LoadFile(const VfsPath& pathname, shared_ptr& fileContents, size_t& size) { - vfsPathname = path_Pool()->UniqueCopy(vfsPathname); - debug_printf("VFS| load %s\n", vfsPathname); - - const bool isCacheHit = m_fileCache.Retrieve(vfsPathname, contents, size); + const bool isCacheHit = m_fileCache.Retrieve(pathname, fileContents, size); if(!isCacheHit) { - VfsFile* file = vfs_LookupFile(vfsPathname, &m_rootDirectory); - if(!file) - WARN_RETURN(ERR::VFS_FILE_NOT_FOUND); - contents = m_fileCache.Reserve(file->Size()); - RETURN_ERR(file->Load((u8*)contents.get())); - m_fileCache.Add(vfsPathname, contents, size); + VfsDirectory* vfsDirectory; VfsFile* vfsFile; + CHECK_ERR(vfs_Lookup(pathname, &m_rootDirectory, vfsDirectory, &vfsFile)); + + size = vfsFile->Size(); + if(size > ChooseCacheSize()) + { + fileContents = io_Allocate(size); + RETURN_ERR(vfsFile->Load(fileContents)); + } + else + { + fileContents = m_fileCache.Reserve(size); + RETURN_ERR(vfsFile->Load(fileContents)); + m_fileCache.Add(pathname, fileContents, size); + } } stats_io_user_request(size); - stats_cache(isCacheHit? CR_HIT : CR_MISS, size, vfsPathname); - m_trace.get()->NotifyLoad(vfsPathname, size); + stats_cache(isCacheHit? CR_HIT : CR_MISS, size); + m_trace->NotifyLoad(pathname.string().c_str(), size); return INFO::OK; } - void RefreshFileInfo(const char* pathname) - { - //VfsFile* file = LookupFile(vfsPathname, &m_rootDirectory); - } - - // "backs off of" all archives - closes their files and allows them to - // be rewritten or deleted (required by archive builder). - // must call mount_rebuild when done with the rewrite/deletes, - // because this call leaves the VFS in limbo!! - void ReleaseArchives() - { - } - // rebuild the VFS, i.e. re-mount everything. open files are not affected. // necessary after loose files or directories change, so that the VFS // "notices" the changes and updates file locations. res calls this after // dir_watch reports changes; can also be called from the console after a // rebuild command. there is no provision for updating single VFS dirs - // it's not worth the trouble. - void Clear() + virtual void Clear() { m_rootDirectory.ClearR(); } - void Display() + virtual void Display() const { m_rootDirectory.DisplayR(0); } + virtual LibError GetRealPath(const VfsPath& pathname, char* realPathname) + { + VfsDirectory* vfsDirectory; + CHECK_ERR(vfs_Lookup(pathname, &m_rootDirectory, vfsDirectory, 0)); + PRealDirectory realDirectory = vfsDirectory->AssociatedDirectory(); + const std::string& name = pathname.leaf(); + path_append(realPathname, realDirectory->GetPath().string().c_str(), name.c_str()); + return INFO::OK; + } private: static size_t ChooseCacheSize() @@ -159,57 +155,14 @@ private: return 96*MiB; } - VfsDirectory m_rootDirectory; + mutable VfsDirectory m_rootDirectory; FileCache m_fileCache; PITrace m_trace; }; //----------------------------------------------------------------------------- -Filesystem_VFS::Filesystem_VFS(void* trace) +PIVFS CreateVfs() { -} - -/*virtual*/ Filesystem_VFS::~Filesystem_VFS() -{ -} - -/*virtual*/ LibError Filesystem_VFS::GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const -{ - return impl.get()->GetFileInfo(vfsPathname, fileInfo); -} - -/*virtual*/ LibError Filesystem_VFS::GetDirectoryEntries(const char* vfsPath, FileInfos* files, Directories* subdirectories) const -{ - return impl.get()->GetDirectoryEntries(vfsPath, files, subdirectories); -} - -LibError Filesystem_VFS::CreateFile(const char* vfsPathname, const u8* data, size_t size) -{ - return impl.get()->CreateFile(vfsPathname, data, size); -} - -LibError Filesystem_VFS::LoadFile(const char* vfsPathname, FileContents& contents, size_t& size) -{ - return impl.get()->LoadFile(vfsPathname, contents, size); -} - -LibError Filesystem_VFS::Mount(const char* vfsPath, const char* path, uint flags, uint priority) -{ - return impl.get()->Mount(vfsPath, path, flags, priority); -} - -void Filesystem_VFS::RefreshFileInfo(const char* pathname) -{ - impl.get()->RefreshFileInfo(pathname); -} - -void Filesystem_VFS::Display() const -{ - impl.get()->Display(); -} - -void Filesystem_VFS::Clear() -{ - impl.get()->Clear(); + return PIVFS(new VFS); } diff --git a/source/lib/file/vfs/vfs.h b/source/lib/file/vfs/vfs.h index 1e47eea210..911489250c 100644 --- a/source/lib/file/vfs/vfs.h +++ b/source/lib/file/vfs/vfs.h @@ -12,7 +12,25 @@ #ifndef INCLUDED_VFS #define INCLUDED_VFS -#include "lib/file/filesystem.h" +#include "lib/file/file_system.h" // FileInfo +#include "lib/file/vfs/vfs_path.h" + +// VFS paths are of the form: "(dir/)*file?" +// in English: '/' as path separator; trailing '/' required for dir names; +// no leading '/', since "" is the root dir. + +// there is no restriction on path length; when dimensioning character +// arrays, prefer PATH_MAX. + +// pathnames are case-insensitive. +// implementation: +// when mounting, we get the exact filenames as reported by the OS; +// we allow open requests with mixed case to match those, +// but still use the correct case when passing to other libraries +// (e.g. the actual open() syscall). +// rationale: +// necessary, because some exporters output .EXT uppercase extensions +// and it's unreasonable to expect that users will always get it right. namespace ERR { @@ -27,43 +45,41 @@ enum VfsMountFlags // all real directories mounted during this operation will be watched // for changes. this flag is provided to avoid watches in output-only // directories, e.g. screenshots/ (only causes unnecessary overhead). - VFS_WATCH = 1, + VFS_MOUNT_WATCH = 1, // anything mounted from here should be added to archive when // building via vfs_optimizer. - VFS_ARCHIVABLE = 2 + VFS_MOUNT_ARCHIVABLE = 2 }; - -typedef boost::shared_ptr FileContents; - -class Filesystem_VFS : public IFilesystem +struct IVFS { -public: - Filesystem_VFS(void* trace); - ~Filesystem_VFS(); - /** * mount a directory into the VFS. * - * @param vfsPath mount point (created if it does not already exist) + * @param mountPoint (will be created if it does not already exist) * @param path real directory path * * if files are encountered that already exist in the VFS (sub)directories, * the most recent / highest priority/precedence version is preferred. * - * the contents of archives in this directory (but not its subdirectories!) - * are added as well; they are processed in alphabetical order. + * if files with archive extensions are seen, their contents are added + * as well. **/ - LibError Mount(const char* vfsPath, const char* path, uint flags = 0, uint priority = 0); + virtual LibError Mount(const VfsPath& mountPoint, const char* path, uint flags = 0, uint priority = 0) = 0; + + virtual LibError GetFileInfo(const VfsPath& pathname, FileInfo* pfileInfo) const = 0; + + // note: this interface avoids having to lock a directory while an + // iterator is extant. + // (don't split this into 2 functions because POSIX can't implement + // that efficiently) + virtual LibError GetDirectoryEntries(const VfsPath& path, FileInfos* files, DirectoryNames* subdirectoryNames) const = 0; - // (see IFilesystem) - virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const; - virtual LibError GetDirectoryEntries(const char* vfsPath, FileInfos* files, Directories* subdirectories) const; // note: only allowing either reads or writes simplifies file cache // coherency (need only invalidate when closing a FILE_WRITE file). - LibError CreateFile(const char* vfsPathname, const u8* data, size_t size); + virtual LibError CreateFile(const VfsPath& pathname, shared_ptr fileContents, size_t size) = 0; // read the entire file. // return number of bytes transferred (see above), or a negative error code. @@ -74,16 +90,15 @@ public: // the callback mechanism is useful for user progress notification or // processing data while waiting for the next I/O to complete // (quasi-parallel, without the complexity of threads). - LibError LoadFile(const char* vfsPathname, FileContents& contents, size_t& size); + virtual LibError LoadFile(const VfsPath& pathname, shared_ptr& fileContents, size_t& size) = 0; - void RefreshFileInfo(const char* pathname); + virtual void Display() const = 0; + virtual void Clear() = 0; - void Display() const; - void Clear(); - -private: - class Impl; - boost::shared_ptr impl; + virtual LibError GetRealPath(const VfsPath& pathname, char* path) = 0; }; +typedef shared_ptr PIVFS; +LIB_API PIVFS CreateVfs(); + #endif // #ifndef INCLUDED_VFS diff --git a/source/lib/file/vfs/vfs_lookup.cpp b/source/lib/file/vfs/vfs_lookup.cpp index 4992a37875..8307023d13 100644 --- a/source/lib/file/vfs/vfs_lookup.cpp +++ b/source/lib/file/vfs/vfs_lookup.cpp @@ -2,7 +2,7 @@ * ========================================================================= * File : vfs_lookup.cpp * Project : 0 A.D. - * Description : + * Description : look up directories/files by traversing path components. * ========================================================================= */ @@ -12,104 +12,182 @@ #include "vfs_lookup.h" #include "lib/path_util.h" // path_foreach_component +#include "lib/file/file_system_posix.h" +#include "lib/file/archive/archive_zip.h" #include "vfs_tree.h" -#include "vfs_populate.h" #include "vfs.h" // error codes +#include "lib/timer.h" +TIMER_ADD_CLIENT(tc_lookup); -class PathResolver + +//----------------------------------------------------------------------------- + +static FileSystem_Posix s_fileSystemPosix; +static std::vector s_looseFiles; +static size_t s_numArchivedFiles; + +// helper class that allows breaking up the logic into sub-functions without +// always having to pass directory/realDirectory as parameters. +class PopulateHelper : boost::noncopyable { public: - PathResolver(VfsDirectory* startDirectory, uint flags = 0) - : m_flags(flags), m_currentDirectory(startDirectory) + PopulateHelper(VfsDirectory* vfsDirectory, PRealDirectory realDirectory) + : m_vfsDirectory(vfsDirectory), m_realDirectory(realDirectory) { } - static LibError Callback(const char* component, bool isDirectory, uintptr_t cbData) + LibError AddEntries() const { - PathResolver* pathResolver = (PathResolver*)cbData; - return pathResolver->Next(component, isDirectory); - } - - LibError Next(const char* component, bool isDirectory) const - { - if((m_flags & VFS_LOOKUP_NO_POPULATE ) == 0) - RETURN_ERR(vfs_Populate(m_currentDirectory)); - - // vfsLookup only sends us pathnames, so all components are directories - debug_assert(isDirectory); - - if((m_flags & VFS_LOOKUP_CREATE) != 0) - m_currentDirectory->AddSubdirectory(component); - - m_currentDirectory = m_currentDirectory->GetSubdirectory(component); - if(!m_currentDirectory) - WARN_RETURN(ERR::VFS_DIR_NOT_FOUND); - - return INFO::CB_CONTINUE; - } - - VfsDirectory* Directory() const - { - return m_currentDirectory; + FileInfos files; files.reserve(100); + DirectoryNames subdirectoryNames; subdirectoryNames.reserve(20); + RETURN_ERR(s_fileSystemPosix.GetDirectoryEntries(m_realDirectory->GetPath(), &files, &subdirectoryNames)); + RETURN_ERR(AddFiles(files)); + AddSubdirectories(subdirectoryNames); + return INFO::OK; } private: - unsigned m_flags; - mutable VfsDirectory* m_currentDirectory; + void AddFile(const FileInfo& fileInfo) const + { + const VfsFile file(fileInfo, m_realDirectory->Priority(), m_realDirectory); + const VfsFile* pfile = m_vfsDirectory->AddFile(file); + + // notify archive builder that this file could be archived but + // currently isn't; if there are too many of these, archive will + // be rebuilt. + // note: check if archivable to exclude stuff like screenshots + // from counting towards the threshold. + if(m_realDirectory->Flags() & VFS_MOUNT_ARCHIVABLE) + s_looseFiles.push_back(pfile); + } + + static void AddArchiveFile(const VfsPath& pathname, const FileInfo& fileInfo, PIArchiveFile archiveFile, uintptr_t cbData) + { + PopulateHelper* this_ = (PopulateHelper*)cbData; + + // (we have to create missing subdirectoryNames because archivers + // don't always place directory entries before their files) + const unsigned flags = VFS_LOOKUP_ADD; + VfsDirectory* vfsDirectory; + WARN_ERR(vfs_Lookup(pathname, this_->m_vfsDirectory, vfsDirectory, 0, flags)); + const VfsFile vfsFile(fileInfo, this_->m_realDirectory->Priority(), archiveFile); + vfsDirectory->AddFile(vfsFile); + s_numArchivedFiles++; + } + + LibError AddFiles(const FileInfos& files) const + { + const Path path(m_realDirectory->GetPath()); + + for(size_t i = 0; i < files.size(); i++) + { + const std::string& name = files[i].Name(); + + const char* extension = path_extension(name.c_str()); + if(strcasecmp(extension, "zip") == 0) + { + PIArchiveReader archiveReader = CreateArchiveReader_Zip(path/name); + RETURN_ERR(archiveReader->ReadEntries(AddArchiveFile, (uintptr_t)this)); + } + else // regular (non-archive) file + AddFile(files[i]); + } + + return INFO::OK; + } + + void AddSubdirectories(const DirectoryNames& subdirectoryNames) const + { + for(size_t i = 0; i < subdirectoryNames.size(); i++) + { + // skip version control directories - this avoids cluttering the + // VFS with hundreds of irrelevant files. + if(strcasecmp(subdirectoryNames[i].c_str(), ".svn") == 0) + continue; + + VfsDirectory* subdirectory = m_vfsDirectory->AddSubdirectory(subdirectoryNames[i]); + subdirectory->Attach(CreateRealSubdirectory(m_realDirectory, subdirectoryNames[i])); + } + } + + VfsDirectory* const m_vfsDirectory; + PRealDirectory m_realDirectory; }; -LibError vfs_Lookup(const char* vfsPathname, VfsDirectory* startDirectory, VfsDirectory*& directory, VfsFile*& file, unsigned flags) +static LibError Populate(VfsDirectory* vfsDirectory) { - const char* vfsPath; const char* name; - path_split(vfsPathname, &vfsPath, &name); + if(!vfsDirectory->ShouldPopulate()) + return INFO::OK; - // optimization: looking up each full path is rather slow, so we - // cache the previous directory and use it if the path string - // addresses match. - static const char* vfsPreviousPath; - static VfsDirectory* previousDirectory; - if(vfsPath == vfsPreviousPath) - directory = previousDirectory; - else - { - PathResolver pathResolver(startDirectory, flags); - RETURN_ERR(path_foreach_component(vfsPathname, PathResolver::Callback, (uintptr_t)&pathResolver)); - directory = pathResolver.Directory(); - } - previousDirectory = directory; + PRealDirectory realDirectory = vfsDirectory->AssociatedDirectory(); - if(name[0] == '\0') - file = 0; - else - { - file = directory->GetFile(name); - if(!file) - WARN_RETURN(ERR::VFS_FILE_NOT_FOUND); - } + if(realDirectory->Flags() & VFS_MOUNT_WATCH) + realDirectory->Watch(); + + PopulateHelper helper(vfsDirectory, realDirectory); + RETURN_ERR(helper.AddEntries()); return INFO::OK; } -VfsFile* vfs_LookupFile(const char* vfsPathname, VfsDirectory* startDirectory, unsigned flags) +//----------------------------------------------------------------------------- + +LibError vfs_Lookup(const VfsPath& pathname, VfsDirectory* vfsStartDirectory, VfsDirectory*& vfsDirectory, VfsFile** pvfsFile, unsigned flags) { - debug_assert(!path_IsDirectory(vfsPathname)); +TIMER_ACCRUE(tc_lookup); - VfsDirectory* directory; VfsFile* file; - if(vfs_Lookup(vfsPathname, startDirectory, directory, file, flags) != INFO::OK) - return 0; - return file; -} - - -VfsDirectory* vfs_LookupDirectory(const char* vfsPath, VfsDirectory* startDirectory, unsigned flags) -{ - debug_assert(path_IsDirectory(vfsPath)); - - VfsDirectory* directory; VfsFile* file; - if(vfs_Lookup(vfsPath, startDirectory, directory, file, flags) != INFO::OK) - return 0; - return directory; + vfsDirectory = vfsStartDirectory; + if(pvfsFile) + *pvfsFile = 0; + RETURN_ERR(Populate(vfsDirectory)); + + if(pathname.empty()) // early out for root directory + return INFO::OK; // (prevents iterator error below) + + VfsPath currentPath; // only used if flags & VFS_LOOKUP_CREATE + VfsPath::iterator it; + + // for each directory component: + for(it = pathname.begin(); it != --pathname.end(); ++it) + { + const std::string& subdirectoryName = *it; + + VfsDirectory* vfsSubdirectory = vfsDirectory->GetSubdirectory(subdirectoryName); + if(!vfsSubdirectory) + { + if(!(flags & VFS_LOOKUP_ADD)) + return ERR::VFS_DIR_NOT_FOUND; // NOWARN + + vfsSubdirectory = vfsDirectory->AddSubdirectory(subdirectoryName); + + if(flags & VFS_LOOKUP_CREATE) + { + currentPath /= subdirectoryName; + +#if 0 + (void)mkdir(currentPath.external_directory_string().c_str(), S_IRWXO|S_IRWXU|S_IRWXG); + + PRealDirectory realDirectory(new RealDirectory(currentPath.string(), 0, 0)); + vfsSubdirectory->Attach(realDirectory); +#endif + } + } + + vfsDirectory = vfsSubdirectory; + RETURN_ERR(Populate(vfsDirectory)); + } + + if(pvfsFile) + { + const std::string& filename = *it; + debug_assert(filename != "."); // asked for file but specified directory path + *pvfsFile = vfsDirectory->GetFile(filename); + if(!*pvfsFile) + return ERR::VFS_FILE_NOT_FOUND; // NOWARN + } + + return INFO::OK; } diff --git a/source/lib/file/vfs/vfs_lookup.h b/source/lib/file/vfs/vfs_lookup.h index 31e8256566..6fd9209f9a 100644 --- a/source/lib/file/vfs/vfs_lookup.h +++ b/source/lib/file/vfs/vfs_lookup.h @@ -2,7 +2,7 @@ * ========================================================================= * File : vfs_lookup.h * Project : 0 A.D. - * Description : + * Description : look up directories/files by traversing path components. * ========================================================================= */ @@ -11,50 +11,37 @@ #ifndef INCLUDED_VFS_LOOKUP #define INCLUDED_VFS_LOOKUP -// VFS paths are of the form: "(dir/)*file?" -// in English: '/' as path separator; trailing '/' required for dir names; -// no leading '/', since "" is the root dir. - -// there is no restriction on path length; when dimensioning character -// arrays, prefer PATH_MAX. - -// pathnames are case-insensitive. -// implementation: -// when mounting, we get the exact filenames as reported by the OS; -// we allow open requests with mixed case to match those, -// but still use the correct case when passing to other libraries -// (e.g. the actual open() syscall). -// rationale: -// necessary, because some exporters output .EXT uppercase extensions -// and it's unreasonable to expect that users will always get it right. - +#include "vfs_path.h" class VfsFile; class VfsDirectory; +// note: VfsDirectory pointers are non-const because they may be +// populated during the lookup. + enum VfsLookupFlags { - // when encountering subdirectory components in the path(name) that - // don't (yet) exist, add them. - VFS_LOOKUP_CREATE = 1, + // add (if they do not already exist) subdirectory components + // encountered in the path[name]. + VFS_LOOKUP_ADD = 1, - VFS_LOOKUP_NO_POPULATE = 2 + // create a real directory + VFS_LOOKUP_CREATE = 2 }; -extern LibError vfs_Lookup(const char* vfsPathname, VfsDirectory* startDirectory, VfsDirectory*& directory, VfsFile*& file, unsigned flags = 0); - /** - * Find a file in the VFS by traversing pathname components. + * Resolve a pathname. * - * @param vfsPath must not end with a slash character. - **/ -extern VfsFile* vfs_LookupFile(const char* vfsPathname, const VfsDirectory* startDirectory, unsigned flags = 0); - -/** - * Find a directory in the VFS by traversing path components. + * @param pathname + * @param vfsStartDirectory + * @param directory is set to the last directory component that is encountered. + * @param file is set to 0 if there is no name component, otherwise the + * corresponding file. + * @param flags see VfsLookupFlags. + * @return LibError (INFO::OK if all components in pathname exist). * - * @param vfsPath must end with a slash character. + * to allow noiseless file-existence queries, this does not raise warnings. **/ -extern VfsDirectory* vfs_LookupDirectory(const char* vfsPath, const VfsDirectory* startDirectory, unsigned flags = 0); +extern LibError vfs_Lookup(const VfsPath& pathname, VfsDirectory* vfsStartDirectory, VfsDirectory*& vfsDirectory, VfsFile** vfsFile, unsigned flags = 0); #endif // #ifndef INCLUDED_VFS_LOOKUP diff --git a/source/lib/file/vfs/vfs_path.cpp b/source/lib/file/vfs/vfs_path.cpp new file mode 100644 index 0000000000..f2ecdb0d7f --- /dev/null +++ b/source/lib/file/vfs/vfs_path.cpp @@ -0,0 +1,21 @@ +#include "precompiled.h" +#include "vfs_path.h" + +bool IsDirectory(const VfsPath& pathname) +{ + return pathname.empty() || pathname.leaf() == "."; +} + +std::string Basename(const VfsPath& pathname) +{ + const std::string name(pathname.leaf()); + const size_t pos = name.rfind('.'); + return name.substr(0, pos); +} + +std::string Extension(const VfsPath& pathname) +{ + const std::string name(pathname.leaf()); + const size_t pos = name.rfind('.'); + return (pos == std::string::npos)? std::string() : name.substr(pos); +} diff --git a/source/lib/file/vfs/vfs_path.h b/source/lib/file/vfs/vfs_path.h new file mode 100644 index 0000000000..520609fd36 --- /dev/null +++ b/source/lib/file/vfs/vfs_path.h @@ -0,0 +1,29 @@ +#ifndef INCLUDED_VFS_PATH +#define INCLUDED_VFS_PATH + +struct VfsPathTraits; +typedef fs::basic_path VfsPath; + +typedef std::vector VfsPaths; + +struct VfsPathTraits +{ + typedef std::string internal_string_type; + typedef std::string external_string_type; + + static external_string_type to_external(const VfsPath&, const internal_string_type& src) + { + return src; + } + + static internal_string_type to_internal(const external_string_type& src) + { + return src; + } +}; + +extern bool IsDirectory(const VfsPath& pathname); +extern std::string Basename(const VfsPath& pathname); +extern std::string Extension(const VfsPath& pathname); + +#endif // #ifndef INCLUDED_VFS_PATH diff --git a/source/lib/file/vfs/vfs_populate.cpp b/source/lib/file/vfs/vfs_populate.cpp deleted file mode 100644 index 6b7bf115d9..0000000000 --- a/source/lib/file/vfs/vfs_populate.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/** - * ========================================================================= - * File : vfs_populate.cpp - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "vfs_populate.h" - -#include "lib/path_util.h" -#include "lib/file/path.h" -#include "lib/file/archive/archive_zip.h" -#include "lib/file/posix/fs_posix.h" -#include "vfs_tree.h" -#include "vfs_lookup.h" -#include "vfs.h" // mount flags - -typedef void* PIWatch; - - -static Filesystem_Posix s_fsPosix; - -// since directories are never removed except when rebuilding the VFS, -// we can store the IWatch references here. -static std::vector s_watches; - -static std::vector s_archiveReaders; - -static std::vector s_looseFiles; -static size_t s_numArchivedFiles; - - -//----------------------------------------------------------------------------- - -// helper class that allows breaking up the logic into sub-functions without -// always having to pass directory/realDirectory as parameters. -class PopulateHelper : boost::noncopyable -{ -public: - PopulateHelper(VfsDirectory* directory, const RealDirectory& realDirectory) - : m_directory(directory) - , m_path(realDirectory.Path()), m_priority(realDirectory.Priority()), m_flags(realDirectory.Flags()) - { - } - - LibError AddEntries() const - { - FileInfos files; files.reserve(100); - Directories subdirectories; subdirectories.reserve(20); - RETURN_ERR(s_fsPosix.GetDirectoryEntries(m_path, &files, &subdirectories)); - RETURN_ERR(AddFiles(files)); - AddSubdirectories(subdirectories); - return INFO::OK; - } - -private: - void AddFile(const FileInfo& fileInfo) const - { - static PIFileProvider provider(new FileProvider_Posix); - const VfsFile* file = m_directory->AddFile(VfsFile(fileInfo, m_priority, provider, m_path)); - - // notify archive builder that this file could be archived but - // currently isn't; if there are too many of these, archive will - // be rebuilt. - // note: check if archivable to exclude stuff like screenshots - // from counting towards the threshold. - if(m_flags & VFS_ARCHIVABLE) - s_looseFiles.push_back(file); - } - - static void AddArchiveFile(const char* pathname, const FileInfo& fileInfo, const ArchiveEntry* archiveEntry, uintptr_t cbData) - { - PopulateHelper* this_ = (PopulateHelper*)cbData; - - // (we have to create missing subdirectories because archivers - // don't always place directory entries before their files) - const unsigned flags = VFS_LOOKUP_CREATE|VFS_LOOKUP_NO_POPULATE; - VfsDirectory* directory = vfs_LookupDirectory(pathname, this_->m_directory, flags); - debug_assert(directory); - - directory->AddFile(VfsFile(fileInfo, this_->m_priority, this_->m_archiveReader, archiveEntry)); - s_numArchivedFiles++; - } - - LibError AddFiles(const FileInfos& files) const - { - PathPackage pp; - path_package_set_dir(&pp, m_path); - - for(size_t i = 0; i < files.size(); i++) - { - const char* name = files[i].Name(); - path_package_append_file(&pp, name); - - PIArchiveReader archiveReader; - const char* extension = path_extension(name); - if(strcasecmp(extension, "zip") == 0) - archiveReader = CreateArchiveReader_Zip(pp.path); - else // not a (supported) archive file - AddFile(files[i]); - - RETURN_ERR(archiveReader->ReadEntries(AddArchiveFile, (uintptr_t)this)); - s_archiveReaders.push_back(archiveReader); - } - - return INFO::OK; - } - - void AddSubdirectories(const Directories& subdirectories) const - { - for(size_t i = 0; i < subdirectories.size(); i++) - { - const char* name = subdirectories[i]; - - // skip version control directories - this avoids cluttering the - // VFS with hundreds of irrelevant files. - static const char* const svnName = path_Pool()->UniqueCopy(".svn"); - if(name == svnName) - continue; - - VfsDirectory* subdirectory = m_directory->AddSubdirectory(name); - - const char* subdirectoryPath = path_append2(m_path, name); - const RealDirectory realSubdirectory(subdirectoryPath, m_priority, m_flags); - subdirectory->Attach(realSubdirectory); - } - } - - VfsDirectory* const m_directory; - - const char* m_path; - const unsigned m_priority; - const unsigned m_flags; - - // used when populating archives - PIArchiveReader m_archiveReader; -}; - - -//----------------------------------------------------------------------------- - -LibError vfs_Populate(VfsDirectory* directory) -{ - const std::vector& realDirectories = directory->AttachedDirectories(); - for(size_t i = 0; i < realDirectories.size(); i++) - { - const RealDirectory& realDirectory = realDirectories[i]; - - // note: we need to do this in each directory because some watch APIs - // (e.g. FAM) cannot register entire directory trees with one call. - if(realDirectory.Flags() & VFS_WATCH) - { - char osPath[PATH_MAX]; - (void)path_MakeAbsolute(realDirectory.Path(), osPath); -// s_watches.push_back(CreateWatch(osPath)); - } - - PopulateHelper helper(directory, realDirectory); - RETURN_ERR(helper.AddEntries()); - } - - return INFO::OK; -} diff --git a/source/lib/file/vfs/vfs_populate.h b/source/lib/file/vfs/vfs_populate.h deleted file mode 100644 index a55b16a27e..0000000000 --- a/source/lib/file/vfs/vfs_populate.h +++ /dev/null @@ -1,3 +0,0 @@ -class VfsDirectory; - -extern LibError vfs_Populate(VfsDirectory* directory); diff --git a/source/lib/file/vfs/vfs_tree.cpp b/source/lib/file/vfs/vfs_tree.cpp index a98cd96703..c681c27c8b 100644 --- a/source/lib/file/vfs/vfs_tree.cpp +++ b/source/lib/file/vfs/vfs_tree.cpp @@ -11,31 +11,31 @@ #include "precompiled.h" #include "vfs_tree.h" -#include "lib/file/file_stats.h" +#include "lib/file/common/file_stats.h" +#include "lib/sysdep/cpu.h" //----------------------------------------------------------------------------- -VfsFile::VfsFile(const FileInfo& fileInfo, unsigned priority, PIFileProvider provider, const void* location) -: m_fileInfo(fileInfo), m_priority(priority) -, m_provider(provider), m_location(location) +VfsFile::VfsFile(const FileInfo& fileInfo, unsigned priority, PIFileLoader loader) + : m_fileInfo(fileInfo), m_priority(priority), m_loader(loader) { } -bool VfsFile::IsSupersededBy(const VfsFile& file) const +bool VfsFile::IsSupersededBy(const VfsFile& vfsFile) const { // 1) priority is lower => no. - if(file.m_priority < m_priority) + if(vfsFile.m_priority < m_priority) return false; // 2) timestamp is older => no. // (note: we need to account for FAT's 2 sec. resolution) - if(difftime(file.m_fileInfo.MTime(), m_fileInfo.MTime()) < -2.0) + if(difftime(vfsFile.m_fileInfo.MTime(), m_fileInfo.MTime()) < -2.0) return false; // 3) provider is less efficient => no. - if(file.m_provider.get()->Precedence() < m_provider.get()->Precedence()) + if(vfsFile.m_loader->Precedence() < m_loader->Precedence()) return false; return true; @@ -51,149 +51,154 @@ void VfsFile::GenerateDescription(char* text, size_t maxChars) const // build format string (set width of name field so that everything // lines up correctly) const char* fmt = "(%c; %6d; %s)\n"; - sprintf_s(text, maxChars, fmt, m_provider.get()->LocationCode(), m_fileInfo.Size(), timestamp); + sprintf_s(text, maxChars, fmt, m_loader->LocationCode(), m_fileInfo.Size(), timestamp); } -LibError VfsFile::Store(const u8* fileContents, size_t size) const +LibError VfsFile::Load(shared_ptr buf) const { - RETURN_ERR(m_provider.get()->Store(m_fileInfo.Name(), m_location, fileContents, size)); - - // update size and mtime - m_fileInfo = FileInfo(m_fileInfo.Name(), (off_t)size, time(0)); - - return INFO::OK; -} - - -LibError VfsFile::Load(u8* fileContents) const -{ - return m_provider.get()->Load(m_fileInfo.Name(), m_location, fileContents, m_fileInfo.Size()); + return m_loader->Load(m_fileInfo.Name(), buf, m_fileInfo.Size()); } //----------------------------------------------------------------------------- VfsDirectory::VfsDirectory() +: m_shouldPopulate(0) { } -VfsFile* VfsDirectory::AddFile(const VfsFile& file) +VfsFile* VfsDirectory::AddFile(const VfsFile& vfsFile) { - std::pair value = std::make_pair(file.Name(), file); - std::pair ret = m_files.insert(value); + std::pair value = std::make_pair(vfsFile.Name(), vfsFile); + std::pair ret = m_vfsFiles.insert(value); if(!ret.second) // already existed { - VfsFile& previousFile = ret.first->second; - const VfsFile& newFile = value.second; - if(previousFile.IsSupersededBy(newFile)) - previousFile = newFile; + VfsFile& vfsPreviousFile = ret.first->second; + const VfsFile& vfsNewFile = value.second; + if(vfsPreviousFile.IsSupersededBy(vfsNewFile)) + vfsPreviousFile = vfsNewFile; } else - stats_vfs_file_add(file.Size()); + stats_vfs_file_add(vfsFile.Size()); return &(*ret.first).second; } // rationale: passing in a pre-constructed VfsDirectory and copying that into -// our map would be less efficient than this approach. -VfsDirectory* VfsDirectory::AddSubdirectory(const char* name) +// our map would be slower and less convenient for the caller. +VfsDirectory* VfsDirectory::AddSubdirectory(const std::string& name) { - std::pair value = std::make_pair(name, VfsDirectory()); - std::pair ret = m_subdirectories.insert(value); + std::pair value = std::make_pair(name, VfsDirectory()); + std::pair ret = m_vfsSubdirectories.insert(value); return &(*ret.first).second; } -VfsFile* VfsDirectory::GetFile(const char* name) +VfsFile* VfsDirectory::GetFile(const std::string& name) { - Files::iterator it = m_files.find(name); - if(it == m_files.end()) + VfsFiles::iterator it = m_vfsFiles.find(name); + if(it == m_vfsFiles.end()) return 0; return &it->second; } -VfsDirectory* VfsDirectory::GetSubdirectory(const char* name) +VfsDirectory* VfsDirectory::GetSubdirectory(const std::string& name) { - Subdirectories::iterator it = m_subdirectories.find(name); - if(it == m_subdirectories.end()) + VfsSubdirectories::iterator it = m_vfsSubdirectories.find(name); + if(it == m_vfsSubdirectories.end()) return 0; return &it->second; } -void VfsDirectory::GetEntries(FileInfos* files, Directories* subdirectories) const +void VfsDirectory::GetEntries(FileInfos* files, DirectoryNames* subdirectoryNames) const { if(files) { // (note: VfsFile doesn't return a pointer to FileInfo; instead, // we have it write directly into the files container) - files->resize(m_files.size()); + files->resize(m_vfsFiles.size()); size_t i = 0; - for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it) - it->second.GetFileInfo((*files)[i++]); + for(VfsFiles::const_iterator it = m_vfsFiles.begin(); it != m_vfsFiles.end(); ++it) + it->second.GetFileInfo(&(*files)[i++]); } - if(subdirectories) + if(subdirectoryNames) { - subdirectories->clear(); - subdirectories->reserve(m_subdirectories.size()); - for(Subdirectories::const_iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it) - subdirectories->push_back(it->first); + subdirectoryNames->clear(); + subdirectoryNames->reserve(m_vfsSubdirectories.size()); + for(VfsSubdirectories::const_iterator it = m_vfsSubdirectories.begin(); it != m_vfsSubdirectories.end(); ++it) + subdirectoryNames->push_back(it->first); } } void VfsDirectory::DisplayR(unsigned depth) const { - const char indent[] = " "; + static const char indent[] = " "; const int maxNameChars = 80 - depth*(sizeof(indent)-1); char fmt[20]; - sprintf_s(fmt, ARRAY_SIZE(fmt), "%%-%d.%ds %s", maxNameChars, maxNameChars); + sprintf_s(fmt, ARRAY_SIZE(fmt), "%%-%d.%ds %%s", maxNameChars, maxNameChars); - for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it) + for(VfsFiles::const_iterator it = m_vfsFiles.begin(); it != m_vfsFiles.end(); ++it) { - const char* name = it->first; - const VfsFile& file = it->second; + const std::string& name = it->first; + const VfsFile& vfsFile = it->second; char description[100]; - file.GenerateDescription(description, ARRAY_SIZE(description)); - - for(unsigned i = 0; i < depth; i++) - printf(indent); - printf(fmt, name, description); - } - - for(Subdirectories::const_iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it) - { - const char* name = it->first; - const VfsDirectory& directory = it->second; + vfsFile.GenerateDescription(description, ARRAY_SIZE(description)); for(unsigned i = 0; i < depth+1; i++) printf(indent); - printf("[%s/]\n", name); + printf(fmt, name.c_str(), description); + } - directory.DisplayR(depth+1); + for(VfsSubdirectories::const_iterator it = m_vfsSubdirectories.begin(); it != m_vfsSubdirectories.end(); ++it) + { + const std::string& name = it->first; + const VfsDirectory& vfsDirectory = it->second; + + for(unsigned i = 0; i < depth+1; i++) + printf(indent); + printf("[%s/]\n", name.c_str()); + + vfsDirectory.DisplayR(depth+1); } } void VfsDirectory::ClearR() { - for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it) + for(VfsSubdirectories::iterator it = m_vfsSubdirectories.begin(); it != m_vfsSubdirectories.end(); ++it) it->second.ClearR(); - m_files.clear(); - m_subdirectories.clear(); - m_attachedDirectories.clear(); + m_vfsFiles.clear(); + m_vfsSubdirectories.clear(); + m_realDirectory.reset(); + m_shouldPopulate = 0; } -void VfsDirectory::Attach(const RealDirectory& realDirectory) +void VfsDirectory::Attach(PRealDirectory realDirectory) { - m_attachedDirectories.push_back(realDirectory); +debug_printf("ATTACH %s\n", realDirectory->GetPath().string().c_str()); + + if(!cpu_CAS(&m_shouldPopulate, 0, 1)) + { + debug_assert(0); // multiple Attach() calls without an intervening ShouldPopulate() + return; + } + + m_realDirectory = realDirectory; +} + + +bool VfsDirectory::ShouldPopulate() +{ + return cpu_CAS(&m_shouldPopulate, 1, 0); // test and reset } diff --git a/source/lib/file/vfs/vfs_tree.h b/source/lib/file/vfs/vfs_tree.h index b0869a952c..cd15b55323 100644 --- a/source/lib/file/vfs/vfs_tree.h +++ b/source/lib/file/vfs/vfs_tree.h @@ -11,41 +11,42 @@ #ifndef INCLUDED_VFS_TREE #define INCLUDED_VFS_TREE -#include "lib/file/filesystem.h" // FileInfo, PIFileProvider -#include "real_directory.h" - +#include "lib/file/file_system.h" // FileInfo +#include "lib/file/common/file_loader.h" // PIFileLoader +#include "lib/file/common/real_directory.h" // PRealDirectory class VfsFile { public: - VfsFile(const FileInfo& fileInfo, unsigned priority, PIFileProvider provider, const void* location); + VfsFile(const FileInfo& fileInfo, unsigned priority, PIFileLoader provider); - const char* Name() const + const std::string& Name() const { return m_fileInfo.Name(); } + off_t Size() const { return m_fileInfo.Size(); } - void GetFileInfo(FileInfo& fileInfo) const + + void GetFileInfo(FileInfo* pfileInfo) const { - fileInfo = m_fileInfo; + *pfileInfo = m_fileInfo; } - bool IsSupersededBy(const VfsFile& file) const; + bool IsSupersededBy(const VfsFile& vfsFile) const; + void GenerateDescription(char* text, size_t maxChars) const; - LibError Store(const u8* fileContents, size_t size) const; - LibError Load(u8* fileContents) const; + LibError Load(shared_ptr buf) const; private: mutable FileInfo m_fileInfo; unsigned m_priority; - PIFileProvider m_provider; - const void* m_location; + PIFileLoader m_loader; }; @@ -58,36 +59,46 @@ public: * @return address of existing or newly inserted file; remains * valid until ClearR is called (i.e. VFS is rebuilt). **/ - VfsFile* AddFile(const VfsFile& file); + VfsFile* AddFile(const VfsFile& vfsFile); /** * @return address of existing or newly inserted subdirectory; remains * valid until ClearR is called (i.e. VFS is rebuilt). **/ - VfsDirectory* AddSubdirectory(const char* name); + VfsDirectory* AddSubdirectory(const std::string& name); - VfsFile* GetFile(const char* name); - VfsDirectory* GetSubdirectory(const char* name); + VfsFile* GetFile(const std::string& name); + VfsDirectory* GetSubdirectory(const std::string& name); - void GetEntries(FileInfos* files, Directories* subdirectories) const; + void GetEntries(FileInfos* files, DirectoryNames* subdirectories) const; void DisplayR(unsigned depth) const; + void ClearR(); - void Attach(const RealDirectory& realDirectory); - const std::vector& AttachedDirectories() const + void Attach(PRealDirectory realDirectory); + + PRealDirectory AssociatedDirectory() const { - return m_attachedDirectories; + return m_realDirectory; } + /** + * @return whether this directory should be populated from its + * AssociatedDirectory(). note that calling this is a promise to + * do so if true is returned -- the flag is immediately reset. + **/ + bool ShouldPopulate(); + private: - typedef std::map Files; - Files m_files; + typedef std::map VfsFiles; + VfsFiles m_vfsFiles; - typedef std::map Subdirectories; - Subdirectories m_subdirectories; + typedef std::map VfsSubdirectories; + VfsSubdirectories m_vfsSubdirectories; - std::vector m_attachedDirectories; + PRealDirectory m_realDirectory; + volatile uintptr_t m_shouldPopulate; // (cpu_CAS can't be used on bool) }; #endif // #ifndef INCLUDED_VFS_TREE diff --git a/source/lib/path_util.cpp b/source/lib/path_util.cpp index 530543111f..5bd3c6388c 100644 --- a/source/lib/path_util.cpp +++ b/source/lib/path_util.cpp @@ -203,15 +203,6 @@ LibError path_append(char* dst, const char* path1, const char* path2, uint flags } -const char* path_append2(const char* path1, const char* path2, uint flags) -{ - char dst[PATH_MAX]; - LibError ret = path_append(dst, path1, path2, flags); - debug_assert(ret == INFO::OK); - return path_Pool()->UniqueCopy(dst); -} - - // strip from the start of , prepend , // and write to . // returns ERR::FAIL (without warning!) if the beginning of doesn't @@ -241,24 +232,23 @@ LibError path_replace(char* dst, const char* src, const char* remove, const char // split paths into specific parts -void path_split(const char* pathname, const char** ppath, const char** pname) +void path_split(const char* pathname, char* path, char* name) { - const char* name = path_name_only(pathname); - if(pname) + const char* namePos = path_name_only(pathname); + + if(name) { - if(name[0] == '\0') - *pname = 0; - else - *pname = path_Pool()->UniqueCopy(name); + path_copy(name, namePos); + + path_component_validate(name); } - if(ppath) + if(path) { - char pathnameCopy[PATH_MAX]; - path_copy(pathnameCopy, pathname); + path_copy(path, pathname); + path[namePos-pathname] = '\0'; // strip filename - pathnameCopy[name-pathname] = '\0'; // strip filename - *ppath = path_Pool()->UniqueCopy(pathnameCopy); + debug_assert(path_IsDirectory(path)); } } @@ -273,7 +263,10 @@ const char* path_name_only(const char* path) return path; // return name, i.e. component after the last portable or platform slash - return std::max(slash1, slash2)+1; + const char* name = std::max(slash1, slash2)+1; + if(name[0] != '\0') // else path_component_validate would complain + path_component_validate(name); + return name; } @@ -302,6 +295,7 @@ const char* path_last_component(const char* path) pos += component_len+1; // +1 for separator } + path_component_validate(last_component); return last_component; } @@ -311,6 +305,7 @@ void path_strip_fn(char* path) { char* name = (char*)path_name_only(path); *name = '\0'; // cut off string here + debug_assert(path_IsDirectory(path)); } @@ -321,13 +316,7 @@ void path_dir_only(const char* pathname, char* path) { path_copy(path, pathname); path_strip_fn(path); -} - -const char* path_dir_only2(const char* pathname) -{ - char path[PATH_MAX]; - path_dir_only(pathname, path); - return path_Pool()->UniqueCopy(path); + // (path_strip_fn already ensures its output is a directory path) } @@ -377,6 +366,8 @@ LibError path_foreach_component(const char* path_org, PathComponentCb cb, uintpt else *slash = '\0'; // 0-terminate cur_component + path_component_validate(cur_component); + LibError ret = cb(cur_component, is_dir, cbData); // callback wants to abort - return its value. if(ret != INFO::CB_CONTINUE) @@ -455,14 +446,3 @@ LibError path_package_append_file(PathPackage* pp, const char* path) CHECK_ERR(strcpy_s(pp->end, pp->chars_left, path)); return INFO::OK; } - - -//----------------------------------------------------------------------------- - -StringPool* path_Pool() -{ - static StringPool* instance; - if(!instance) - instance = new StringPool(8*MiB); - return instance; -} diff --git a/source/lib/path_util.h b/source/lib/path_util.h index a0aac75707..924cd164d7 100644 --- a/source/lib/path_util.h +++ b/source/lib/path_util.h @@ -9,7 +9,7 @@ // license: GPL; see lib/license.txt // notes: -// - this module is split out of lib/res/file so that it can be used from +// - this module is independent of lib/file so that it can be used from // other code without pulling in the entire file manager. // - there is no restriction on buffer lengths except the underlying OS. // input buffers must not exceed PATH_MAX chars, while outputs @@ -23,8 +23,6 @@ #include "lib/allocators/string_pool.h" -#include "posix/posix_types.h" // PATH_MAX - namespace ERR { @@ -42,7 +40,7 @@ namespace ERR * * @return LibError (ERR::PATH_* or INFO::OK) **/ -extern LibError path_validate(const char* path); +LIB_API LibError path_validate(const char* path); /** * return appropriate code if path is invalid, otherwise continue. @@ -54,7 +52,7 @@ extern LibError path_validate(const char* path); * * @return LibError (ERR::PATH_* or INFO::OK) **/ -extern LibError path_component_validate(const char* name); +LIB_API LibError path_component_validate(const char* name); /** * is the given character a path separator character? @@ -62,14 +60,14 @@ extern LibError path_component_validate(const char* name); * @param c character to test * @return bool **/ -extern bool path_is_dir_sep(char c); +LIB_API bool path_is_dir_sep(char c); /** * is the given path(name) a directory? * * @return bool **/ -extern bool path_IsDirectory(const char* path); +LIB_API bool path_IsDirectory(const char* path); /** * is s2 a subpath of s1, or vice versa? (equal counts as subpath) @@ -77,7 +75,7 @@ extern bool path_IsDirectory(const char* path); * @param s1, s2 comparand strings * @return bool **/ -extern bool path_is_subpath(const char* s1, const char* s2); +LIB_API bool path_is_subpath(const char* s1, const char* s2); /** @@ -87,7 +85,7 @@ extern bool path_is_subpath(const char* s1, const char* s2); * and should hold PATH_MAX chars. * @param src source; should not exceed PATH_MAX chars **/ -extern void path_copy(char* dst, const char* src); +LIB_API void path_copy(char* dst, const char* src); /** * flags controlling path_append behavior @@ -111,10 +109,7 @@ enum PathAppendFlags * @param flags see PathAppendFlags. * @return LibError **/ -extern LibError path_append(char* dst, const char* path1, const char* path2, uint flags = 0); - -// same as path_append, but returns a unique pointer to the result -extern const char* path_append2(const char* path1, const char* path2, uint flags = 0); +LIB_API LibError path_append(char* dst, const char* path1, const char* path2, uint flags = 0); /** * at the start of a path, replace the given substring with another. @@ -127,13 +122,13 @@ extern const char* path_append2(const char* path1, const char* path2, uint flags * @return LibError; ERR::FAIL (without warning!) if doesn't * match . **/ -extern LibError path_replace(char* dst, const char* src, const char* remove, const char* replace); +LIB_API LibError path_replace(char* dst, const char* src, const char* remove, const char* replace); /** * combination of path_name_only and path_dir_only2 * (more efficient than calling them separately) **/ -extern void path_split(const char* pathname, const char** path, const char** name); +LIB_API void path_split(const char* pathname, char* path, char* name); /** @@ -143,7 +138,7 @@ extern void path_split(const char* pathname, const char** path, const char** nam * @param path input path. * @return pointer to name component within . **/ -extern const char* path_name_only(const char* path); +LIB_API const char* path_name_only(const char* path); /** * get the last component of a path. @@ -153,15 +148,14 @@ extern const char* path_name_only(const char* path); * @param path input path. * @return pointer to last component within . **/ -extern const char* path_last_component(const char* path); +LIB_API const char* path_last_component(const char* path); /** * strip away the name component in a path. * * @param path input and output; chopped by inserting '\0'. **/ - -extern void path_strip_fn(char* path); +LIB_API void path_strip_fn(char* path); /** * retrieve only the directory path portion of a path. @@ -171,9 +165,7 @@ extern void path_strip_fn(char* path); * otherwise it ends with '/'). * note: implementation via path_copy and path_strip_fn. **/ -extern void path_dir_only(const char* path, char* dir); - -extern const char* path_dir_only2(const char* path); +LIB_API void path_dir_only(const char* path, char* dir); /** * get filename's extension. @@ -181,7 +173,7 @@ extern const char* path_dir_only2(const char* path); * @return pointer to extension within , or "" if there is none. * NOTE: does not include the period; e.g. "a.bmp" yields "bmp". **/ -extern const char* path_extension(const char* fn); +LIB_API const char* path_extension(const char* fn); /** @@ -208,7 +200,7 @@ typedef LibError (*PathComponentCb)(const char* component, bool is_dir, uintptr_ * * @return LibError **/ -extern LibError path_foreach_component(const char* path, PathComponentCb cb, uintptr_t cbData); +LIB_API LibError path_foreach_component(const char* path, PathComponentCb cb, uintptr_t cbData); //----------------------------------------------------------------------------- @@ -235,26 +227,18 @@ struct PathPackage * does not care if paths are relative/portable/absolute. * @return LibError **/ -extern LibError path_package_set_dir(PathPackage* pp, const char* dir); +LIB_API LibError path_package_set_dir(PathPackage* pp, const char* dir); /** * copy one PathPackage into another (doing so directly is incorrect!) **/ -extern void path_package_copy(PathPackage* pp_dst, const PathPackage* pp_src); +LIB_API void path_package_copy(PathPackage* pp_dst, const PathPackage* pp_src); /** * append the given filename to the directory established by the last * path_package_set_dir on this package. the whole path is accessible at pp->path. * @return LibError **/ -extern LibError path_package_append_file(PathPackage* pp, const char* path); - - -//----------------------------------------------------------------------------- - -/** - * @return a shared instance of a StringPool (used to store pathnames). - **/ -extern StringPool* path_Pool(); +LIB_API LibError path_package_append_file(PathPackage* pp, const char* path); #endif // #ifndef INCLUDED_PATH_UTIL diff --git a/source/lib/res/file/archive/archive.cpp b/source/lib/res/file/archive/archive.cpp deleted file mode 100644 index f0b6f02844..0000000000 --- a/source/lib/res/file/archive/archive.cpp +++ /dev/null @@ -1,717 +0,0 @@ -/** - * ========================================================================= - * File : archive.cpp - * Project : 0 A.D. - * Description : provide access to archive "resources". allows - * : opening, reading from, and creating them. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "archive.h" - -#include "lib/timer.h" -#include "lib/allocators.h" -#include "lib/res/res.h" -#include "../file_internal.h" - -#include - - -// components: -// - za_*: Zip archive handling -// passes the list of files in an archive to lookup. -// - lookup_*: file lookup -// per archive: return file info (e.g. offset, size), given filename. -// - Archive_*: Handle-based container for archive info -// owns archive file and its lookup mechanism. -// - inf_*: in-memory inflate routines (zlib wrapper) -// decompresses blocks from file_io callback. -// - afile_*: file from Zip archive -// uses lookup to get file information; holds inflate state. -// - sync and async I/O -// uses file_* and inf_*. -// - file mapping - - -ERROR_ASSOCIATE(ERR::IS_COMPRESSED, "Invalid operation for a compressed file", -1); - - -/////////////////////////////////////////////////////////////////////////////// -// -// lookup_*: file lookup -// per archive: return file info (e.g. offset, size), given filename. -// -/////////////////////////////////////////////////////////////////////////////// - - -// rationale: -// - we don't export a "key" (currently array index) that would allow faster -// file lookup. this would only be useful if higher-level code were to -// store the key and use it more than once. also, lookup is currently fast -// enough. finally, this would also make our file enumerate callback -// incompatible with the others (due to the extra key param). -// -// - we don't bother with a directory tree to speed up lookup. the above -// is fast enough: O(1) if accessed sequentially, otherwise O(log(files)). - - -/////////////////////////////////////////////////////////////////////////////// -// -// Archive_*: Handle-based container for archive info -// owns archive file and its lookup mechanism. -// -/////////////////////////////////////////////////////////////////////////////// - - -struct Archive -{ - File f; - - ArchiveEntry* ents; - // number of valid entries in above array (see lookup_add_file_cb) - uint num_files; - - // note: we need to keep track of what resources reload() allocated, - // so the dtor can free everything correctly. - uint is_open : 1; - uint is_loaded : 1; -}; - -H_TYPE_DEFINE(Archive); - - -static void Archive_init(Archive*, va_list) -{ -} - -static void Archive_dtor(Archive* a) -{ - if(a->is_loaded) - { - (void)mem_free(a->ents); - - a->is_loaded = 0; - } - if(a->is_open) - { - (void)file_close(&a->f); - a->is_open = 0; - } -} - -static LibError Archive_reload(Archive* a, const char* fn, Handle) -{ - // must be enabled in archive files for efficiency (see decl). - // note that afile_read overrides archive file flags for - // uncompressed IOs, but this flag is re-added there. - const uint flags = FILE_CACHE_BLOCK; - // (note: don't warn on failure - this happens when - // vfs_mount blindly archive_open-s a dir) - RETURN_ERR(file_open(fn, flags, &a->f)); - a->is_open = 1; - - RETURN_ERR(zip_populate_archive(&a->f, a)); - a->is_loaded = 1; - - return INFO::OK; -} - -static LibError Archive_validate(const Archive* a) -{ - RETURN_ERR(file_validate(&a->f)); - - if(debug_is_pointer_bogus(a->ents)) - WARN_RETURN(ERR::_1); - - return INFO::OK; -} - -static LibError Archive_to_string(const Archive* a, char* buf) -{ - snprintf(buf, H_STRING_LEN, "(%u files)", a->num_files); - return INFO::OK; -} - - - -// open and return a handle to the archive indicated by . -// somewhat slow - each file is added to an internal index. -Handle archive_open(const char* fn) -{ -TIMER("archive_open"); - // note: must not keep the archive open. the archive builder asks - // vfs_mount to back away from all archives and close them, - // which must happen immediately or else deleting archives will fail. - return h_alloc(H_Archive, fn, RES_NO_CACHE); -} - - -// close the archive and set ha to 0 -LibError archive_close(Handle& ha) -{ - return h_free(ha, H_Archive); -} - - - - -// look up ArchiveEntry, given filename (untrusted!). -static LibError archive_get_file_info(Archive* a, const char* fn, uintptr_t memento, ArchiveEntry*& ent) -{ - if(memento) - { - ent = (ArchiveEntry*)memento; - return INFO::OK; - } - else - { - const char* atom_fn = file_make_unique_fn_copy(fn); - for(uint i = 0; i < a->num_files; i++) - if(a->ents[i].atom_fn == atom_fn) - { - ent = &a->ents[i]; - return INFO::OK; - } - } - - WARN_RETURN(ERR::TNODE_NOT_FOUND); -} - - -// successively call for each valid file in the archive , -// passing the complete path and . -// if it returns a nonzero value, abort and return that, otherwise 0. -// -// FileCB's name parameter will be the full path and unique -// (i.e. returned by file_make_unique_fn_copy). -LibError archive_enum(const Handle ha, const FileCB cb, const uintptr_t user) -{ - H_DEREF(ha, Archive, a); - - struct stat s; - memset(&s, 0, sizeof(s)); - - for(uint i = 0; i < a->num_files; i++) - { - const ArchiveEntry* ent = &a->ents[i]; - s.st_mode = S_IFREG; - s.st_size = (off_t)ent->usize; - s.st_mtime = ent->mtime; - const uintptr_t memento = (uintptr_t)ent; - LibError ret = cb(ent->atom_fn, &s, memento, user); - if(ret != INFO::CB_CONTINUE) - return ret; - } - - return INFO::OK; -} - - -LibError archive_allocate_entries(Archive* a, size_t num_entries) -{ - debug_assert(num_entries != 0); // =0 makes no sense but wouldn't be fatal - - debug_assert(a->ents == 0); // must not have been allocated yet - a->ents = (ArchiveEntry*)mem_alloc(num_entries * sizeof(ArchiveEntry), 32); - if(!a->ents) - WARN_RETURN(ERR::NO_MEM); - return INFO::OK; -} - - -// add file to the lookup data structure. -// called from za_enum_files in order (0 <= idx < num_entries). -// the first call notifies us of # entries, so we can allocate memory. -// -// note: ent is only valid during the callback! must be copied or saved. -LibError archive_add_file(Archive* a, const ArchiveEntry* ent) -{ - a->ents[a->num_files++] = *ent; - return INFO::OK; -} - - - -/////////////////////////////////////////////////////////////////////////////// -// -// afile_*: file from Zip archive -// uses lookup to get file information; holds inflate state. -// -/////////////////////////////////////////////////////////////////////////////// - -struct ArchiveFile -{ - off_t ofs; // in archive - off_t csize; - CompressionMethod method; - u32 checksum; - - off_t last_cofs; // in compressed file - - Handle ha; - uintptr_t ctx; - - // this File has been successfully afile_map-ped, i.e. reference - // count of the archive's mapping has been increased. - // we need to undo that when closing it. - uint is_mapped : 1; -}; -cassert(sizeof(ArchiveFile) <= FILE_OPAQUE_SIZE); - -// convenience function, allows implementation change in File. -// note that size == usize isn't foolproof, and adding a flag to -// ofs or size is ugly and error-prone. -// no error checking - always called from functions that check af. -static inline bool is_compressed(ArchiveFile* af) -{ - return af->method != CM_NONE; -} - - - - -// get file status (size, mtime). output param is zeroed on error. -LibError afile_stat(Handle ha, const char* fn, struct stat* s) -{ - // zero output param in case we fail below. - memset(s, 0, sizeof(struct stat)); - - H_DEREF(ha, Archive, a); - - ArchiveEntry* ent; - RETURN_ERR(archive_get_file_info(a, fn, 0, ent)); - - s->st_size = ent->usize; - s->st_mtime = ent->mtime; - return INFO::OK; -} - - - - -LibError afile_validate(const File* f) -{ - if(!f) - WARN_RETURN(ERR::INVALID_PARAM); - const ArchiveFile* af = (const ArchiveFile*)f->opaque; - UNUSED2(af); - // note: don't check af->ha - it may be freed at shutdown before - // its files. TODO: revisit once dependency support is added. - if(!f->size) - WARN_RETURN(ERR::_1); - // note: af->ctx is 0 if file is not compressed. - - return INFO::OK; -} - -#define CHECK_AFILE(f) RETURN_ERR(afile_validate(f)) - - -// open file, and fill *af with information about it. -// return < 0 on error (output param zeroed). -LibError afile_open(const Handle ha, const char* fn, uintptr_t memento, uint flags, File* f) -{ - // zero output param in case we fail below. - memset(f, 0, sizeof(*f)); - - if(flags & FILE_WRITE) - WARN_RETURN(ERR::IS_COMPRESSED); - - H_DEREF(ha, Archive, a); - - // this is needed for File below. optimization: archive_get_file_info - // wants the original filename, but by passing the unique copy - // we avoid work there (its file_make_unique_fn_copy returns immediately) - const char* atom_fn = file_make_unique_fn_copy(fn); - - ArchiveEntry* ent; - // don't want File to contain a ArchiveEntry struct - - // its usize member must be 'loose' for compatibility with File. - // => need to copy ArchiveEntry fields into File. - RETURN_ERR(archive_get_file_info(a, atom_fn, memento, ent)); - - zip_fixup_lfh(&a->f, ent); - - uintptr_t ctx = 0; - // slight optimization: do not allocate context if not compressed - if(ent->method != CM_NONE) - { - ctx = comp_alloc(CT_DECOMPRESSION, ent->method); - if(!ctx) - WARN_RETURN(ERR::NO_MEM); - } - - f->flags = flags; - f->size = ent->usize; - f->atom_fn = atom_fn; - ArchiveFile* af = (ArchiveFile*)f->opaque; - af->ofs = ent->ofs; - af->csize = ent->csize; - af->method = ent->method; - af->checksum = ent->checksum; - af->ha = ha; - af->ctx = ctx; - af->is_mapped = 0; - CHECK_AFILE(f); - return INFO::OK; -} - - -// close file. -LibError afile_close(File* f) -{ - CHECK_AFILE(f); - ArchiveFile* af = (ArchiveFile*)f->opaque; - // other File fields don't need to be freed/cleared - comp_free(af->ctx); - af->ctx = 0; - return INFO::OK; -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// sync and async I/O -// uses file_* and inf_*. -// -/////////////////////////////////////////////////////////////////////////////// - -struct ArchiveFileIo -{ - // note: this cannot be embedded into the struct due to the FileIo - // interface (fixed size limit and type field). - // it is passed by afile_read to file_io, so we'll have to allocate - // and point to it. - FileIo* io; - - uintptr_t ctx; - - size_t max_output_size; - u8* user_buf; -}; -cassert(sizeof(ArchiveFileIo) <= FILE_IO_OPAQUE_SIZE); - -static const size_t CHUNK_SIZE = 16*KiB; - -static SingleAllocator io_allocator; - -// begin transferring bytes, starting at . get result -// with afile_io_wait; when no longer needed, free via afile_io_discard. -LibError afile_io_issue(File* f, off_t user_ofs, size_t max_output_size, u8* user_buf, FileIo* io) -{ - // zero output param in case we fail below. - memset(io, 0, sizeof(FileIo)); - - CHECK_AFILE(f); - ArchiveFile* af = (ArchiveFile*)f->opaque; - H_DEREF(af->ha, Archive, a); - - ArchiveFileIo* aio = (ArchiveFileIo*)io->opaque; - aio->io = io_allocator.Allocate(); - if(!aio->io) - WARN_RETURN(ERR::NO_MEM); - - // not compressed; we'll just read directly from the archive file. - // no need to clamp to EOF - that's done already by the VFS. - if(!is_compressed(af)) - { - // aio->ctx is 0 (due to memset) - const off_t ofs = af->ofs+user_ofs; - return file_io_issue(&a->f, ofs, max_output_size, user_buf, aio->io); - } - - - aio->ctx = af->ctx; - aio->max_output_size = max_output_size; - aio->user_buf = user_buf; - - const off_t cofs = af->ofs + af->last_cofs; // needed to determine csize - - // read up to next chunk (so that the next read is aligned - - // less work for aio) or up to EOF. - const ssize_t left_in_chunk = CHUNK_SIZE - (cofs % CHUNK_SIZE); - const ssize_t left_in_file = af->csize - cofs; - const size_t csize = std::min(left_in_chunk, left_in_file); - - u8* cbuf = (u8*)mem_alloc(csize, 4*KiB); - if(!cbuf) - WARN_RETURN(ERR::NO_MEM); - - RETURN_ERR(file_io_issue(&a->f, cofs, csize, cbuf, aio->io)); - - af->last_cofs += (off_t)csize; - return INFO::OK; -} - - -// indicates if the IO referenced by has completed. -// return value: 0 if pending, 1 if complete, < 0 on error. -int afile_io_has_completed(FileIo* io) -{ - ArchiveFileIo* aio = (ArchiveFileIo*)io->opaque; - return file_io_has_completed(aio->io); -} - - -// wait until the transfer completes, and return its buffer. -// output parameters are zeroed on error. -LibError afile_io_wait(FileIo* io, u8*& buf, size_t& size) -{ - buf = 0; - size = 0; - - ArchiveFileIo* aio = (ArchiveFileIo*)io->opaque; - - u8* raw_buf; - size_t raw_size; - RETURN_ERR(file_io_wait(aio->io, raw_buf, raw_size)); - - // file is compressed and we need to decompress - if(aio->ctx) - { - comp_set_output(aio->ctx, aio->user_buf, aio->max_output_size); - const ssize_t ubytes_output = comp_feed(aio->ctx, raw_buf, raw_size); - free(raw_buf); - RETURN_ERR(ubytes_output); - - buf = aio->user_buf; - size = ubytes_output; - } - else - { - buf = raw_buf; - size = raw_size; - } - - return INFO::OK; -} - - -// finished with transfer - free its buffer (returned by afile_io_wait) -LibError afile_io_discard(FileIo* io) -{ - ArchiveFileIo* aio = (ArchiveFileIo*)io->opaque; - LibError ret = file_io_discard(aio->io); - io_allocator.Free(aio->io); - return ret; -} - - -LibError afile_io_validate(const FileIo* io) -{ - ArchiveFileIo* aio = (ArchiveFileIo*)io->opaque; - if(debug_is_pointer_bogus(aio->user_buf)) - WARN_RETURN(ERR::_1); - // and have no invariants we could check. - RETURN_ERR(file_io_validate(aio->io)); - return INFO::OK; -} - - -//----------------------------------------------------------------------------- - -class Decompressor -{ -public: - Decompressor(uintptr_t ctx, FileIOBuf* pbuf, size_t usizeMax, FileIOCB cb, uintptr_t cbData) - : m_ctx(ctx) - , m_udataSize(usizeMax), m_csizeTotal(0), m_usizeTotal(0) - , m_cb(cb), m_cbData(cbData) - { - debug_assert(m_ctx != 0); - - if(pbuf == FILE_BUF_TEMP) - { - m_tmpBuf.reset((u8*)page_aligned_alloc(m_udataSize), PageAlignedDeleter(m_udataSize)); - m_udata = m_tmpBuf.get(); - } - else - m_udata = (u8*)*pbuf; // WARNING: FileIOBuf is nominally const; if that's ever enforced, this may need to change. - } - - LibError Feed(const u8* cblock, size_t cblockSize, size_t* bytes_processed) - { - // when decompressing into the temp buffer, always start at ofs=0. - const size_t ofs = m_tmpBuf.get()? 0 : m_usizeTotal; - u8* const ublock = m_udata + ofs; - comp_set_output(m_ctx, ublock, m_udataSize-ofs); - - const size_t ublockSize = comp_feed(m_ctx, cblock, cblockSize); - - m_csizeTotal += cblockSize; - m_usizeTotal += ublockSize; - debug_assert(m_usizeTotal <= m_udataSize); - - *bytes_processed = ublockSize; - LibError ret = INFO::CB_CONTINUE; - if(m_cb) - ret = m_cb(m_cbData, ublock, ublockSize, bytes_processed); - if(m_usizeTotal == m_udataSize) - ret = INFO::OK; - return ret; - } - - LibError Finish(u32& checksum) - { - u8* out; size_t outSize; // unused - return comp_finish(m_ctx, &out, &outSize, &checksum); - } - - size_t NumCompressedBytesProcessed() const - { - return m_csizeTotal; - } - -private: - uintptr_t m_ctx; - - size_t m_csizeTotal; - size_t m_usizeTotal; - - u8* m_udata; - size_t m_udataSize; - - boost::shared_ptr m_tmpBuf; - - // allow user-specified callbacks: "chain" them, because file_io's - // callback mechanism is already used to return blocks. - FileIOCB m_cb; - uintptr_t m_cbData; -}; - - -static LibError decompressor_feed_cb(uintptr_t cbData, - const u8* cblock, size_t cblockSize, size_t* bytes_processed) -{ - Decompressor& decompressor = *(Decompressor*)cbData; - return decompressor.Feed(cblock, cblockSize, bytes_processed); -} - - -// read from the (possibly compressed) file as if it were a normal file. -// starting at the beginning of the logical (decompressed) file, -// skip bytes of data; read the next bytes into <*pbuf>. -// -// if non-NULL, is called for each block read, passing . -// if it returns a negative error code, -// the read is aborted and that value is returned. -// the callback mechanism is useful for user progress notification or -// processing data while waiting for the next I/O to complete -// (quasi-parallel, without the complexity of threads). -// -// return bytes read, or a negative error code. -ssize_t afile_read(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb, uintptr_t cbData) -{ - CHECK_AFILE(f); - ArchiveFile* af = (ArchiveFile*)f->opaque; - H_DEREF(af->ha, Archive, a); - - if(!is_compressed(af)) - { - // HACK - // background: file_io will operate according to the - // *archive* file's flags, but the File may contain some overrides - // set via vfs_open. one example is FILE_LONG_LIVED - - // that must be copied over (temporarily) into a->f flags. - // - // we currently copy all flags - this may mean that setting - // global policy flags for all archive files is difficult, - // but that can be worked around by setting them in afile_open. - // this is better than the alternative of copying individual - // flags because it'd need to be updated as new flags are added. - a->f.flags = f->flags; - // this was set in Archive_reload and must be re-enabled for efficiency. - a->f.flags |= FILE_CACHE_BLOCK; - - bool we_allocated = (pbuf != FILE_BUF_TEMP) && (*pbuf == FILE_BUF_ALLOC); - // no need to set last_cofs - only checked if compressed. - ssize_t bytes_read = file_io(&a->f, af->ofs+ofs, size, pbuf, cb, cbData); - RETURN_ERR(bytes_read); - if(we_allocated) - (void)file_buf_set_real_fn(*pbuf, f->atom_fn); - return bytes_read; - } - - RETURN_ERR(file_io_get_buf(pbuf, size, f->atom_fn, f->flags, cb)); - - const off_t cofs = af->ofs+af->last_cofs; - // remaining bytes in file. callback will cause IOs to stop when - // enough udata has been produced. - const size_t csize_max = af->csize - af->last_cofs; - - Decompressor decompressor(af->ctx, pbuf, size, cb, cbData); - const ssize_t usize_read = file_io(&a->f, cofs, csize_max, FILE_BUF_TEMP, decompressor_feed_cb, (uintptr_t)&decompressor); - u32 checksum; - RETURN_ERR(decompressor.Finish(checksum)); - //debug_assert(checksum == af->checksum); - - af->last_cofs += (off_t)decompressor.NumCompressedBytesProcessed(); - - return usize_read; -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// file mapping -// -/////////////////////////////////////////////////////////////////////////////// - - -// map the entire file into memory. mapping compressed files -// isn't allowed, since the compression algorithm is unspecified. -// output parameters are zeroed on failure. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -LibError afile_map(File* f, u8*& p, size_t& size) -{ - p = 0; - size = 0; - - CHECK_AFILE(f); - ArchiveFile* af = (ArchiveFile*)f->opaque; - - // mapping compressed files doesn't make sense because the - // compression algorithm is unspecified - disallow it. - if(is_compressed(af)) - WARN_RETURN(ERR::IS_COMPRESSED); - - // note: we mapped the archive in archive_open, but unmapped it - // in the meantime to save memory in case it wasn't going to be mapped. - // now we do so again; it's unmapped in afile_unmap (refcounted). - H_DEREF(af->ha, Archive, a); - u8* archive_p; size_t archive_size; - RETURN_ERR(file_map(&a->f, archive_p, archive_size)); - - p = archive_p + af->ofs; - size = f->size; - - af->is_mapped = 1; - return INFO::OK; -} - - -// remove the mapping of file ; fail if not mapped. -// -// the mapping will be removed (if still open) when its archive is closed. -// however, map/unmap calls should be paired so that the archive mapping -// may be removed when no longer needed. -LibError afile_unmap(File* f) -{ - CHECK_AFILE(f); - ArchiveFile* af = (ArchiveFile*)f->opaque; - - // make sure archive mapping refcount remains balanced: - // don't allow multiple|"false" unmaps. - if(!af->is_mapped) - WARN_RETURN(ERR::FILE_NOT_MAPPED); - af->is_mapped = 0; - - H_DEREF(af->ha, Archive, a); - return file_unmap(&a->f); -} diff --git a/source/lib/res/file/archive/archive.h b/source/lib/res/file/archive/archive.h deleted file mode 100644 index 8a302ef715..0000000000 --- a/source/lib/res/file/archive/archive.h +++ /dev/null @@ -1,214 +0,0 @@ -/** - * ========================================================================= - * File : archive.h - * Project : 0 A.D. - * Description : provide access to archive "resources". allows - * : opening, reading from, and creating them. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_ARCHIVE -#define INCLUDED_ARCHIVE - -#include "lib/res/handle.h" -#include "../file.h" // FileCB for afile_enum -#include "compression.h" // CompressionMethod - -namespace ERR -{ - const LibError IS_COMPRESSED = -110400; -} - -// note: filenames are case-insensitive. - - -// -// archive -// - -// open and return a handle to the archive indicated by . -// somewhat slow - each file is added to an internal index. -extern Handle archive_open(const char* fn); - -// close the archive and set ha to 0 -extern LibError archive_close(Handle& ha); - -// successively call for each valid file in the archive , -// passing the complete path and . -// if it returns a nonzero value, abort and return that, otherwise 0. -// -// FileCB's name parameter will be the full path and unique -// (i.e. returned by file_make_unique_fn_copy). -extern LibError archive_enum(const Handle ha, const FileCB cb, const uintptr_t user); - - -// -// file -// - -// get file status (size, mtime). output param is zeroed on error. -extern LibError afile_stat(Handle ha, const char* fn, struct stat* s); - -// open file, and fill *f with information about it. -// return < 0 on error (output param zeroed). -extern LibError afile_open(Handle ha, const char* fn, uintptr_t memento, uint flags, File* f); - -// close file. -extern LibError afile_close(File* f); - -extern LibError afile_validate(const File* f); - -extern LibError afile_open_vfs(const char* fn, uint flags, File* f, TFile* tf); - -// -// asynchronous read -// - -// begin transferring bytes, starting at . get result -// with afile_io_wait; when no longer needed, free via afile_io_discard. -extern LibError afile_io_issue(File* f, off_t ofs, size_t size, u8* buf, FileIo* io); - -// indicates if the IO referenced by has completed. -// return value: 0 if pending, 1 if complete, < 0 on error. -extern int afile_io_has_completed(FileIo* io); - -// wait until the transfer completes, and return its buffer. -// output parameters are zeroed on error. -extern LibError afile_io_wait(FileIo* io, u8*& p, size_t& size); - -// finished with transfer - free its buffer (returned by afile_io_wait) -extern LibError afile_io_discard(FileIo* io); - -extern LibError afile_io_validate(const FileIo* io); - - -// -// synchronous read -// - -// read from the (possibly compressed) file as if it were a normal file. -// starting at the beginning of the logical (decompressed) file, -// skip bytes of data; read the next bytes into . -// -// if non-NULL, is called for each block read, passing . -// if it returns a negative error code, -// the read is aborted and that value is returned. -// the callback mechanism is useful for user progress notification or -// processing data while waiting for the next I/O to complete -// (quasi-parallel, without the complexity of threads). -// -// return bytes read, or a negative error code. -extern ssize_t afile_read(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb = 0, uintptr_t ctx = 0); - - -// -// memory mapping -// - -// useful for files that are too large to be loaded into memory, -// or if only (non-sequential) portions of a file are needed at a time. -// -// this is of course only possible for uncompressed files - compressed files -// would have to be inflated sequentially, which defeats the point of mapping. - - -// map the entire file into memory. mapping compressed files -// isn't allowed, since the compression algorithm is unspecified. -// output parameters are zeroed on failure. -// -// the mapping will be removed (if still open) when its archive is closed. -// however, map/unmap calls should still be paired so that the archive mapping -// may be removed when no longer needed. -extern LibError afile_map(File* f, u8*& p, size_t& size); - -// remove the mapping of file ; fail if not mapped. -// -// the mapping will be removed (if still open) when its archive is closed. -// however, map/unmap calls should be paired so that the archive mapping -// may be removed when no longer needed. -extern LibError afile_unmap(File* f); - - - -// -// interface for backends -// - -// the archive-specific backends call back here for each file; -// this module provides storage for the file table. - -enum ArchiveFileFlags -{ - // indicates ArchiveEntry.ofs points to a "local file header" - // instead of the file data. a fixup routine is called upon - // file open; it skips past LFH and clears this flag. - // this is somewhat of a hack, but vital to archive open - // performance. without it, we'd have to scan through the - // entire Zip file, which can take *seconds*. - // (we cannot use the information in CDFH, because its 'extra' field - // has been observed to differ from that of the LFH) - // by reading LFH when a file in archive is opened, the block cache - // absorbs the IO cost because the file will likely be read anyway. - ZIP_LFH_FIXUP_NEEDED = 1 -}; - -// holds all per-file information extracted from the header. -// this is intended to work for all archive types. -// -// note: File* (state of a currently open file) is separate because -// some of its fields need not be stored here; we'd like to minimize -// size of the file table. -struct ArchiveEntry -{ - // these are returned by afile_stat: - off_t usize; - time_t mtime; - - // used in IO - off_t ofs; - off_t csize; - CompressionMethod method; - u32 checksum; - - uint flags; // ArchiveFileFlags - - const char* atom_fn; - - // why csize? - // file I/O may be N-buffered, so it's good to know when the raw data - // stops, or else we potentially overshoot by N-1 blocks. - // if we do read too much though, nothing breaks - inflate would just - // ignore it, since Zip files are compressed individually. - // - // we also need a way to check if a file is compressed (e.g. to fail - // mmap requests if the file is compressed). packing a bit in ofs or - // usize is error prone and ugly (1 bit less won't hurt though). - // any other way will mess up the nice 2^n byte size anyway, so - // might as well store csize. -}; - -// successively called for each valid file in the archive, -// passing the complete path and . -// return INFO::CB_CONTINUE to continue calling; anything else will cause -// the caller to abort and immediately return that value. -// -// HACK: call back with negative index the first time; its abs. value is -// the number of entries in the archive. lookup needs to know this so it can -// preallocate memory. having lookup_init call z_get_num_files and then -// za_enum_files would require passing around a ZipInfo struct, or searching -// for the ECDR twice - both ways aren't nice. nor is expanding on demand - -// we try to minimize allocations (faster, less fragmentation). - -// fn (filename) is not necessarily 0-terminated! -// loc is only valid during the callback! must be copied or saved. -typedef LibError (*CDFH_CB)(uintptr_t user, i32 i, const ArchiveEntry* loc, size_t fn_len); - - -struct Archive; - -extern LibError archive_allocate_entries(Archive* a, size_t num_entries); -extern LibError archive_add_file(Archive* a, const ArchiveEntry* ent); - -#endif // #ifndef INCLUDED_ARCHIVE diff --git a/source/lib/res/file/archive/archive_builder.cpp b/source/lib/res/file/archive/archive_builder.cpp deleted file mode 100644 index 94393b7881..0000000000 --- a/source/lib/res/file/archive/archive_builder.cpp +++ /dev/null @@ -1,289 +0,0 @@ -/** - * ========================================================================= - * File : archive_builder.cpp - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "archive_builder.h" - -#include "lib/timer.h" -#include "../file_internal.h" - -// un-nice dependencies: -#include "ps/Loader.h" - - -// vfs_load callback that compresses the data in parallel with IO -// (for incompressible files, we just calculate the checksum) -class Compressor -{ -public: - Compressor(uintptr_t ctx, const char* atom_fn, size_t usize) - : m_ctx(ctx) - , m_usize(usize) - , m_skipCompression(IsFileTypeIncompressible(atom_fn)) - , m_cdata(0), m_csize(0), m_checksum(0) - { - comp_reset(m_ctx); - m_csizeBound = comp_max_output_size(m_ctx, usize); - THROW_ERR(comp_alloc_output(m_ctx, m_csizeBound)); - } - - LibError Feed(const u8* ublock, size_t ublockSize, size_t* bytes_processed) - { - // comp_feed already makes note of total #bytes fed, and we need - // vfs_io to return the usize (to check if all data was read). - *bytes_processed = ublockSize; - - if(m_skipCompression) - { - // (since comp_finish returns the checksum, we only need to update this - // when not compressing.) - m_checksum = comp_update_checksum(m_ctx, m_checksum, ublock, ublockSize); - } - else - { - // note: we don't need the return value because comp_finish - // will tell us the total csize. - (void)comp_feed(m_ctx, ublock, ublockSize); - } - - return INFO::CB_CONTINUE; - } - - LibError Finish() - { - if(m_skipCompression) - return INFO::OK; - - RETURN_ERR(comp_finish(m_ctx, &m_cdata, &m_csize, &m_checksum)); - debug_assert(m_csize <= m_csizeBound); - return INFO::OK; - } - - u32 Checksum() const - { - return m_checksum; - } - - // final decision on whether to store the file as compressed, - // given the observed compressed/uncompressed sizes. - bool IsCompressionProfitable() const - { - // file is definitely incompressible. - if(m_skipCompression) - return false; - - const float ratio = (float)m_usize / m_csize; - const ssize_t bytes_saved = (ssize_t)m_usize - (ssize_t)m_csize; - UNUSED2(bytes_saved); - - // tiny - store compressed regardless of savings. - // rationale: - // - CPU cost is negligible and overlapped with IO anyway; - // - reading from compressed files uses less memory because we - // don't need to allocate space for padding in the final buffer. - if(m_usize < 512) - return true; - - // large high-entropy file - store uncompressed. - // rationale: - // - any bigger than this and CPU time becomes a problem: it isn't - // necessarily hidden by IO time anymore. - if(m_usize >= 32*KiB && ratio < 1.02f) - return false; - - // we currently store everything else compressed. - return true; - } - - void GetOutput(const u8*& cdata, size_t& csize) const - { - debug_assert(!m_skipCompression); - debug_assert(m_cdata && m_csize); - cdata = m_cdata; - csize = m_csize; - - // note: no need to free cdata - it is owned by the - // compression context and can be reused. - } - -private: - static bool IsFileTypeIncompressible(const char* fn) - { - const char* ext = path_extension(fn); - - // this is a selection of file types that are certainly not - // further compressible. we need not include every type under the sun - - // this is only a slight optimization that avoids wasting time - // compressing files. the real decision as to cmethod is made based - // on attained compression ratio. - static const char* incompressible_exts[] = - { - "zip", "rar", - "jpg", "jpeg", "png", - "ogg", "mp3" - }; - - for(uint i = 0; i < ARRAY_SIZE(incompressible_exts); i++) - { - if(!strcasecmp(ext+1, incompressible_exts[i])) - return true; - } - - return false; - } - - - uintptr_t m_ctx; - size_t m_usize; - size_t m_csizeBound; - bool m_skipCompression; - - u8* m_cdata; - size_t m_csize; - u32 m_checksum; -}; - -static LibError compressor_feed_cb(uintptr_t cbData, - const u8* ublock, size_t ublockSize, size_t* bytes_processed) -{ - Compressor& compressor = *(Compressor*)cbData; - return compressor.Feed(ublock, ublockSize, bytes_processed); -} - - -static LibError read_and_compress_file(const char* atom_fn, uintptr_t ctx, - ArchiveEntry& ent, const u8*& file_contents, FileIOBuf& buf) // out -{ - struct stat s; - RETURN_ERR(vfs_stat(atom_fn, &s)); - const size_t usize = s.st_size; - // skip 0-length files. - // rationale: zip.cpp needs to determine whether a CDFH entry is - // a file or directory (the latter are written by some programs but - // not needed - they'd only pollute the file table). - // it looks like checking for usize=csize=0 is the safest way - - // relying on file attributes (which are system-dependent!) is - // even less safe. - // we thus skip 0-length files to avoid confusing them with directories. - if(!usize) - return INFO::SKIPPED; - - Compressor compressor(ctx, atom_fn, usize); - - // read file into newly allocated buffer and run compressor. - size_t usize_read; - const uint flags = 0; - RETURN_ERR(vfs_load(atom_fn, buf, usize_read, flags, compressor_feed_cb, (uintptr_t)&compressor)); - debug_assert(usize_read == usize); - - LibError ret = compressor.Finish(); - if(ret < 0) - { - file_buf_free(buf); - return ret; - } - - // store file info - ent.usize = (off_t)usize; - ent.mtime = s.st_mtime; - // .. ent.ofs is set by zip_archive_add_file - ent.flags = 0; - ent.atom_fn = atom_fn; - ent.checksum = compressor.Checksum(); - if(compressor.IsCompressionProfitable()) - { - ent.method = CM_DEFLATE; - size_t csize; - compressor.GetOutput(file_contents, csize); - ent.csize = (off_t)csize; - } - else - { - ent.method = CM_NONE; - ent.csize = (off_t)usize; - file_contents = buf; - } - - return INFO::OK; -} - - -//----------------------------------------------------------------------------- - -LibError archive_build_init(const char* P_archive_filename, Filenames V_fns, ArchiveBuildState* ab) -{ - RETURN_ERR(zip_archive_create(P_archive_filename, &ab->za)); - ab->ctx = comp_alloc(CT_COMPRESSION, CM_DEFLATE); - ab->V_fns = V_fns; - - // count number of files (needed to estimate progress) - for(ab->num_files = 0; ab->V_fns[ab->num_files]; ab->num_files++) {} - - ab->i = 0; - return INFO::OK; -} - - -int archive_build_continue(ArchiveBuildState* ab) -{ - const double end_time = get_time() + 200e-3; - - for(;;) - { - const char* V_fn = ab->V_fns[ab->i]; - if(!V_fn) - break; - - ArchiveEntry ent; const u8* file_contents; FileIOBuf buf; - if(read_and_compress_file(V_fn, ab->ctx, ent, file_contents, buf) == INFO::OK) - { - (void)zip_archive_add_file(ab->za, &ent, file_contents); - (void)file_buf_free(buf); - } - - ab->i++; - LDR_CHECK_TIMEOUT((int)ab->i, (int)ab->num_files); - } - - // note: this is currently known to fail if there are no files in the list - // - zlib.h says: Z_DATA_ERROR is returned if freed prematurely. - // safe to ignore. - comp_free(ab->ctx); ab->ctx = 0; - (void)zip_archive_finish(ab->za); - - return INFO::OK; -} - - -void archive_build_cancel(ArchiveBuildState* ab) -{ - // note: the GUI may call us even though no build was ever in progress. - // be sure to make all steps no-op if is zeroed (initial state) or - // no build is in progress. - - comp_free(ab->ctx); ab->ctx = 0; - if(ab->za) - (void)zip_archive_finish(ab->za); - memset(ab, 0, sizeof(*ab)); -} - - -LibError archive_build(const char* P_archive_filename, Filenames V_fns) -{ - ArchiveBuildState ab; - RETURN_ERR(archive_build_init(P_archive_filename, V_fns, &ab)); - for(;;) - { - int ret = archive_build_continue(&ab); - RETURN_ERR(ret); - if(ret == INFO::OK) - return INFO::OK; - } -} diff --git a/source/lib/res/file/archive/archive_builder.h b/source/lib/res/file/archive/archive_builder.h deleted file mode 100644 index 8b72163ab4..0000000000 --- a/source/lib/res/file/archive/archive_builder.h +++ /dev/null @@ -1,43 +0,0 @@ -/** - * ========================================================================= - * File : archive_builder.h - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_ARCHIVE_BUILDER -#define INCLUDED_ARCHIVE_BUILDER - -// array of pointers to VFS filenames (including path), terminated by a -// NULL entry. -typedef const char** Filenames; - -struct ZipArchive; - -// rationale: this is fairly lightweight and simple, so we don't bother -// making it opaque. -struct ArchiveBuildState -{ - ZipArchive* za; - uintptr_t ctx; - Filenames V_fns; - size_t num_files; // number of filenames in V_fns (excluding final 0) - size_t i; -}; - -extern LibError archive_build_init(const char* P_archive_filename, Filenames V_fns, - ArchiveBuildState* ab); - -// create an archive (overwriting previous file) and fill it with the given -// files. compression method is chosen intelligently based on extension and -// file entropy / achieved compression ratio. -extern int archive_build_continue(ArchiveBuildState* ab); - -extern void archive_build_cancel(ArchiveBuildState* ab); - -extern LibError archive_build(const char* P_archive_filename, Filenames V_fns); - -#endif // #ifndef INCLUDED_ARCHIVE_BUILDER diff --git a/source/lib/res/file/archive/compression.cpp b/source/lib/res/file/archive/compression.cpp deleted file mode 100644 index c9f797d2fa..0000000000 --- a/source/lib/res/file/archive/compression.cpp +++ /dev/null @@ -1,693 +0,0 @@ -/** - * ========================================================================= - * File : compression.cpp - * Project : 0 A.D. - * Description : interface for compressing/decompressing data streams. - * : currently implements "deflate" (RFC1951). - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "compression.h" - -#include - -#include "lib/res/mem.h" -#include "lib/allocators.h" -#include "lib/timer.h" -#include "../file_io.h" // IO_EOF - -#include - -// rationale: this layer allows for other compression methods/libraries -// besides ZLib. it also simplifies the interface for user code and -// does error checking, etc. - - -ERROR_ASSOCIATE(ERR::COMPRESSION_UNKNOWN_METHOD, "Unknown/unsupported compression method", -1); - - -// provision for removing all ZLib code (all inflate calls will fail). -// used for checking DLL dependency; might also simulate corrupt Zip files. -//#define NO_ZLIB - -#ifndef NO_ZLIB -# include "lib/external_libraries/zlib.h" -#else -// several switch statements are going to have all cases removed. -// squelch the corresponding warning. -# pragma warning(disable: 4065) -#endif - - -TIMER_ADD_CLIENT(tc_zip_inflate); -TIMER_ADD_CLIENT(tc_zip_memcpy); - - -//----------------------------------------------------------------------------- - -class ICodec -{ -public: - /** - * note: the implementation should not check whether any data remains - - * codecs are sometimes destroyed without completing a transfer. - **/ - virtual ~ICodec() - { - } - - /** - * @return an upper bound on the output size for the given amount of input. - * this is used when allocating a single buffer for the whole operation. - **/ - virtual size_t MaxOutputSize(size_t inSize) const = 0; - - /** - * clear all previous state and prepare for reuse. - * - * this is as if the object were destroyed and re-created, but more - * efficient since memory buffers can be kept, etc. - **/ - virtual LibError Reset() = 0; - - /** - * process (i.e. compress or decompress) data. - * - * @param outSize bytes remaining in the output buffer; shall not be zero. - * @param inConsumed, outProduced how many bytes in the input and - * output buffers were used. either or both of these can be zero if - * the input size is small or there's not enough output space. - **/ - virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) = 0; - - /** - * flush buffers and make sure all output has been produced. - * - * @param out, outSize - the entire output buffer. this assumes the - * output pointers passed to Process were contiguous; if not, these - * values will not be meaningful. - * @param checksum over all input data. - * @return error status for the entire operation. - **/ - virtual LibError Finish(u8*& out, size_t& outSize, u32& checksum) = 0; - - /** - * update a checksum to reflect the contents of a buffer. - * - * @param checksum the initial value (must be 0 on first call) - * @return the new checksum. - **/ - virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const = 0; -}; - - -//----------------------------------------------------------------------------- - -#ifndef NO_ZLIB - -class ZLibCodec : public ICodec -{ -protected: - ZLibCodec() - { - memset(&m_zs, 0, sizeof(m_zs)); - InitializeChecksum(); - } - - void InitializeChecksum() - { - m_checksum = crc32(0, 0, 0); - } - - typedef int ZEXPORT (*ZLibFunc)(z_streamp strm, int flush); - - static LibError LibError_from_zlib(int zlib_err, bool warn_if_failed = true) - { - LibError err = ERR::FAIL; - switch(zlib_err) - { - case Z_OK: - return INFO::OK; - case Z_STREAM_END: - err = ERR::IO_EOF; break; - case Z_MEM_ERROR: - err = ERR::NO_MEM; break; - case Z_DATA_ERROR: - err = ERR::CORRUPTED; break; - case Z_STREAM_ERROR: - err = ERR::INVALID_PARAM; break; - default: - err = ERR::FAIL; break; - } - - if(warn_if_failed) - DEBUG_WARN_ERR(err); - return err; - } - - static void WarnIfZLibError(int zlib_ret) - { - (void)LibError_from_zlib(zlib_ret, true); - } - - LibError Process(ZLibFunc func, int flush, const u8* in, const size_t inSize, u8* out, const size_t outSize, size_t& inConsumed, size_t& outConsumed) - { - m_zs.next_in = (Byte*)in; - m_zs.avail_in = (uInt)inSize; - m_zs.next_out = (Byte*)out; - m_zs.avail_out = (uInt)outSize; - - int ret = func(&m_zs, flush); - // sanity check: if ZLib reports end of stream, all input data - // must have been consumed. - if(ret == Z_STREAM_END) - { - debug_assert(m_zs.avail_in == 0); - ret = Z_OK; - } - - debug_assert(inSize >= m_zs.avail_in && outSize >= m_zs.avail_out); - inConsumed = inSize - m_zs.avail_in; - outConsumed = outSize - m_zs.avail_out; - - return LibError_from_zlib(ret); - } - - virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const - { - return (u32)crc32(checksum, in, (uInt)inSize); - } - - mutable z_stream m_zs; - - // note: z_stream does contain an 'adler' checksum field, but that's - // not updated in streams lacking a gzip header, so we'll have to - // calculate a checksum ourselves. - // adler32 is somewhat weaker than CRC32, but a more important argument - // is that we should use the latter for compatibility with Zip archives. - mutable u32 m_checksum; -}; - -class ZLibCompressor : public ZLibCodec -{ -public: - ZLibCompressor() - { - // note: with Z_BEST_COMPRESSION, 78% percent of - // archive builder CPU time is spent in ZLib, even though - // that is interleaved with IO; everything else is negligible. - // we therefore enable this only in final builds; during - // development, 1.5% bigger archives are definitely worth much - // faster build time. -#if CONFIG_FINAL - const int level = Z_BEST_COMPRESSION; -#else - const int level = Z_BEST_SPEED; -#endif - const int windowBits = -MAX_WBITS; // max window size; omit ZLib header - const int memLevel = 9; // max speed; total mem ~= 384KiB - const int strategy = Z_DEFAULT_STRATEGY; // normal data - not RLE - const int ret = deflateInit2(&m_zs, level, Z_DEFLATED, windowBits, memLevel, strategy); - debug_assert(ret == Z_OK); - } - - virtual ~ZLibCompressor() - { - const int ret = deflateEnd(&m_zs); - WarnIfZLibError(ret); - } - - virtual size_t MaxOutputSize(size_t inSize) const - { - return (size_t)deflateBound(&m_zs, (uLong)inSize); - } - - virtual LibError Reset() - { - ZLibCodec::InitializeChecksum(); - const int ret = deflateReset(&m_zs); - return LibError_from_zlib(ret); - } - - virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed) - { - m_checksum = UpdateChecksum(m_checksum, in, inSize); - return ZLibCodec::Process(deflate, 0, in, inSize, out, outSize, inConsumed, outConsumed); - } - - virtual LibError Finish(u8*& out, size_t& outSize, u32& checksum) - { - // notify zlib that no more data is forthcoming and have it flush output. - // our output buffer has enough space due to use of deflateBound; - // therefore, deflate must return Z_STREAM_END. - const int ret = deflate(&m_zs, Z_FINISH); - if(ret != Z_STREAM_END) - debug_warn("deflate: unexpected Z_FINISH behavior"); - - out = m_zs.next_out - m_zs.total_out; - outSize = m_zs.total_out; - checksum = m_checksum; - return INFO::OK; - } -}; - - -class ZLibDecompressor : public ZLibCodec -{ -public: - ZLibDecompressor() - { - const int windowBits = -MAX_WBITS; // max window size; omit ZLib header - const int ret = inflateInit2(&m_zs, windowBits); - debug_assert(ret == Z_OK); - } - - virtual ~ZLibDecompressor() - { - const int ret = inflateEnd(&m_zs); - WarnIfZLibError(ret); - } - - virtual size_t MaxOutputSize(size_t inSize) const - { - // relying on an upper bound for the output is a really bad idea for - // large files. archive formats store the uncompressed file sizes, - // so callers should use that when allocating the output buffer. - debug_assert(inSize < 1*MiB); - - // http://www.zlib.org/zlib_tech.html - return inSize*1032; - } - - virtual LibError Reset() - { - ZLibCodec::InitializeChecksum(); - const int ret = inflateReset(&m_zs); - return LibError_from_zlib(ret); - } - - virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed) - { - const LibError ret = ZLibCodec::Process(inflate, Z_SYNC_FLUSH, in, inSize, out, outSize, inConsumed, outConsumed); - m_checksum = UpdateChecksum(m_checksum, in, inSize); - return ret; - } - - virtual LibError Finish(u8*& out, size_t& outSize, u32& checksum) - { - // no action needed - decompression always flushes immediately. - - out = m_zs.next_out - m_zs.total_out; - outSize = m_zs.total_out; - checksum = m_checksum; - return INFO::OK; - } -}; - -#endif // #ifndef NO_ZLIB - - -//----------------------------------------------------------------------------- - -#include "lib/nommgr.h" // protect placement new - -class CodecFactory -{ -public: - ICodec* Create(ContextType type, CompressionMethod method) - { - debug_assert(type == CT_COMPRESSION || type == CT_DECOMPRESSION); - - switch(method) - { -#ifndef NO_ZLIB - case CM_DEFLATE: - if(type == CT_COMPRESSION) - { - cassert(sizeof(ZLibCompressor) <= MAX_CODEC_SIZE); - return new(AllocateMemory()) ZLibCompressor; - } - else - { - cassert(sizeof(ZLibDecompressor) <= MAX_CODEC_SIZE); - return new(AllocateMemory()) ZLibDecompressor; - } - break; -#endif - default: - WARN_ERR(ERR::COMPRESSION_UNKNOWN_METHOD); - return 0; - } - } - - void Destroy(ICodec* codec) - { - codec->~ICodec(); - m_allocator.Free((Allocator::value_type*)codec); - } - -private: - void* AllocateMemory() - { - void* mem = m_allocator.Allocate(); - if(!mem) - throw std::bad_alloc(); - return mem; - } - - // double: see explanation in SingleAllocator - static const size_t MAX_CODEC_SIZE = 100; - typedef SingleAllocator Allocator; - Allocator m_allocator; -}; - -#include "lib/mmgr.h" - - -//----------------------------------------------------------------------------- -// BufferManager - -class BufferManager -{ -public: - void Enqueue(const u8* data, size_t size) - { - // note: calling with inSize = 0 is allowed and just means - // we don't enqueue a new buffer. it happens when compressing - // newly decompressed data if nothing was output (due to a - // small compressed input buffer). - if(size != 0) - m_pendingBuffers.push_back(Buffer(data, size)); - } - - bool GetNext(const u8*& data, size_t& size) const - { - if(m_pendingBuffers.empty()) - return false; - const Buffer& buffer = m_pendingBuffers.front(); - data = buffer.RemainingData(); - size = buffer.RemainingSize(); - return true; - } - - void MarkAsProcessed(size_t numBytes) - { - Buffer& buffer = m_pendingBuffers.front(); - buffer.MarkAsProcessed(numBytes); - if(buffer.RemainingSize() == 0) - m_pendingBuffers.pop_front(); - } - - void Reset() - { - m_pendingBuffers.clear(); - } - -private: - class Buffer - { - public: - Buffer(const u8* data, size_t size) - : m_data(data), m_size(size), m_pos(0) - { - } - - const u8* RemainingData() const - { - return m_data + m_pos; - } - - size_t RemainingSize() const - { - return m_size - m_pos; - } - - void MarkAsProcessed(size_t numBytes) - { - m_pos += numBytes; - debug_assert(m_pos <= m_size); - - // everything has been consumed. (this buffer will now be - // destroyed by removing it from the deque) - if(m_pos == m_size) - return; - - // if there is any data left, the caller must have "choked" - // (i.e. filled their output buffer). - - // this buffer currently references data allocated by the caller. - if(!m_copy.get()) - { - // since we have to return and they could free it behind our - // back, we'll need to allocate a copy of the remaining data. - m_size = RemainingSize(); - m_copy.reset(new u8[m_size]); - cpu_memcpy(m_copy.get(), RemainingData(), m_size); - m_data = m_copy.get(); // must happen after cpu_memcpy - m_pos = 0; - } - } - - private: - const u8* m_data; - size_t m_size; - size_t m_pos; - boost::shared_ptr m_copy; - }; - - // note: a 'list' (deque is more efficient) is necessary. - // lack of output space can result in leftover input data; - // since we do not want Feed() to always have to check for and - // use up any previous remnants, we allow queuing them. - std::deque m_pendingBuffers; -}; - -//----------------------------------------------------------------------------- - -class Stream -{ -public: - Stream(ContextType type, CompressionMethod method) - : m_out(0), m_outSize(0), m_outPos(0) - , m_codec(m_codecFactory.Create(type, method)) - { - } - - ~Stream() - { - m_codecFactory.Destroy(m_codec); - } - - size_t MaxOutputSize(size_t inSize) const - { - return m_codec->MaxOutputSize(inSize); - } - - void Reset() - { - m_bufferManager.Reset(); - - m_out = 0; - m_outSize = 0; - m_outPos = 0; - - m_codec->Reset(); - } - - void SetOutput(u8* out, size_t outSize) - { - debug_assert(IsAllowableOutputBuffer(out, outSize)); - - m_out = out; - m_outSize = outSize; - m_outPos = 0; - } - - LibError AllocOutput(size_t size) - { - // notes: - // - this implementation allows reusing previous buffers if they - // are big enough, which reduces the number of allocations. - // - no further attempts to reduce allocations (e.g. by doubling - // the current size) are made; this strategy is enough. - // - Pool etc. cannot be used because files may be huge (larger - // than the address space of 32-bit systems). - - // no buffer or the previous one wasn't big enough: reallocate - if(!m_outMem.get() || m_outMemSize < size) - { - m_outMem.reset((u8*)page_aligned_alloc(size), PageAlignedDeleter(size)); - m_outMemSize = size; - } - - SetOutput(m_outMem.get(), size); - - return INFO::OK; - } - - ssize_t Feed(const u8* in, size_t inSize) - { - size_t outTotal = 0; // returned unless error occurs - - m_bufferManager.Enqueue(in, inSize); - - // work off any pending buffers and the new one - const u8* cdata; size_t csize; - while(m_bufferManager.GetNext(cdata, csize)) - { - if(m_outSize == m_outPos) // output buffer full; must not call Process - break; - - size_t inConsumed, outProduced; - LibError err = m_codec->Process(cdata, csize, m_out+m_outPos, m_outSize-m_outPos, inConsumed, outProduced); - if(err < 0) - return err; - - m_bufferManager.MarkAsProcessed(inConsumed); - outTotal += outProduced; - m_outPos += outProduced; - } - - return (ssize_t)outTotal; - } - - LibError Finish(u8*& out, size_t& outSize, u32& checksum) - { - return m_codec->Finish(out, outSize, checksum); - } - - u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const - { - return m_codec->UpdateChecksum(checksum, in, inSize); - } - -private: - // ICodec::Finish is allowed to assume that output buffers were identical - // or contiguous; we verify this here. - bool IsAllowableOutputBuffer(u8* out, size_t outSize) - { - // none yet established - if(m_out == 0 && m_outSize == 0 && m_outPos == 0) - return true; - - // same as last time (happens with temp buffers) - if(m_out == out && m_outSize == outSize) - return true; - - // located after the last buffer (note: not necessarily after - // the entire buffer; a lack of input can cause the output buffer - // to only partially be used before the next call.) - if((unsigned)(out - m_out) <= m_outSize) - return true; - - return false; - } - - BufferManager m_bufferManager; - - u8* m_out; - size_t m_outSize; - size_t m_outPos; - - boost::shared_ptr m_outMem; - size_t m_outMemSize; - - static CodecFactory m_codecFactory; - ICodec* m_codec; -}; - -/*static*/ CodecFactory Stream::m_codecFactory; - - -//----------------------------------------------------------------------------- - -#include "lib/nommgr.h" // protect placement new - -class StreamFactory -{ -public: - Stream* Create(ContextType type, CompressionMethod method) - { - void* mem = m_allocator.Allocate(); - if(!mem) - throw std::bad_alloc(); - return new(mem) Stream(type, method); - } - - void Destroy(Stream* stream) - { - stream->~Stream(); - m_allocator.Free(stream); - } - -private: - SingleAllocator m_allocator; -}; - -#include "lib/mmgr.h" - - -//----------------------------------------------------------------------------- - -static StreamFactory streamFactory; - -uintptr_t comp_alloc(ContextType type, CompressionMethod method) -{ - Stream* stream = streamFactory.Create(type, method); - return (uintptr_t)stream; -} - -void comp_free(uintptr_t ctx) -{ - // no-op if context is 0 (i.e. was never allocated) - if(!ctx) - return; - - Stream* stream = (Stream*)ctx; - streamFactory.Destroy(stream); -} - -void comp_reset(uintptr_t ctx) -{ - Stream* stream = (Stream*)ctx; - stream->Reset(); -} - -size_t comp_max_output_size(uintptr_t ctx, size_t inSize) -{ - Stream* stream = (Stream*)ctx; - return stream->MaxOutputSize(inSize); -} - -void comp_set_output(uintptr_t ctx, u8* out, size_t outSize) -{ - Stream* stream = (Stream*)ctx; - stream->SetOutput(out, outSize); -} - -LibError comp_alloc_output(uintptr_t ctx, size_t inSize) -{ - Stream* stream = (Stream*)ctx; - return stream->AllocOutput(inSize); -} - -ssize_t comp_feed(uintptr_t ctx, const u8* in, size_t inSize) -{ - Stream* stream = (Stream*)ctx; - return stream->Feed(in, inSize); -} - -LibError comp_finish(uintptr_t ctx, u8** out, size_t* outSize, u32* checksum) -{ - Stream* stream = (Stream*)ctx; - return stream->Finish(*out, *outSize, *checksum); -} - -u32 comp_update_checksum(uintptr_t ctx, u32 checksum, const u8* in, size_t inSize) -{ - Stream* stream = (Stream*)ctx; - return stream->UpdateChecksum(checksum, in, inSize); -} diff --git a/source/lib/res/file/archive/compression.h b/source/lib/res/file/archive/compression.h deleted file mode 100644 index db8e6fee28..0000000000 --- a/source/lib/res/file/archive/compression.h +++ /dev/null @@ -1,118 +0,0 @@ -/** - * ========================================================================= - * File : compression.h - * Project : 0 A.D. - * Description : interface for compressing/decompressing data streams. - * : currently implements "deflate" (RFC1951). - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_COMPRESSION -#define INCLUDED_COMPRESSION - - -namespace ERR -{ - const LibError COMPRESSION_UNKNOWN_METHOD = -110300; -} - -enum ContextType -{ - CT_COMPRESSION, - CT_DECOMPRESSION -}; - -enum CompressionMethod -{ - CM_NONE, - - // zlib "deflate" (RFC 1750, 1751) and CRC32 - CM_DEFLATE, - - CM_UNSUPPORTED -}; - -/** - * allocate a new compression/decompression context. - **/ -extern uintptr_t comp_alloc(ContextType type, CompressionMethod method); - -/** - * free this context and all associated memory. - **/ -extern void comp_free(uintptr_t ctx); - -/** - * clear all previous state and prepare for reuse. - * - * this is as if the object were destroyed and re-created, but more - * efficient since it avoids reallocating a considerable amount of memory - * (about 200KB for LZ). - **/ -extern void comp_reset(uintptr_t ctx); - -/** - * @return an upper bound on the output size for the given amount of input. - * this is used when allocating a single buffer for the whole operation. - **/ -extern size_t comp_max_output_size(uintptr_t ctx, size_t inSize); - -/** - * set output buffer for subsequent comp_feed() calls. - * - * due to the comp_finish interface, output buffers must be contiguous or - * identical (otherwise IsAllowableOutputBuffer will complain). - **/ -extern void comp_set_output(uintptr_t ctx, u8* out, size_t outSize); - -/** - * allocate a new output buffer. - * - * @param size [bytes] to allocate. - * - * if a buffer had previously been allocated and is large enough, it is - * reused (this reduces the number of allocations). the buffer is - * automatically freed by comp_free. - **/ -extern LibError comp_alloc_output(uintptr_t ctx, size_t inSize); - -/** - * 'feed' the given buffer to the compressor/decompressor. - * - * @return number of output bytes produced or a negative LibError. - * note that 0 is a legitimate return value - this happens if the input - * buffer is small and the codec hasn't produced any output. - * - * note: after this call returns, the buffer may be overwritten or freed; - * we take care of copying and queuing any data that remains (e.g. due to - * lack of output buffer space). - **/ -extern ssize_t comp_feed(uintptr_t ctx, const u8* in, size_t inSize); - -/** - * conclude the compression/decompression operation. - * - * @param out, outSize receive the output buffer. this assumes identical or - * contiguous addresses were passed, which comp_set_output ensures. - * @param checksum - * - * note: this must always be called (even if the output buffer is already - * known) because it feeds any remaining queued input buffers. - **/ -extern LibError comp_finish(uintptr_t ctx, u8** out, size_t* out_size, u32* checksum); - -/** - * update a checksum to reflect the contents of a buffer. - * - * @param checksum the initial value (must be 0 on first call) - * @return the new checksum. - * - * note: this routine is stateless but still requires a context to establish - * the type of checksum to calculate. the results are the same as yielded by - * comp_finish after comp_feed-ing all input buffers. - **/ -extern u32 comp_update_checksum(uintptr_t ctx, u32 checksum, const u8* in, size_t inSize); - -#endif // #ifndef INCLUDED_COMPRESSION diff --git a/source/lib/res/file/archive/tests/test_archive_builder.h b/source/lib/res/file/archive/tests/test_archive_builder.h deleted file mode 100644 index 3239072889..0000000000 --- a/source/lib/res/file/archive/tests/test_archive_builder.h +++ /dev/null @@ -1,146 +0,0 @@ -#include "lib/self_test.h" - -#include "lib/base32.h" -#include "lib/res/file/path.h" -#include "lib/res/file/file.h" -#include "lib/res/file/file_cache.h" -#include "lib/res/file/vfs.h" -#include "lib/res/file/archive/archive.h" -#include "lib/res/file/archive/archive_builder.h" -#include "lib/res/h_mgr.h" -#include "lib/res/mem.h" -#include "lib/rand.h" - -class TestArchiveBuilder : public CxxTest::TestSuite -{ - const char* const archive_fn; - static const size_t NUM_FILES = 30; - static const size_t MAX_FILE_SIZE = 20000; - - std::set existing_names; - const char* gen_random_name() - { - // 10 chars is enough for (10-1)*5 bits = 45 bits > u32 - char name_tmp[10]; - - for(;;) - { - u32 rand_num = rand(0, 100000); - base32(4, (const u8*)&rand_num, (u8*)name_tmp); - - // store filename in atom pool - const char* atom_fn = file_make_unique_fn_copy(name_tmp); - // done if the filename is unique (not been generated yet) - if(existing_names.find(atom_fn) == existing_names.end()) - { - existing_names.insert(atom_fn); - return atom_fn; - } - } - } - - struct TestFile - { - off_t size; - u8* data; // must be delete[]-ed after comparing - }; - // (must be separate array and end with NULL entry (see Filenames)) - const char* filenames[NUM_FILES+1]; - TestFile files[NUM_FILES]; - - void generate_random_files() - { - for(size_t i = 0; i < NUM_FILES; i++) - { - const off_t size = rand(0, MAX_FILE_SIZE); - u8* data = new u8[size]; - - // random data won't compress at all, and we want to exercise - // the uncompressed codepath as well => make some of the files - // easily compressible (much less values). - const bool make_easily_compressible = (rand(0, 100) > 50); - if(make_easily_compressible) - { - for(off_t i = 0; i < size; i++) - data[i] = rand() & 0x0F; - } - else - { - for(off_t i = 0; i < size; i++) - data[i] = rand() & 0xFF; - } - - filenames[i] = gen_random_name(); - files[i].size = size; - files[i].data = data; - - ssize_t bytes_written = vfs_store(filenames[i], data, size, FILE_NO_AIO); - TS_ASSERT_EQUALS(bytes_written, size); - } - - // 0-terminate the list - see Filenames decl. - filenames[NUM_FILES] = NULL; - } - -public: - TestArchiveBuilder() - : archive_fn("test_archive_random_data.zip") {} - - void setUp() - { - (void)file_init(); - (void)file_set_root_dir(0, "."); - vfs_init(); - } - - void tearDown() - { - vfs_shutdown(); - file_shutdown(); - path_reset_root_dir(); - } - - void test_create_archive_with_random_files() - { - if(!file_exists("archivetest")) // don't get stuck if this test fails and never deletes the directory it created - TS_ASSERT_OK(dir_create("archivetest")); - - TS_ASSERT_OK(vfs_mount("", "archivetest")); - - generate_random_files(); - TS_ASSERT_OK(archive_build(archive_fn, filenames)); - - // wipe out file cache, otherwise we're just going to get back - // the file contents read during archive_build . - file_cache_reset(); - - // read in each file and compare file contents - Handle ha = archive_open(archive_fn); - TS_ASSERT(ha > 0); - for(size_t i = 0; i < NUM_FILES; i++) - { - File f; - TS_ASSERT_OK(afile_open(ha, filenames[i], 0, 0, &f)); - FileIOBuf buf = FILE_BUF_ALLOC; - ssize_t bytes_read = afile_read(&f, 0, files[i].size, &buf); - TS_ASSERT_EQUALS(bytes_read, files[i].size); - - TS_ASSERT_SAME_DATA(buf, files[i].data, files[i].size); - - TS_ASSERT_OK(file_buf_free(buf)); - TS_ASSERT_OK(afile_close(&f)); - SAFE_ARRAY_DELETE(files[i].data); - } - TS_ASSERT_OK(archive_close(ha)); - - dir_delete("archivetest"); - file_delete(archive_fn); - } - - void test_multiple_init_shutdown() - { - // setUp has already vfs_init-ed it and tearDown will vfs_shutdown. - vfs_shutdown(); - vfs_init(); - } -}; diff --git a/source/lib/res/file/archive/tests/test_compression.h b/source/lib/res/file/archive/tests/test_compression.h deleted file mode 100644 index 94edea435b..0000000000 --- a/source/lib/res/file/archive/tests/test_compression.h +++ /dev/null @@ -1,54 +0,0 @@ -#include "lib/self_test.h" - -#include "lib/self_test.h" -#include "lib/res/file/archive/compression.h" - -class TestCompression : public CxxTest::TestSuite -{ -public: - void test_compress_decompress_compare() - { - // generate random input data - // (limit values to 0..7 so that the data will actually be compressible) - const size_t data_size = 10000; - u8 data[data_size]; - for(size_t i = 0; i < data_size; i++) - data[i] = rand() & 0x07; - - u8* cdata; size_t csize; - u8 udata[data_size]; - - // compress - uintptr_t c = comp_alloc(CT_COMPRESSION, CM_DEFLATE); - { - TS_ASSERT(c != 0); - const size_t csizeBound = comp_max_output_size(c, data_size); - TS_ASSERT_OK(comp_alloc_output(c, csizeBound)); - const ssize_t cdata_produced = comp_feed(c, data, data_size); - TS_ASSERT(cdata_produced >= 0); - u32 checksum; - TS_ASSERT_OK(comp_finish(c, &cdata, &csize, &checksum)); - TS_ASSERT(cdata_produced <= (ssize_t)csize); // can't have produced more than total - } - - // decompress - uintptr_t d = comp_alloc(CT_DECOMPRESSION, CM_DEFLATE); - { - TS_ASSERT(d != 0); - comp_set_output(d, udata, data_size); - const ssize_t udata_produced = comp_feed(d, cdata, csize); - TS_ASSERT(udata_produced >= 0); - u8* udata_final; size_t usize_final; u32 checksum; - TS_ASSERT_OK(comp_finish(d, &udata_final, &usize_final, &checksum)); - TS_ASSERT(udata_produced <= (ssize_t)usize_final); // can't have produced more than total - TS_ASSERT_EQUALS(udata_final, udata); // output buffer address is same - TS_ASSERT_EQUALS(usize_final, data_size); // correct amount of output - } - - comp_free(c); - comp_free(d); - - // verify data survived intact - TS_ASSERT_SAME_DATA(data, udata, data_size); - } -}; diff --git a/source/lib/res/file/archive/tests/test_zip.h b/source/lib/res/file/archive/tests/test_zip.h deleted file mode 100644 index d052434ec4..0000000000 --- a/source/lib/res/file/archive/tests/test_zip.h +++ /dev/null @@ -1,25 +0,0 @@ -#include "lib/self_test.h" - -#include - -#include "lib/res/file/archive/zip.h" - -class TestZip : public CxxTest::TestSuite -{ -public: - void test_fat_timedate_conversion() - { - // note: FAT time stores second/2, which means converting may - // end up off by 1 second. - - time_t t, converted_t; - - t = time(0); - converted_t = time_t_from_FAT(FAT_from_time_t(t)); - TS_ASSERT_DELTA(t, converted_t, 2); - - t++; - converted_t = time_t_from_FAT(FAT_from_time_t(t)); - TS_ASSERT_DELTA(t, converted_t, 2); - } -}; diff --git a/source/lib/res/file/archive/trace.cpp b/source/lib/res/file/archive/trace.cpp deleted file mode 100644 index 9022d15ab4..0000000000 --- a/source/lib/res/file/archive/trace.cpp +++ /dev/null @@ -1,473 +0,0 @@ -/** - * ========================================================================= - * File : trace.cpp - * Project : 0 A.D. - * Description : allows recording and 'playing back' a sequence of - * : I/Os - useful for benchmarking and archive builder. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "trace.h" - -#include "lib/allocators.h" -#include "lib/timer.h" -#include "lib/sysdep/cpu.h" -#include "../file_internal.h" - - -ERROR_ASSOCIATE(ERR::TRACE_EMPTY, "No valid entries in trace", -1); - - -static uintptr_t trace_initialized; // set via CAS -static Pool trace_pool; - -// call at before using trace_pool. no-op if called more than once. -static inline void trace_init() -{ - if(cpu_CAS(&trace_initialized, 0, 1)) - (void)pool_create(&trace_pool, 4*MiB, sizeof(TraceEntry)); -} - -void trace_shutdown() -{ - if(cpu_CAS(&trace_initialized, 1, 0)) - (void)pool_destroy(&trace_pool); -} - - -// enabled by default. by the time we can decide whether a trace needs to -// be generated (see should_rebuild_main_archive), file accesses will -// already have occurred; hence default enabled and disable if not needed. -static bool trace_enabled = true; -static bool trace_force_enabled = false; // see below - -// note: explicitly enabling trace means the user wants one to be -// generated even if an up-to-date version exists. -// (mechanism: ignore any attempts to disable) -void trace_enable(bool want_enabled) -{ - trace_enabled = want_enabled; - - if(want_enabled) - trace_force_enabled = true; - if(trace_force_enabled) - trace_enabled = true; -} - - -static LibError trace_add(TraceOp op, const char* P_fn, size_t size, - uint flags = 0, double timestamp = 0.0) -{ - trace_init(); - if(!trace_enabled) - return INFO::OK; - - if(timestamp == 0.0) - timestamp = get_time(); - - TraceEntry* t = (TraceEntry*)pool_alloc(&trace_pool, 0); - if(!t) - return ERR::LIMIT; // NOWARN - t->timestamp = timestamp; - t->atom_fn = file_make_unique_fn_copy(P_fn); - t->size = size; - t->op = op; - t->flags = flags; - return INFO::OK; -} - -static void trace_get_raw_ents(const TraceEntry*& ents, size_t& num_ents) -{ - ents = (const TraceEntry*)trace_pool.da.base; - num_ents = (uint)(trace_pool.da.pos / sizeof(TraceEntry)); -} - - -void trace_notify_io(const char* P_fn, size_t size, uint flags) -{ - trace_add(TO_IO, P_fn, size, flags); -} - -void trace_notify_free(const char* P_fn, size_t size) -{ - trace_add(TO_FREE, P_fn, size); -} - - -//----------------------------------------------------------------------------- - -// put all entries in one trace file: easier to handle; obviates FS enum code -// rationale: don't go through trace in order; instead, process most recent -// run first, to give more weight to it (TSP code should go with first entry -// when #occurrences are equal) - - -static const TraceEntry delimiter_entry = -{ - 0.0f, // timestamp - "------------------------------------------------------------", - 0, // size - TO_IO, // TraceOp (never seen by user; value doesn't matter) - 0 // flags -}; - -// storage for Trace.runs. -static const uint MAX_RUNS = 100; -static TraceRun runs[MAX_RUNS]; - -// note: the last entry may be one past number of actual entries. -// WARNING: due to misfeature in DelimiterAdder, indices are added twice. -// this is fixed in trace_get; just don't rely on run_start_indices.size()! -static std::vector run_start_indices; - -class DelimiterAdder -{ -public: - enum Consequence - { - SKIP_ADD, - CONTINUE - }; - Consequence operator()(size_t i, double timestamp, const char* P_path) - { - // this entry is a delimiter - if(!strcmp(P_path, delimiter_entry.atom_fn)) - { - run_start_indices.push_back(i+1); // skip this entry - // note: its timestamp is invalid, so don't set cur_timestamp! - return SKIP_ADD; - } - - const double last_timestamp = cur_timestamp; - cur_timestamp = timestamp; - - // first item is always start of a run - if((i == 0) || - // timestamp started over from 0 (e.g. 29, 30, 1) -> start of new run. - (timestamp < last_timestamp)) - run_start_indices.push_back(i); - - return CONTINUE; - } -private: - double cur_timestamp; -}; - - -//----------------------------------------------------------------------------- - - -void trace_get(Trace* t) -{ - const TraceEntry* ents; size_t num_ents; - trace_get_raw_ents(ents, num_ents); - - // nobody had split ents up into runs; just create one big 'run'. - if(run_start_indices.empty()) - run_start_indices.push_back(0); - - t->runs = runs; - t->num_runs = 0; // counted up - t->total_ents = num_ents; - - size_t last_start_idx = num_ents; - - std::vector::reverse_iterator it; - for(it = run_start_indices.rbegin(); it != run_start_indices.rend(); ++it) - { - const size_t start_idx = *it; - // run_start_indices.back() may be = num_ents (could happen if - // a zero-length run gets written out); skip that to avoid - // zero-length run here. - // also fixes DelimiterAdder misbehavior of adding 2 indices per run. - if(last_start_idx == start_idx) - continue; - - debug_assert(start_idx < t->total_ents); - - TraceRun& run = runs[t->num_runs++]; - run.num_ents = last_start_idx - start_idx; - run.ents = &ents[start_idx]; - last_start_idx = start_idx; - - if(t->num_runs == MAX_RUNS) - break; - } - - debug_assert(t->num_runs != 0); -} - -void trace_clear() -{ - pool_free_all(&trace_pool); - run_start_indices.clear(); - memset(runs, 0, sizeof(runs)); // for safety -} - -//----------------------------------------------------------------------------- - - - -static void write_entry(FILE* f, const TraceEntry* ent) -{ - char opcode = '?'; - switch(ent->op) - { - case TO_IO: opcode = 'L'; break; - case TO_FREE: opcode = 'F'; break; - default: debug_warn("invalid TraceOp"); - } - - debug_assert(ent->op == TO_IO || ent->op == TO_FREE); - fprintf(f, "%#010f: %c \"%s\" %d %04x\n", ent->timestamp, opcode, - ent->atom_fn, ent->size, ent->flags); -} - - -// *appends* entire current trace contents to file (with delimiter first) -LibError trace_write_to_file(const char* trace_filename) -{ - if(!trace_enabled) - return INFO::SKIPPED; - - char N_fn[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(trace_filename, N_fn)); - // append at end of file, otherwise we'd only have the most - // recently stored trace. vfs_optimizer correctly deals with - // several trace runs per file. - FILE* f = fopen(N_fn, "at"); - if(!f) - WARN_RETURN(ERR::FILE_ACCESS); - - write_entry(f, &delimiter_entry); - - // somewhat of a hack: write all entries in original order, not the - // reverse order returned by trace_get. - const TraceEntry* ent; size_t num_ents; - trace_get_raw_ents(ent, num_ents); - for(size_t i = 0; i < num_ents; i++, ent++) - write_entry(f, ent); - - (void)fclose(f); - return INFO::OK; -} - - -LibError trace_read_from_file(const char* trace_filename, Trace* t) -{ - trace_clear(); - - char N_fn[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(trace_filename, N_fn)); - FILE* f = fopen(N_fn, "rt"); - if(!f) - WARN_RETURN(ERR::TNODE_NOT_FOUND); - - // we use trace_add, which is the same mechanism called by trace_notify*; - // therefore, tracing needs to be enabled. - trace_enabled = true; - - DelimiterAdder delim_adder; - - // parse lines and stuff them in trace_pool - // (as if they had been trace_add-ed; replaces any existing data) - // .. bake PATH_MAX limit into string. - char fmt[30]; - snprintf(fmt, ARRAY_SIZE(fmt), "%%lf: %%c \"%%%d[^\"]\" %%d %%04x\n", PATH_MAX); - for(size_t i = 0; ; i++) - { - double timestamp; char opcode; char P_path[PATH_MAX]; size_t size; uint flags; - int chars_read = fscanf(f, fmt, ×tamp, &opcode, P_path, &size, &flags); - if(chars_read == EOF) - break; - debug_assert(chars_read == 5); - - TraceOp op = TO_IO; // default in case file is garbled - switch(opcode) - { - case 'L': op = TO_IO; break; - case 'F': op = TO_FREE; break; - default: debug_warn("invalid TraceOp"); - } - - if(delim_adder(i, timestamp, P_path) != DelimiterAdder::SKIP_ADD) - { - LibError ret = trace_add(op, P_path, size, flags, timestamp); - // storage in trace pool exhausted. must abort to avoid later - // adding delimiters for items that weren't actually stored - // into the pool. - if(ret == ERR::LIMIT) - break; - } - } - - fclose(f); - - trace_get(t); - - // all previous trace entries were hereby lost (overwritten), - // so there's no sense in continuing. - trace_enabled = false; - - if(t->total_ents == 0) - WARN_RETURN(ERR::TRACE_EMPTY); - - return INFO::OK; -} - - -void trace_gen_random(size_t num_entries) -{ - trace_clear(); - - for(size_t i = 0; i < num_entries; i++) - { - // generate random names until we get a valid file; - // remember its name and size. - const char* atom_fn; - off_t size; - for(;;) - { - atom_fn = file_get_random_name(); - // use instead of vfs_stat to avoid warnings, since some of - // atom_fn will actually be directory names. - if(vfs_exists(atom_fn)) - { - struct stat s; - LibError ret = vfs_stat(atom_fn, &s); - // ought to apply due to vfs_exists above. - debug_assert(ret == INFO::OK && S_ISREG(s.st_mode)); - - size = s.st_size; - break; - } - } - - trace_add(TO_IO, atom_fn, size); - trace_add(TO_FREE, atom_fn, size); - } -} - - -//----------------------------------------------------------------------------- - -// simulate carrying out the entry's TraceOp to determine -// whether this IO would be satisfied by the file_buf cache. -// -// note: TO_IO's handling of uncached buffers means the simulated and -// real cache contents will diverge if the real caller doesn't free their -// buffer immediately. -// this is a bit of a bother, but only slightly influences results -// because it works by affecting the cache allocator's eviction pattern. -// alternatives: -// - only allocate if file_cache_would_add. this would actually -// cause divergence whenever skipping any allocation, which is worse. -// - maintain a list of "buffers we allocated" and use that instead of -// file_cache_retrieve in TO_FREE. this would keep both caches in sync but -// add considerable complexity (function would no longer be "stateless"). -bool trace_entry_causes_io(const TraceEntry* ent) -{ - uint fb_flags = FB_NO_STATS; - if(ent->flags & FILE_LONG_LIVED) - fb_flags |= FB_LONG_LIVED; - - FileIOBuf buf; - size_t size = ent->size; - const char* atom_fn = ent->atom_fn; - uint file_flags = ent->flags; - switch(ent->op) - { - case TO_IO: - { - // we're not interested in writes - if(file_flags & FILE_WRITE) - return false; - buf = file_cache_retrieve(atom_fn, &size, fb_flags); - // would not be in cache - if(!buf) - { - buf = file_buf_alloc(size, atom_fn, fb_flags); - LibError ret = file_cache_add(buf, size, atom_fn, file_flags); - // the cache decided not to add buf (see file_cache_would_add). - // since TO_FREE below uses the cache to find out which - // buffer was allocated for atom_fn, we have to free it manually. - // see note above. - if(ret == INFO::SKIPPED) - (void)file_buf_free(buf, fb_flags); - return true; - } - break; - } - case TO_FREE: - buf = file_cache_retrieve(atom_fn, &size, fb_flags|FB_NO_ACCOUNTING); - // note: if buf == 0, file_buf_free is a no-op. this happens in the - // abovementioned cached-at-higher-level case. - (void)file_buf_free(buf, fb_flags); - break; - default: - debug_warn("unknown TraceOp"); - } - - return false; -} - - -// carry out all operations specified in the trace. -// if flags&TRF_SYNC_TO_TIMESTAMP, waits until timestamp for each event is -// reached; otherwise, they are run as fast as possible. -LibError trace_run(const char* trace_filename, uint flags) -{ - Trace t; - RETURN_ERR(trace_read_from_file(trace_filename, &t)); - - // prevent the actions we carry out below from generating - // trace_add-s. - trace_enabled = false; - - const double start_time = get_time(); - const double first_timestamp = t.runs[t.num_runs-1].ents[0].timestamp; - - for(uint r = 0; r < t.num_runs; r++) - { - const TraceRun& run = t.runs[r]; - const TraceEntry* ent = run.ents; - for(uint i = 0; i < run.num_ents; i++, ent++) - { - // wait until time for next entry if caller requested this - if(flags & TRF_SYNC_TO_TIMESTAMP) - { - while(get_time()-start_time < ent->timestamp-first_timestamp) - { - // busy-wait (don't sleep - can skew results) - } - } - - // carry out this entry's operation - FileIOBuf buf; size_t size; - switch(ent->op) - { - case TO_IO: - // do not 'run' writes - we'd destroy the existing data. - if(ent->flags & FILE_WRITE) - continue; - (void)vfs_load(ent->atom_fn, buf, size, ent->flags); - break; - case TO_FREE: - buf = file_cache_retrieve(ent->atom_fn, &size, FB_NO_STATS|FB_NO_ACCOUNTING); - (void)file_buf_free(buf); - break; - default: - debug_warn("unknown TraceOp"); - } - } - } - - trace_clear(); - - return INFO::OK; -} - - diff --git a/source/lib/res/file/archive/trace.h b/source/lib/res/file/archive/trace.h deleted file mode 100644 index 0e39f2ac15..0000000000 --- a/source/lib/res/file/archive/trace.h +++ /dev/null @@ -1,91 +0,0 @@ -/** - * ========================================================================= - * File : trace.h - * Project : 0 A.D. - * Description : allows recording and 'playing back' a sequence of - * : I/Os - useful for benchmarking and archive builder. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_TRACE -#define INCLUDED_TRACE - -namespace ERR -{ - const LibError TRACE_EMPTY = -110500; -} - -extern void trace_enable(bool want_enabled); -extern void trace_shutdown(); - -extern void trace_notify_io(const char* P_fn, size_t size, uint flags); -extern void trace_notify_free(const char* P_fn, size_t size); - -// TraceEntry operation type. -// note: rather than only a list of accessed files, we also need to -// know the application's behavior WRT caching (e.g. when it releases -// cached buffers). this is necessary so that our simulation can -// yield the same behavior. -enum TraceOp -{ - TO_IO, - TO_FREE, -}; - -// stores one event that is relevant for file IO / caching. -// -// size-optimized a bit since these are all kept in memory -// (to prevent trace file writes from affecting other IOs) -struct TraceEntry -{ - // note: float instead of double for nice 16 byte struct size - float timestamp; // returned by get_time before operation starts - const char* atom_fn; // path+name of affected file - // rationale: store size in the trace because other applications - // that use this trace format but not our IO code wouldn't know - // size (since they cannot retrieve the file info given atom_fn). - size_t size; // of IO (usually the entire file) - uint op : 8; // operation - see TraceOp - uint flags : 24; // misc, e.g. file_io flags. -}; - -struct TraceRun -{ - const TraceEntry* ents; - size_t num_ents; -}; - -struct Trace -{ - // most recent first! (see rationale in source) - const TraceRun* runs; - size_t num_runs; - - size_t total_ents; -}; - -extern void trace_get(Trace* t); -extern LibError trace_write_to_file(const char* trace_filename); -extern LibError trace_read_from_file(const char* trace_filename, Trace* t); - -extern void trace_gen_random(size_t num_entries); - - -// simulate carrying out the entry's TraceOp to determine -// whether this IO would be satisfied by the file_buf cache. -extern bool trace_entry_causes_io(const TraceEntry* ent); - -enum TraceRunFlags -{ - TRF_SYNC_TO_TIMESTAMP = 1 -}; - -// carry out all operations specified in the trace. -// if flags&TRF_SYNC_TO_TIMESTAMP, waits until timestamp for each event is -// reached; otherwise, they are run as fast as possible. -extern LibError trace_run(const char* trace_filename, uint flags = 0); - - -#endif // #ifndef INCLUDED_TRACE diff --git a/source/lib/res/file/archive/vfs_optimizer.cpp b/source/lib/res/file/archive/vfs_optimizer.cpp deleted file mode 100644 index 8feb16a962..0000000000 --- a/source/lib/res/file/archive/vfs_optimizer.cpp +++ /dev/null @@ -1,731 +0,0 @@ -/** - * ========================================================================= - * File : vfs_optimizer.cpp - * Project : 0 A.D. - * Description : automatically bundles files into archives in order of - * : access to optimize I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "vfs_optimizer.h" - -#include -#include -#include -#include - -#include "../file_internal.h" - -// enough for 64K unique files - ought to suffice. -typedef u16 FileId; -static const FileId NULL_ID = 0; -static const size_t MAX_IDS = 0x10000 -1; // -1 due to NULL_ID - - -struct FileNode -{ - const char* atom_fn; - - FileId prev_id; - FileId next_id; - u32 visited : 1; - u32 output : 1; - - FileNode(const char* atom_fn_) - { - atom_fn = atom_fn_; - - prev_id = next_id = NULL_ID; - visited = output = 0; - } -}; - -typedef std::vector FileNodes; - -//----------------------------------------------------------------------------- - -// check if the file is supposed to be added to archive. -// this avoids adding e.g. screenshots (wasteful because they're never used) -// or config (bad because they are written to and that's not supported for -// archived files). -static bool is_archivable(const TFile* tf) -{ - const Mount* m = tfile_get_mount(tf); - return mount_is_archivable(m); -} - -class IdMgr -{ - FileId cur; - typedef std::map Map; - Map map; - FileNodes* nodes; - - // dummy return value so this can be called via for_each/mem_fun_ref - void associate_node_with_fn(const FileNode& node) - { - FileId id = id_from_node(&node); - const Map::value_type item = std::make_pair(node.atom_fn, id); - std::pair ret = map.insert(item); - if(!ret.second) - debug_warn("atom_fn already associated with node"); - } - -public: - FileId id_from_node(const FileNode* node) const - { - // +1 to skip NULL_ID value - FileId id = node - &((*nodes)[0]) +1; - debug_assert(id <= nodes->size()); - return id; - } - - FileNode* node_from_id(FileId id) const - { - debug_assert(id != NULL_ID); - return &(*nodes)[id-1]; - } - - FileId id_from_fn(const char* atom_fn) const - { - Map::const_iterator cit = map.find(atom_fn); - if(cit == map.end()) - { - debug_warn("id_from_fn: not found"); - return NULL_ID; - } - return cit->second; - } - - void init(FileNodes* nodes_) - { - cur = NULL_ID+1; - map.clear(); - nodes = nodes_; - - // can't use for_each (mem_fun requires const function and - // non-reference-type argument) - for(FileNodes::const_iterator cit = nodes->begin(); cit != nodes->end(); ++cit) - { - const FileNode& node = *cit; - associate_node_with_fn(node); - } - } -}; -static IdMgr id_mgr; - -//----------------------------------------------------------------------------- - -// build list of FileNode - exactly one per file in VFS. -// -// time cost: 13ms for 5500 files; we therefore do not bother with -// optimizations like reading from vfs_tree container directly. -class FileGatherer -{ - static void EntCb(const char* path, const DirEnt* ent, uintptr_t cbData) - { - FileNodes* file_nodes = (FileNodes*)cbData; - - // we only want files - if(DIRENT_IS_DIR(ent)) - return; - - if(is_archivable(ent->tf)) - { - const char* atom_fn = file_make_unique_fn_copy(path); - file_nodes->push_back(FileNode(atom_fn)); - } - } - -public: - FileGatherer(FileNodes& file_nodes) - { - // jump-start allocation (avoids frequent initial reallocs) - file_nodes.reserve(500); - - // TODO: only add entries from mount points that have - // VFS_MOUNT_ARCHIVE flag set (avoids adding screenshots etc.) - vfs_dir_enum("", VFS_DIR_RECURSIVE, 0, EntCb, (uintptr_t)&file_nodes); - - // MAX_IDS is a rather large limit on number of files, but must not - // be exceeded (otherwise FileId overflows). - // check for this here and not in EntCb because it's not - // expected to happen. - if(file_nodes.size() > MAX_IDS) - { - // note: use this instead of resize because FileNode doesn't have - // a default ctor. NB: this is how resize is implemented anyway. - file_nodes.erase(file_nodes.begin() + MAX_IDS, file_nodes.end()); - WARN_ERR(ERR::LIMIT); - } - } -}; - -//----------------------------------------------------------------------------- - -typedef u32 ConnectionId; -cassert(sizeof(FileId)*2 <= sizeof(ConnectionId)); -static ConnectionId cid_make(FileId first, FileId second) -{ - return u32_from_u16(first, second); -} -static FileId cid_first(ConnectionId id) -{ - return u32_hi(id); -} -static FileId cid_second(ConnectionId id) -{ - return u32_lo(id); -} - -struct Connection -{ - ConnectionId id; - // repeated edges ("connections") are reflected in - // the 'occurrences' count; we optimize the ordering so that - // files with frequent connections are nearby. - uint occurrences; - - Connection(ConnectionId id_) - : id(id_), occurrences(1) {} -}; - -typedef std::vector Connections; - -// builds a list of Connection-s (basically edges in the FileNode graph) -// defined by the trace. -// -// time cost: 70ms for 1000 trace entries. this is rather heavy; -// the main culprit is simulating file_cache to see if an IO would result. -class ConnectionBuilder -{ - // functor: on every call except the first, adds a connection between - // the previous file (remembered here) and the current file. - // if the connection already exists, its occurrence count is incremented. - class ConnectionAdder - { - // speeds up "already exists" overhead from n*n to n*log(n). - typedef std::map Map; - typedef std::pair MapItem; - typedef Map::const_iterator MapCIt; - Map map; - - FileId prev_id; - - public: - ConnectionAdder() : prev_id(NULL_ID) {} - - void operator()(Connections& connections, const char* new_fn) - { - const bool was_first_call = (prev_id == NULL_ID); - FileId id = id_mgr.id_from_fn(new_fn); - const ConnectionId c_id = cid_make(prev_id, id); - prev_id = id; - - if(was_first_call) - return; // bail after setting prev_id - - // note: always insert-ing and checking return value would be - // more efficient (saves 1 iteration over map), but would not - // be safe: VC8's STL disallows &vector[0] if empty - // (even though memory has been reserved). - // it doesn't matter much anyway (decently fast and offline task). - MapCIt it = map.find(c_id); - const bool already_exists = (it != map.end()); - if(already_exists) - { - Connection* c = it->second; // Map "payload" - c->occurrences++; - } - // seen this connection for the first time: add to map and list. - else - { - connections.push_back(Connection(c_id)); - const MapItem item = std::make_pair(c_id, &connections.back()); - map.insert(item); - } - - stats_ab_connection(already_exists); - } - }; - - void add_connections_from_runs(const Trace& t, Connections& connections) - { - file_cache_reset(); - - // (note: lifetime = entire connection build process; if re-created - // in between, entries in Connections will no longer be unique, - // which may break TourBuilder) - ConnectionAdder add_connection; - - // extract accesses from each run (starting with most recent - // first. this isn't critical, but may help a bit since - // files that are equally strongly 'connected' are ordered - // according to position in file_nodes. that means files from - // more recent traces tend to go first, which is good.) - for(size_t r = 0; r < t.num_runs; r++) - { - const TraceRun& run = t.runs[r]; - for(uint i = 0; i < run.num_ents; i++) - { - const TraceEntry* te = &run.ents[i]; - // improvement: postprocess the trace and remove all IOs that would be - // satisfied by our cache. often repeated IOs would otherwise potentially - // be arranged badly. - if(trace_entry_causes_io(te)) - { - // only add connection if this file exists and is in - // file_nodes list. otherwise, ConnectionAdder's - // id_from_fn call will fail. - // note: this happens when trace contains by now - // deleted or unarchivable files. - TFile* tf; - if(tree_lookup(te->atom_fn, &tf) == INFO::OK) - if(is_archivable(tf)) - add_connection(connections, te->atom_fn); - } - } - - file_cache_reset(); - } - } - -public: - LibError run(const char* trace_filename, Connections& connections) - { - Trace t; - RETURN_ERR(trace_read_from_file(trace_filename, &t)); - - // reserve memory for worst-case amount of connections (happens if - // all accesses are unique). this is necessary because we store - // pointers to Connection in the map, which would be invalidated if - // connections[] ever expands. - // may waste up to ~3x the memory (about 1mb) for a short time, - // which is ok. - connections.reserve(t.total_ents-1); - - add_connections_from_runs(t, connections); - - return INFO::OK; - } -}; - -//----------------------------------------------------------------------------- - -// given graph and known edges, stitch together FileNodes so that -// Hamilton tour (TSP solution) length of the graph is minimized. -// heuristic is greedy adding edges sorted by decreasing 'occurrences'. -// -// time cost: 7ms for 1000 connections; quite fast despite DFS. -// -// could be improved (if there are lots of files) by storing in each node -// a pointer to end of list; if adding a new edge, check if end.endoflist -// is the start of edge. -class TourBuilder -{ - // sort by decreasing occurrence - struct Occurrence_greater: public std::binary_function - { - bool operator()(const Connection& c1, const Connection& c2) const - { - return (c1.occurrences > c2.occurrences); - } - }; - - bool has_cycle; - void detect_cycleR(FileId id) - { - FileNode* pnode = id_mgr.node_from_id(id); - pnode->visited = 1; - FileId next_id = pnode->next_id; - if(next_id != NULL_ID) - { - FileNode* pnext = id_mgr.node_from_id(next_id); - if(pnext->visited) - has_cycle = true; - else - detect_cycleR(next_id); - } - } - bool is_cycle_at(FileNodes& file_nodes, FileId node) - { - has_cycle = false; - for(FileNodes::iterator it = file_nodes.begin(); it != file_nodes.end(); ++it) - it->visited = 0; - detect_cycleR(node); - return has_cycle; - } - - void try_add_edge(FileNodes& file_nodes, const Connection& c) - { - FileId first_id = cid_first(c.id); - FileId second_id = cid_second(c.id); - - FileNode* first = id_mgr.node_from_id(first_id); - FileNode* second = id_mgr.node_from_id(second_id); - // one of them has already been hooked up - bail - if(first->next_id != NULL_ID || second->prev_id != NULL_ID) - return; - - first->next_id = second_id; - second->prev_id = first_id; - - const bool introduced_cycle = is_cycle_at(file_nodes, second_id); -#ifndef NDEBUG - debug_assert(introduced_cycle == is_cycle_at(file_nodes, first_id)); -#endif - if(introduced_cycle) - { - // undo - first->next_id = second->prev_id = NULL_ID; - return; - } - } - - - void output_chain(FileNode& node, std::vector& fn_vector) - { - // early out: if this access was already visited, so must the entire - // chain of which it is a part. bail to save lots of time. - if(node.output) - return; - - // follow prev links starting with c until no more are left; - // start ends up the beginning of the chain including . - FileNode* start = &node; - while(start->prev_id != NULL_ID) - start = id_mgr.node_from_id(start->prev_id); - - // iterate over the chain - add to Filenames list and mark as visited - FileNode* cur = start; - for(;;) - { - if(!cur->output) - { - fn_vector.push_back(cur->atom_fn); - cur->output = 1; - } - if(cur->next_id == NULL_ID) - break; - cur = id_mgr.node_from_id(cur->next_id); - } - } - -public: - TourBuilder(FileNodes& file_nodes, Connections& connections, std::vector& fn_vector) - { - std::stable_sort(connections.begin(), connections.end(), Occurrence_greater()); - - for(Connections::iterator it = connections.begin(); it != connections.end(); ++it) - try_add_edge(file_nodes, *it); - - for(FileNodes::iterator it = file_nodes.begin(); it != file_nodes.end(); ++it) - output_chain(*it, fn_vector); - } -}; - - -//----------------------------------------------------------------------------- -// autobuild logic: decides when to (re)build an archive. -//----------------------------------------------------------------------------- - -// for each loose or archived file encountered during mounting: add to a -// std::set; if there are more than *_THRESHOLD non-archived files, rebuild. -// this ends up costing 50ms for 5000 files, so disable it in final release. -#if CONFIG_FINAL -# define AB_COUNT_LOOSE_FILES 0 -#else -# define AB_COUNT_LOOSE_FILES 1 -#endif -// rebuild if the archive is much older than most recent VFS timestamp. -// this makes sense during development: the archive will periodically be -// rebuilt with the newest trace. however, it would be annoying in the -// final release, where users will frequently mod things, which should not -// end up rebuilding the main archive. -#if CONFIG_FINAL -# define AB_COMPARE_MTIME 0 -#else -# define AB_COMPARE_MTIME 1 -#endif - -#if AB_COUNT_LOOSE_FILES -static const ssize_t REBUILD_MAIN_ARCHIVE_THRESHOLD = 50; -static const ssize_t BUILD_MINI_ARCHIVE_THRESHOLD = 20; - -typedef std::set FnSet; -static FnSet loose_files; -static FnSet archived_files; -#endif - -void vfs_opt_notify_loose_file(const char* atom_fn) -{ -#if AB_COUNT_LOOSE_FILES - // note: files are added before archives, so we can't stop adding to - // set after one of the above thresholds are reached. - loose_files.insert(atom_fn); -#endif -} - -void vfs_opt_notify_non_loose_file(const char* atom_fn) -{ -#if AB_COUNT_LOOSE_FILES - archived_files.insert(atom_fn); -#endif -} - - -static bool should_rebuild_main_archive(const char* trace_filename, - DirEnts& existing_archives) -{ - // if there's no trace file, no point in building a main archive. - // (we wouldn't know how to order the files) - if(!file_exists(trace_filename)) - return false; - -#if AB_COUNT_LOOSE_FILES - // too many (eligible for archiving!) loose files not in archive: rebuild. - const ssize_t loose_files_only = (ssize_t)loose_files.size() - (ssize_t)archived_files.size(); - if(loose_files_only >= REBUILD_MAIN_ARCHIVE_THRESHOLD) - return true; -#endif - - // scan dir and see what archives are already present.. - { - time_t most_recent_archive_mtime = 0; - // note: a loop is more convenient than std::for_each, which would - // require referencing the returned functor (since param is a copy). - for(DirEnts::const_iterator it = existing_archives.begin(); it != existing_archives.end(); ++it) - most_recent_archive_mtime = std::max(it->mtime, most_recent_archive_mtime); - // .. no archive yet OR 'lots' of them: rebuild so that they'll be - // merged into one archive and the rest deleted. - if(existing_archives.empty() || existing_archives.size() >= 4) - return true; -#if AB_COMPARE_MTIME - // .. archive is much older than most recent data: rebuild. - const double max_diff = 14*86400; // 14 days - if(difftime(tree_most_recent_mtime(), most_recent_archive_mtime) > max_diff) - return true; -#endif - } - - return false; -} - -//----------------------------------------------------------------------------- - - -static char archive_fn[PATH_MAX]; -static ArchiveBuildState ab; -static std::vector fn_vector; -static DirEnts existing_archives; // and possibly other entries - -class IsArchive -{ - const char* archive_ext; - -public: - IsArchive(const char* archive_fn) - { - archive_ext = path_extension(archive_fn); - } - - bool operator()(DirEnt& ent) const - { - // remove if not file - if(DIRENT_IS_DIR(&ent)) - return true; - - // remove if not same extension - const char* ext = path_extension(ent.name); - if(strcasecmp(archive_ext, ext) != 0) - return true; - - // keep - return false; - } -}; - -static LibError vfs_opt_init(const char* trace_filename, const char* archive_fn_fmt, bool force_build) -{ - // get next not-yet-existing archive filename. - static NextNumberedFilenameInfo archive_nfi; - bool use_vfs = false; // can't use VFS for archive files - next_numbered_filename(archive_fn_fmt, &archive_nfi, archive_fn, use_vfs); - - // get list of existing archives in root dir. - // note: this is needed by should_rebuild_main_archive and later in - // vfs_opt_continue; must be done here instead of inside the former - // because that is not called when force_build == true. - { - char dir[PATH_MAX]; - path_dir_only(archive_fn_fmt, dir); - RETURN_ERR(file_get_sorted_dirents(dir, existing_archives)); - DirEntIt new_end = std::remove_if(existing_archives.begin(), existing_archives.end(), IsArchive(archive_fn)); - existing_archives.erase(new_end, existing_archives.end()); - } - - // bail if we shouldn't rebuild the archive. - if(!force_build && !should_rebuild_main_archive(trace_filename, existing_archives)) - return INFO::SKIPPED; - - // build 'graph' (nodes only) of all files that must be added. - FileNodes file_nodes; - FileGatherer gatherer(file_nodes); - if(file_nodes.empty()) - WARN_RETURN(ERR::DIR_END); - - // scan nodes and add them to filename->FileId mapping. - id_mgr.init(&file_nodes); - - // build list of edges between FileNodes (referenced via FileId) that - // are defined by trace entries. - Connections connections; - ConnectionBuilder cbuilder; - RETURN_ERR(cbuilder.run(trace_filename, connections)); - - // create output filename list by first adding the above edges (most - // frequent first) and then adding the rest sequentially. - TourBuilder builder(file_nodes, connections, fn_vector); - fn_vector.push_back(0); // 0-terminate for use as Filenames - Filenames V_fns = &fn_vector[0]; - - RETURN_ERR(archive_build_init(archive_fn, V_fns, &ab)); - return INFO::OK; -} - - -static int vfs_opt_continue() -{ - int ret = archive_build_continue(&ab); - if(ret == INFO::OK) - { - // do NOT delete source files! some apps might want to - // keep them (e.g. for source control), or name them differently. - - mount_release_all_archives(); - - // delete old archives - PathPackage pp; // need path to each existing_archive, not only name - { - char archive_dir[PATH_MAX]; - path_dir_only(archive_fn, archive_dir); - (void)path_package_set_dir(&pp, archive_dir); - } - for(DirEntCIt it = existing_archives.begin(); it != existing_archives.end(); ++it) - { - (void)path_package_append_file(&pp, it->name); - (void)file_delete(pp.path); - } - - // rebuild is required due to mount_release_all_archives. - // the dir watcher may already have rebuilt the VFS once, - // which is a waste of time here. - (void)mount_rebuild(); - - // it is believed that wiping out the file cache is not necessary. - // building archive doesn't change the game data files, and any - // cached contents of the previous archives are irrelevant. - } - return ret; -} - - - - - -static bool should_build_mini_archive(const char* UNUSED(mini_archive_fn_fmt)) -{ -#if AB_COUNT_LOOSE_FILES - // too many (eligible for archiving!) loose files not in archive - const ssize_t loose_files_only = (ssize_t)loose_files.size() - (ssize_t)archived_files.size(); - if(loose_files_only >= BUILD_MINI_ARCHIVE_THRESHOLD) - return true; -#endif - return false; -} - -static LibError build_mini_archive(const char* mini_archive_fn_fmt) -{ - if(!should_build_mini_archive(mini_archive_fn_fmt)) - return INFO::SKIPPED; - -#if AB_COUNT_LOOSE_FILES - Filenames V_fns = new const char*[loose_files.size()+1]; - std::copy(loose_files.begin(), loose_files.end(), &V_fns[0]); - V_fns[loose_files.size()] = 0; // terminator - - // get new unused mini archive name at P_dst_path - char mini_archive_fn[PATH_MAX]; - static NextNumberedFilenameInfo nfi; - bool use_vfs = false; // can't use VFS for archive files - next_numbered_filename(mini_archive_fn_fmt, &nfi, mini_archive_fn, use_vfs); - - RETURN_ERR(archive_build(mini_archive_fn, V_fns)); - delete[] V_fns; - return INFO::OK; -#else - return ERR::NOT_IMPLEMENTED; -#endif -} - - - -static enum -{ - DECIDE_IF_BUILD, - IN_PROGRESS, - NOP -} -state = DECIDE_IF_BUILD; - -void vfs_opt_auto_build_cancel() -{ - archive_build_cancel(&ab); - state = NOP; -} - -int vfs_opt_auto_build(const char* trace_filename, - const char* archive_fn_fmt, const char* mini_archive_fn_fmt, bool force_build) -{ - if(state == NOP) - return INFO::ALL_COMPLETE; - - if(state == DECIDE_IF_BUILD) - { - if(vfs_opt_init(trace_filename, archive_fn_fmt, force_build) != INFO::SKIPPED) - state = IN_PROGRESS; - else - { - // create mini-archive (if needed) - RETURN_ERR(build_mini_archive(mini_archive_fn_fmt)); - - state = NOP; - return INFO::OK; // "finished" - } - } - - if(state == IN_PROGRESS) - { - int ret = vfs_opt_continue(); - // just finished - if(ret == INFO::OK) - state = NOP; - return ret; - } - - UNREACHABLE; -} - -LibError vfs_opt_rebuild_main_archive(const char* trace_filename, const char* archive_fn_fmt) -{ - for(;;) - { - int ret = vfs_opt_auto_build(trace_filename, archive_fn_fmt, 0, true); - RETURN_ERR(ret); - if(ret == INFO::OK) - return INFO::OK; - } -} diff --git a/source/lib/res/file/archive/vfs_optimizer.h b/source/lib/res/file/archive/vfs_optimizer.h deleted file mode 100644 index 619e078743..0000000000 --- a/source/lib/res/file/archive/vfs_optimizer.h +++ /dev/null @@ -1,26 +0,0 @@ -/** - * ========================================================================= - * File : vfs_optimizer.h - * Project : 0 A.D. - * Description : automatically bundles files into archives in order of - * : access to optimize I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_VFS_OPTIMIZER -#define INCLUDED_VFS_OPTIMIZER - - -extern LibError vfs_opt_rebuild_main_archive(const char* trace_filename, const char* archive_fn_fmt); - -extern void vfs_opt_auto_build_cancel(); - -extern int vfs_opt_auto_build(const char* trace_filename, - const char* archive_fn_fmt, const char* mini_archive_fn_fmt, bool force_build = false); - -extern void vfs_opt_notify_loose_file(const char* atom_fn); -extern void vfs_opt_notify_non_loose_file(const char* atom_fn); - -#endif // #ifndef INCLUDED_VFS_OPTIMIZER diff --git a/source/lib/res/file/archive/zip.cpp b/source/lib/res/file/archive/zip.cpp deleted file mode 100644 index 0595ed95f1..0000000000 --- a/source/lib/res/file/archive/zip.cpp +++ /dev/null @@ -1,591 +0,0 @@ -/** - * ========================================================================= - * File : zip.cpp - * Project : 0 A.D. - * Description : archive backend for Zip files. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "zip.h" - -#include -#include - -#include "lib/bits.h" -#include "lib/byte_order.h" -#include "lib/allocators.h" -#include "lib/timer.h" -#include "lib/res/res.h" -#include "../file_internal.h" -#include "lib/fat_time.h" - - -//----------------------------------------------------------------------------- -// Zip file data structures and signatures -//----------------------------------------------------------------------------- - -enum ZipCompressionMethod -{ - ZIP_CM_NONE = 0, - ZIP_CM_DEFLATE = 8 -}; - -// translate ArchiveEntry.method to zip_method. -static ZipCompressionMethod zip_method_for(CompressionMethod method) -{ - switch(method) - { - case CM_NONE: - return ZIP_CM_NONE; - case CM_DEFLATE: - return ZIP_CM_DEFLATE; - default: - WARN_ERR(ERR::COMPRESSION_UNKNOWN_METHOD); - return ZIP_CM_NONE; - } -} - -// translate to (not Zip-specific) CompressionMethod for use in ArchiveEntry. -static CompressionMethod method_for_zip_method(ZipCompressionMethod zip_method) -{ - switch(zip_method) - { - case ZIP_CM_NONE: - return CM_NONE; - case ZIP_CM_DEFLATE: - return CM_DEFLATE; - default: - WARN_ERR(ERR::COMPRESSION_UNKNOWN_METHOD); - return CM_UNSUPPORTED; - } -} - - -static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2'); -static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4'); -static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6'); - -#pragma pack(push, 1) - -struct LFH -{ - u32 magic; - u16 x1; // version needed - u16 flags; - u16 method; - u32 fat_mtime; // last modified time (DOS FAT format) - u32 crc; - u32 csize; - u32 usize; - u16 fn_len; - u16 e_len; -}; - -const size_t LFH_SIZE = sizeof(LFH); -cassert(LFH_SIZE == 30); - -// convenience (allows writing out LFH and fn in 1 IO). -// must be declared here to avoid any struct padding. -struct LFH_Package -{ - LFH lfh; - char fn[PATH_MAX]; -}; - - -struct CDFH -{ - u32 magic; - u32 x1; // versions - u16 flags; - u16 method; - u32 fat_mtime; // last modified time (DOS FAT format) - u32 crc; - u32 csize; - u32 usize; - u16 fn_len; - u16 e_len; - u16 c_len; - u32 x2; // spanning - u32 x3; // attributes - u32 lfh_ofs; -}; - -const size_t CDFH_SIZE = sizeof(CDFH); -cassert(CDFH_SIZE == 46); - -// convenience (avoids need for pointer arithmetic) -// must be declared here to avoid any struct padding. -struct CDFH_Package -{ - CDFH cdfh; - char fn[PATH_MAX]; -}; - - -struct ECDR -{ - u32 magic; - u8 x1[6]; // multiple-disk support - u16 cd_entries; - u32 cd_size; - u32 cd_ofs; - u16 comment_len; -}; - -const size_t ECDR_SIZE = sizeof(ECDR); -cassert(ECDR_SIZE == 22); - -#pragma pack(pop) - - -static off_t lfh_total_size(const LFH* lfh_le) -{ - debug_assert(lfh_le->magic == lfh_magic); - const size_t fn_len = read_le16(&lfh_le->fn_len); - const size_t e_len = read_le16(&lfh_le->e_len); - // note: LFH doesn't have a comment field! - - return (off_t)(LFH_SIZE + fn_len + e_len); -} - -static void lfh_assemble(LFH* lfh_le, - CompressionMethod method, time_t mtime, u32 crc, - off_t csize, off_t usize, size_t fn_len) -{ - const ZipCompressionMethod zip_method = zip_method_for(method); - const u32 fat_mtime = FAT_from_time_t(mtime); - - lfh_le->magic = lfh_magic; - lfh_le->x1 = to_le16(0); - lfh_le->flags = to_le16(0); - lfh_le->method = to_le16(zip_method); - lfh_le->fat_mtime = to_le32(fat_mtime); - lfh_le->crc = to_le32(crc); - lfh_le->csize = to_le32(u32_from_larger(csize)); - lfh_le->usize = to_le32(u32_from_larger(usize)); - lfh_le->fn_len = to_le16(u16_from_larger(fn_len)); - lfh_le->e_len = to_le16(0); -} - - -static void cdfh_decompose(const CDFH* cdfh_le, - CompressionMethod& method, time_t& mtime, u32& crc, off_t& csize, off_t& usize, - const char*& fn, off_t& lfh_ofs, size_t& total_size) -{ - const u16 zip_method = read_le16(&cdfh_le->method); - const u32 fat_mtime = read_le32(&cdfh_le->fat_mtime); - crc = read_le32(&cdfh_le->crc); - csize = (off_t)read_le32(&cdfh_le->csize); - usize = (off_t)read_le32(&cdfh_le->usize); - const u16 fn_len = read_le16(&cdfh_le->fn_len); - const u16 e_len = read_le16(&cdfh_le->e_len); - const u16 c_len = read_le16(&cdfh_le->c_len); - lfh_ofs = (off_t)read_le32(&cdfh_le->lfh_ofs); - - method = method_for_zip_method((ZipCompressionMethod)zip_method); - mtime = time_t_from_FAT(fat_mtime); - - // return 0-terminated copy of filename - const char* fn_src = (const char*)cdfh_le+CDFH_SIZE; // not 0-terminated! - char fn_buf[PATH_MAX]; - cpu_memcpy(fn_buf, fn_src, fn_len*sizeof(char)); - fn_buf[fn_len] = '\0'; - fn = file_make_unique_fn_copy(fn_buf); - - total_size = CDFH_SIZE + fn_len + e_len + c_len; -} - -static void cdfh_assemble(CDFH* dst_cdfh_le, - CompressionMethod method, time_t mtime, u32 crc, - size_t csize, size_t usize, size_t fn_len, size_t slack, u32 lfh_ofs) -{ - const ZipCompressionMethod zip_method = zip_method_for(method); - const u32 fat_mtime = FAT_from_time_t(mtime); - - dst_cdfh_le->magic = cdfh_magic; - dst_cdfh_le->x1 = to_le32(0); - dst_cdfh_le->flags = to_le16(0); - dst_cdfh_le->method = to_le16(zip_method); - dst_cdfh_le->fat_mtime = to_le32(fat_mtime); - dst_cdfh_le->crc = to_le32(crc); - dst_cdfh_le->csize = to_le32(u32_from_larger(csize)); - dst_cdfh_le->usize = to_le32(u32_from_larger(usize)); - dst_cdfh_le->fn_len = to_le16(u16_from_larger(fn_len)); - dst_cdfh_le->e_len = to_le16(0); - dst_cdfh_le->c_len = to_le16(u16_from_larger(slack)); - dst_cdfh_le->x2 = to_le32(0); - dst_cdfh_le->x3 = to_le32(0); - dst_cdfh_le->lfh_ofs = to_le32(lfh_ofs); -} - - -static void ecdr_decompose(ECDR* ecdr_le, - uint& cd_entries, off_t& cd_ofs, size_t& cd_size) -{ - cd_entries = (uint)read_le16(&ecdr_le->cd_entries); - cd_ofs = (off_t)read_le32(&ecdr_le->cd_ofs); - cd_size = (size_t)read_le32(&ecdr_le->cd_size); -} - -static void ecdr_assemble(ECDR* dst_ecdr_le, uint cd_entries, off_t cd_ofs, size_t cd_size) -{ - dst_ecdr_le->magic = ecdr_magic; - memset(dst_ecdr_le->x1, 0, sizeof(dst_ecdr_le->x1)); - dst_ecdr_le->cd_entries = to_le16(u16_from_larger(cd_entries)); - dst_ecdr_le->cd_size = to_le32(u32_from_larger(cd_size)); - dst_ecdr_le->cd_ofs = to_le32(u32_from_larger(cd_ofs)); - dst_ecdr_le->comment_len = to_le16(0); -} - - -//----------------------------------------------------------------------------- - -// scan for and return a pointer to a Zip record, or 0 if not found. -// is the expected position; we scan from there until EOF for -// the given ID (fourcc). includes ID field) bytes must -// remain before EOF - this makes sure the record is completely in the file. -// used by z_find_ecdr and z_extract_cdfh. -static const u8* za_find_id(const u8* buf, size_t size, const u8* start, u32 magic, size_t record_size) -{ - ssize_t bytes_left = (ssize_t)((buf+size) - (u8*)start - record_size); - - const u8* p = (const u8*)start; - // don't increment function argument directly, - // so we can warn the user if we had to scan. - - while(bytes_left-- >= 0) - { - // found it - if(*(u32*)p == magic) - { -#ifndef NDEBUG - if(p != start) - debug_warn("archive damaged, but still found next record."); -#endif - return p; - } - - p++; - // be careful not to increment before comparison; - // magic may already be found at . - } - - // passed EOF, didn't find it. - // note: do not warn - this happens in the initial ECDR search at - // EOF if the archive contains a comment field. - return 0; -} - - -// search for ECDR in the last bytes of the file. -// if found, fill with a copy of the (little-endian) ECDR and -// return INFO::OK, otherwise IO error or ERR::CORRUPTED. -static LibError za_find_ecdr(File* f, size_t max_scan_amount, ECDR* dst_ecdr_le) -{ - // don't scan more than the entire file - const size_t file_size = f->size; - const size_t scan_amount = std::min(max_scan_amount, file_size); - - // read desired chunk of file into memory - const off_t ofs = (off_t)(file_size - scan_amount); - FileIOBuf buf = FILE_BUF_ALLOC; - ssize_t bytes_read = file_io(f, ofs, scan_amount, &buf); - RETURN_ERR(bytes_read); - debug_assert(bytes_read == (ssize_t)scan_amount); - - // look for ECDR in buffer - LibError ret = ERR::CORRUPTED; - const u8* start = (const u8*)buf; - const ECDR* ecdr_le = (const ECDR*)za_find_id(start, bytes_read, start, ecdr_magic, ECDR_SIZE); - if(ecdr_le) - { - *dst_ecdr_le = *ecdr_le; - ret = INFO::OK; - } - - file_buf_free(buf); - return ret; -} - - -static LibError za_find_cd(File* f, uint& cd_entries, off_t& cd_ofs, size_t& cd_size) -{ - // sanity check: file size must be > header size. - // (this speeds up determining if the file is a Zip file at all) - const size_t file_size = f->size; - if(file_size < LFH_SIZE+CDFH_SIZE+ECDR_SIZE) - { -completely_bogus: - // this file is definitely not a valid Zip file. - // note: the VFS blindly opens files when mounting; it needs to open - // all archives, but doesn't know their extension (e.g. ".pk3"). - // therefore, do not warn user. - return ERR::RES_UNKNOWN_FORMAT; // NOWARN - } - - ECDR ecdr_le; - // expected case: ECDR at EOF; no file comment (=> we only need to - // read 512 bytes) - LibError ret = za_find_ecdr(f, ECDR_SIZE, &ecdr_le); - if(ret == INFO::OK) - { -have_ecdr: - ecdr_decompose(&ecdr_le, cd_entries, cd_ofs, cd_size); - return INFO::OK; - } - // last resort: scan last 66000 bytes of file - // (the Zip archive comment field - up to 64k - may follow ECDR). - // if the zip file is < 66000 bytes, scan the whole file. - ret = za_find_ecdr(f, 66000u, &ecdr_le); - if(ret == INFO::OK) - goto have_ecdr; - - // both ECDR scans failed - this is not a valid Zip file. - // now see if the beginning of the file holds a valid LFH: - const off_t ofs = 0; const size_t scan_amount = LFH_SIZE; - FileIOBuf buf = FILE_BUF_ALLOC; - ssize_t bytes_read = file_io(f, ofs, scan_amount, &buf); - RETURN_ERR(bytes_read); - debug_assert(bytes_read == (ssize_t)scan_amount); - const bool has_LFH = (za_find_id(buf, scan_amount, buf, lfh_magic, LFH_SIZE) != 0); - file_buf_free(buf); - if(!has_LFH) - goto completely_bogus; - // the Zip file is mostly valid but lacking an ECDR. (can happen if - // user hard-exits while building an archive) - // notes: - // - return ERR::CORRUPTED so VFS will not include this file. - // - we could work around this by scanning all LFHs, but won't bother - // because it'd be slow. - // - do not warn - the corrupt archive will be deleted on next - // successful archive builder run anyway. - return ERR::CORRUPTED; // NOWARN -} - - -// analyse an opened Zip file; call back into archive.cpp to -// populate the Archive object with a list of the files it contains. -// returns INFO::OK on success, ERR::CORRUPTED if file is recognizable as -// a Zip file but invalid, otherwise ERR::RES_UNKNOWN_FORMAT or IO error. -// -// fairly slow - must read Central Directory from disk -// (size ~= 60 bytes*num_files); observed time ~= 80ms. -LibError zip_populate_archive(File* f, Archive* a) -{ - uint cd_entries; off_t cd_ofs; size_t cd_size; - RETURN_ERR(za_find_cd(f, cd_entries, cd_ofs, cd_size)); - - // call back with number of entries in archives (an upper bound - // for valid files; we're not interested in the directory entries). - // we'd have to scan through the central dir to count them out; we'll - // just skip them and waste a bit of preallocated memory. - RETURN_ERR(archive_allocate_entries(a, cd_entries)); - - FileIOBuf buf = FILE_BUF_ALLOC; - RETURN_ERR(file_io(f, cd_ofs, cd_size, &buf)); - - // iterate through Central Directory - LibError ret = INFO::OK; - const CDFH* cdfh = (const CDFH*)buf; - size_t ofs_to_next_cdfh = 0; - for(uint i = 0; i < cd_entries; i++) - { - // scan for next CDFH (at or beyond current cdfh position) - cdfh = (const CDFH*)((u8*)cdfh + ofs_to_next_cdfh); - cdfh = (CDFH*)za_find_id((const u8*)buf, cd_size, (const u8*)cdfh, cdfh_magic, CDFH_SIZE); - if(!cdfh) // no (further) CDFH found: - { - ret = ERR::CORRUPTED; - break; - } - - // copy translated fields from CDFH into ArchiveEntry. - ArchiveEntry ae; - cdfh_decompose(cdfh, ae.method, ae.mtime, ae.checksum, ae.csize, ae.usize, ae.atom_fn, ae.ofs, ofs_to_next_cdfh); - ae.flags = ZIP_LFH_FIXUP_NEEDED; - - // if file (we don't care about directories): - if(ae.csize && ae.usize) - { - ret = archive_add_file(a, &ae); - if(ret != INFO::OK) - break; - } - } - - file_buf_free(buf); - return ret; -} - - -//----------------------------------------------------------------------------- - -// this code grabs an LFH struct from file block(s) that are -// passed to the callback. usually, one call copies the whole thing, -// but the LFH may straddle a block boundary. -// -// rationale: this allows using temp buffers for zip_fixup_lfh, -// which avoids involving the file buffer manager and thus -// unclutters the trace and cache contents. - -struct LFH_Copier -{ - u8* lfh_dst; - size_t lfh_bytes_remaining; -}; - -static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size, size_t* bytes_processed) -{ - LFH_Copier* p = (LFH_Copier*)cbData; - - debug_assert(size <= p->lfh_bytes_remaining); - cpu_memcpy(p->lfh_dst, block, size); - p->lfh_dst += size; - p->lfh_bytes_remaining -= size; - - *bytes_processed = size; - return INFO::CB_CONTINUE; -} - - -// ensures points to the actual file contents; it is initially -// the offset of the LFH. we cannot use CDFH filename and extra field -// lengths to skip past LFH since that may not mirror CDFH (has happened). -// -// this is called at file-open time instead of while mounting to -// reduce seeks: since reading the file will typically follow, the -// block cache entirely absorbs the IO cost. -void zip_fixup_lfh(File* f, ArchiveEntry* ent) -{ - // already fixed up - done. - if(!(ent->flags & ZIP_LFH_FIXUP_NEEDED)) - return; - - // performance note: this ends up reading one file block, which is - // only in the block cache if the file starts in the same block as a - // previously read file (i.e. both are small). - LFH lfh; - LFH_Copier params = { (u8*)&lfh, sizeof(LFH) }; - ssize_t ret = file_io(f, ent->ofs, LFH_SIZE, FILE_BUF_TEMP, lfh_copier_cb, (uintptr_t)¶ms); - debug_assert(ret == sizeof(LFH)); - - ent->ofs += lfh_total_size(&lfh); - ent->flags &= ~ZIP_LFH_FIXUP_NEEDED; -} - - -//----------------------------------------------------------------------------- -// archive builder backend -//----------------------------------------------------------------------------- - -// rationale: don't support partial adding, i.e. updating archive with -// only one file. this would require overwriting parts of the Zip archive, -// which is annoying and slow. also, archives are usually built in -// seek-optimal order, which would break if we start inserting files. -// while testing, loose files can be used, so there's no loss. - -// we don't want to expose ZipArchive to callers, -// (would require defining File, Pool and CDFH) -// so allocate the storage here and return opaque pointer. -struct ZipArchive -{ - File f; - off_t cur_file_size; - - Pool cdfhs; - uint cd_entries; - CDFH* prev_cdfh; -}; - -static SingleAllocator za_mgr; - - -// create a new Zip archive and return a pointer for use in subsequent -// zip_archive_add_file calls. previous archive file is overwritten. -LibError zip_archive_create(const char* zip_filename, ZipArchive** pza) -{ - // local za_copy simplifies things - if something fails, no cleanup is - // needed. upon success, we copy into the newly allocated real za. - ZipArchive za_copy; - za_copy.cur_file_size = 0; - za_copy.cd_entries = 0; - za_copy.prev_cdfh = 0; - - RETURN_ERR(file_open(zip_filename, FILE_WRITE|FILE_NO_AIO, &za_copy.f)); - RETURN_ERR(pool_create(&za_copy.cdfhs, 10*MiB, 0)); - - ZipArchive* za = za_mgr.Allocate(); - if(!za) - WARN_RETURN(ERR::NO_MEM); - *za = za_copy; - *pza = za; - return INFO::OK; -} - - -// add a file (described by ArchiveEntry) to the archive. file_contents -// is the actual file data; its compression method is given in ae->method and -// can be CM_NONE. -// IO cost: writes out to disk (we don't currently attempt -// any sort of write-buffering). -LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, const u8* file_contents) -{ - const size_t fn_len = strlen(ae->atom_fn); - - // write (LFH, filename, file contents) to archive - // .. put LFH and filename into one 'package' - LFH_Package header; - lfh_assemble(&header.lfh, ae->method, ae->mtime, ae->checksum, ae->csize, ae->usize, fn_len); - strcpy_s(header.fn, ARRAY_SIZE(header.fn), ae->atom_fn); - // .. write that out in 1 IO - const off_t lfh_ofs = za->cur_file_size; - FileIOBuf buf; - buf = (FileIOBuf)&header; - file_io(&za->f, lfh_ofs, LFH_SIZE+fn_len, &buf); - // .. write out file contents - buf = (FileIOBuf)file_contents; - file_io(&za->f, lfh_ofs+(off_t)(LFH_SIZE+fn_len), ae->csize, &buf); - za->cur_file_size += (off_t)(LFH_SIZE+fn_len+ae->csize); - - // append a CDFH to the central dir (in memory) - // .. note: pool_alloc may round size up for padding purposes. - const size_t prev_pos = za->cdfhs.da.pos; - CDFH_Package* p = (CDFH_Package*)pool_alloc(&za->cdfhs, CDFH_SIZE+fn_len); - if(!p) - WARN_RETURN(ERR::NO_MEM); - const size_t slack = za->cdfhs.da.pos-prev_pos - (CDFH_SIZE+fn_len); - cdfh_assemble(&p->cdfh, ae->method, ae->mtime, ae->checksum, ae->csize, ae->usize, fn_len, slack, lfh_ofs); - cpu_memcpy(p->fn, ae->atom_fn, fn_len); - - za->cd_entries++; - - return INFO::OK; -} - - -// write out the archive to disk; only hereafter is it valid. -// frees the ZipArchive instance. -// IO cost: writes out Central Directory to disk (about 70 bytes per file). -LibError zip_archive_finish(ZipArchive* za) -{ - const size_t cd_size = za->cdfhs.da.pos; - - // append an ECDR to the CDFH list (this allows us to - // write out both to the archive file in one burst) - ECDR* ecdr = (ECDR*)pool_alloc(&za->cdfhs, ECDR_SIZE); - if(!ecdr) - WARN_RETURN(ERR::NO_MEM); - ecdr_assemble(ecdr, za->cd_entries, za->cur_file_size, cd_size); - - FileIOBuf buf = za->cdfhs.da.base; - file_io(&za->f, za->cur_file_size, cd_size+ECDR_SIZE, &buf); - - (void)file_close(&za->f); - (void)pool_destroy(&za->cdfhs); - za_mgr.Free(za); - return INFO::OK; -} diff --git a/source/lib/res/file/archive/zip.h b/source/lib/res/file/archive/zip.h deleted file mode 100644 index 256479ac71..0000000000 --- a/source/lib/res/file/archive/zip.h +++ /dev/null @@ -1,67 +0,0 @@ -/** - * ========================================================================= - * File : zip.h - * Project : 0 A.D. - * Description : archive backend for Zip files. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_ZIP -#define INCLUDED_ZIP - -struct File; -struct Archive; -struct ArchiveEntry; - - -// analyse an opened Zip file; call back into archive.cpp to -// populate the Archive object with a list of the files it contains. -// returns INFO::OK on success, ERR::CORRUPTED if file is recognizable as -// a Zip file but invalid, otherwise ERR::RES_UNKNOWN_FORMAT or IO error. -// -// fairly slow - must read Central Directory from disk -// (size ~= 60 bytes*num_files); observed time ~= 80ms. -extern LibError zip_populate_archive(File* f, Archive* a); - - -// ensures points to the actual file contents; it is initially -// the offset of the LFH. we cannot use CDFH filename and extra field -// lengths to skip past LFH since that may not mirror CDFH (has happened). -// -// this is called at file-open time instead of while mounting to -// reduce seeks: since reading the file will typically follow, the -// block cache entirely absorbs the IO cost. -extern void zip_fixup_lfh(File* f, ArchiveEntry* ent); - - -// -// archive builder backend -// - -struct ZipArchive; // opaque - -// create a new Zip archive and return a pointer for use in subsequent -// zip_archive_add_file calls. previous archive file is overwritten. -extern LibError zip_archive_create(const char* zip_filename, ZipArchive** pza); - -// add a file (described by ArchiveEntry) to the archive. file_contents -// is the actual file data; its compression method is given in ae->method and -// can be CM_NONE. -// IO cost: writes out to disk (we don't currently attempt -// any sort of write-buffering). -extern LibError zip_archive_add_file(ZipArchive* za, const ArchiveEntry* ae, const u8* file_contents); - -// write out the archive to disk; only hereafter is it valid. -// frees the ZipArchive instance. -// IO cost: writes out Central Directory to disk (about 70 bytes per file). -extern LibError zip_archive_finish(ZipArchive* za); - - -// for self-test - -extern time_t time_t_from_FAT(u32 fat_timedate); -extern u32 FAT_from_time_t(time_t time); - -#endif // #ifndef INCLUDED_ZIP diff --git a/source/lib/res/file/file.cpp b/source/lib/res/file/file.cpp deleted file mode 100644 index 96ce6a7bb6..0000000000 --- a/source/lib/res/file/file.cpp +++ /dev/null @@ -1,581 +0,0 @@ -/** - * ========================================================================= - * File : file.cpp - * Project : 0 A.D. - * Description : file layer on top of POSIX. avoids the need for - * : absolute paths. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "file.h" - -#include -#include -#include - -#include "lib/posix/posix_filesystem.h" -#include "lib/posix/posix_aio.h" -#include "lib/posix/posix_mman.h" -#include "lib/adts.h" -#include "lib/sysdep/sysdep.h" -#include "lib/byte_order.h" -#include "lib/allocators.h" -#include "file_internal.h" - - -ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES); -ERROR_ASSOCIATE(ERR::DIR_END, "End of directory reached (no more files)", -1); -ERROR_ASSOCIATE(ERR::FILE_NOT_MAPPED, "File was not mapped", -1); - -// rationale for aio, instead of only using mmap: -// - parallelism: instead of just waiting for the transfer to complete, -// other work can be done in the meantime. -// example: decompressing from a Zip archive is practically free, -// because we inflate one block while reading the next. -// - throughput: with aio, the drive always has something to do, as opposed -// to read requests triggered by the OS for mapped files, which come -// in smaller chunks. this leads to much higher transfer rates. -// - memory: when used with VFS, aio makes better use of a file cache. -// data is generally compressed in an archive. a cache should store the -// decompressed and decoded (e.g. TGA colour swapping) data; mmap would -// keep the original, compressed data in memory, which doesn't help. -// we bypass the OS file cache via aio, and store partial blocks here (*); -// higher level routines will cache the actual useful data. -// * requests for part of a block are usually followed by another. - - - -// layer on top of POSIX opendir/readdir/closedir that handles -// portable -> native path conversion, ignores non-file/directory entries, -// and additionally returns the file status (size and mtime). - -// rationale: see DirIterator definition in header. -struct PosixDirIterator -{ - DIR* os_dir; - - // to support stat(), we need to either chdir or store the complete path. - // the former is unacceptable because it isn't thread-safe. therefore, - // we latch dir_open's path and append entry name every dir_next_ent call. - // this is also the storage to which DirEnt.name points! - // PathPackage avoids repeated memory allocs and strlen() overhead. - // - // it can't be stored here directly because then the struct would - // no longer fit in HDATA; we'll allocate it separately. - PathPackage* pp; -}; - -cassert(sizeof(PosixDirIterator) <= DIR_ITERATOR_OPAQUE_SIZE); - -static SingleAllocator pp_allocator; - - -// prepare to iterate (once) over entries in the given directory. -// if INFO::OK is returned, is ready for subsequent dir_next_ent calls and -// must be freed via dir_close. -LibError dir_open(const char* P_path, DirIterator* di) -{ - PosixDirIterator* pdi = (PosixDirIterator*)di->opaque; - - // note: copying to n_path and then pp.path is inefficient but - // more clear/robust. this is only called a few hundred times anyway. - char n_path[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(P_path, n_path)); - - pdi->pp = pp_allocator.Allocate(); - if(!pdi->pp) - WARN_RETURN(ERR::NO_MEM); - - errno = 0; - pdi->os_dir = opendir(n_path); - if(!pdi->os_dir) - return LibError_from_errno(); - - (void)path_package_set_dir(pdi->pp, n_path); - return INFO::OK; -} - - -// return ERR::DIR_END if all entries have already been returned once, -// another negative error code, or INFO::OK on success, in which case -// describes the next (order is unspecified) directory entry. -LibError dir_next_ent(DirIterator* di, DirEnt* ent) -{ - PosixDirIterator* pdi = (PosixDirIterator*)di->opaque; - -get_another_entry: - errno = 0; - struct dirent* os_ent = readdir(pdi->os_dir); - if(!os_ent) - { - // no error, just no more entries to return - if(!errno) - return ERR::DIR_END; // NOWARN - return LibError_from_errno(); - } - - // copy os_ent.name[]; we need it for stat() #if !OS_WIN and - // return it as ent.name (since os_ent.name[] is volatile). - path_package_append_file(pdi->pp, os_ent->d_name); - const char* name = pdi->pp->end; - - // get file information (mode, size, mtime) - struct stat s; -#if OS_WIN - // .. wposix readdir has enough information to return dirent - // status directly (much faster than calling stat). - CHECK_ERR(readdir_stat_np(pdi->os_dir, &s)); -#else - // .. call regular stat(). - // we need the full pathname for this. don't use path_append because - // it would unnecessarily call strlen. - - CHECK_ERR(stat(pdi->pp->path, &s)); -#endif - - // skip "undesirable" entries that POSIX readdir returns: - if(S_ISDIR(s.st_mode)) - { - // .. dummy directory entries ("." and "..") - if(name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) - goto get_another_entry; - - s.st_size = -1; // our way of indicating it's a directory - } - // .. neither dir nor file - else if(!S_ISREG(s.st_mode)) - goto get_another_entry; - - ent->size = s.st_size; - ent->mtime = s.st_mtime; - ent->name = name; - return INFO::OK; -} - - -// indicate the directory iterator is no longer needed; all resources it -// held are freed. -LibError dir_close(DirIterator* di) -{ - PosixDirIterator* pdi = (PosixDirIterator*)di->opaque; - pp_allocator.Free(pdi->pp); - - errno = 0; - if(closedir(pdi->os_dir) < 0) - return LibError_from_errno(); - return INFO::OK; -} - - -bool dir_exists(const char* P_path) -{ - // modified from file_stat_impl - we don't want errors to be raised here. - char N_path[PATH_MAX]; - THROW_ERR(file_make_full_native_path(P_path, N_path)); - - // if path ends in slash, remove it (required by stat) - char* last_char = N_path+strlen(N_path)-1; - if(path_is_dir_sep(*last_char)) - *last_char = '\0'; - - struct stat s; - if(stat(N_path, &s) != 0) - return false; - debug_assert(S_ISDIR(s.st_mode)); - return true; -} - - -LibError dir_create(const char* P_path) -{ - char N_path[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(P_path, N_path)); - - struct stat s; - int ret = stat(N_path, &s); - if(ret == 0) - return INFO::ALREADY_EXISTS; - - errno = 0; - ret = mkdir(N_path, S_IRWXO|S_IRWXU|S_IRWXG); - return LibError_from_posix(ret); -} - - -// note: we have to recursively empty the directory before it can -// be deleted (required by Windows and POSIX rmdir()). -LibError dir_delete(const char* P_path) -{ - char N_path[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(P_path, N_path)); - PathPackage N_pp; - RETURN_ERR(path_package_set_dir(&N_pp, N_path)); - - DirIterator di; - RETURN_ERR(dir_open(P_path, &di)); - - LibError ret; - - for(;;) - { - DirEnt ent; - ret = dir_next_ent(&di, &ent); - if(ret == ERR::DIR_END) - break; - if(ret != INFO::OK) goto fail; - - if(DIRENT_IS_DIR(&ent)) - { - char P_subdir[PATH_MAX]; - ret = path_append(P_subdir, P_path, ent.name); - if(ret != INFO::OK) goto fail; - ret = dir_delete(P_subdir); - if(ret != INFO::OK) goto fail; - } - else - { - ret = path_package_append_file(&N_pp, ent.name); - if(ret != INFO::OK) goto fail; - - errno = 0; - int posix_ret = unlink(N_pp.path); - ret = LibError_from_posix(posix_ret); - if(ret != INFO::OK) goto fail; - } - } - - // must happen before rmdir - RETURN_ERR(dir_close(&di)); - - { - errno = 0; - int posix_ret = rmdir(N_path); - return LibError_from_posix(posix_ret); - } - -fail: - RETURN_ERR(dir_close(&di)); - return ret; -} - - -// get file information. output param is zeroed on error. -static LibError file_stat_impl(const char* fn, struct stat* s, bool warn_if_failed = true) -{ - memset(s, 0, sizeof(struct stat)); - - char N_fn[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(fn, N_fn)); - - errno = 0; - int ret = stat(N_fn, s); - return LibError_from_posix(ret, warn_if_failed); -} - -LibError file_stat(const char* fn, struct stat* s) -{ - return file_stat_impl(fn, s); -} - -// does the given file exist? (implemented via file_stat) -bool file_exists(const char* fn) -{ - struct stat s; - const bool warn_if_failed = false; - return file_stat_impl(fn, &s, warn_if_failed) == INFO::OK; -} - - -// permanently delete the file. be very careful with this! -LibError file_delete(const char* fn) -{ - char N_fn[PATH_MAX+1]; - RETURN_ERR(file_make_full_native_path(fn, N_fn)); - - errno = 0; - int ret = unlink(N_fn); - return LibError_from_posix(ret); -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// file open/close -// stores information about file (e.g. size) in File struct -// -/////////////////////////////////////////////////////////////////////////////// - -// interface rationale: -// - this module depends on the handle manager for IO management, -// but should be useable without the VFS (even if they are designed -// to work together). -// - allocating a Handle for the file info would solve several problems -// (see below), but we don't want to allocate 2..3 (VFS, file, Zip file) -// for every file opened - that'd add up quickly. -// the Files are always freed at exit though, since they're part of -// VFile handles in the VFS. -// - we want the VFS open logic to be triggered on file invalidate -// (if the dev. file is deleted, we should use what's in the archives). -// we don't want to make this module depend on VFS, so we don't -// have access to the file location DB; VFS needs to allocate the handle. -// - no problem exposing our internals via File struct - -// we're only used by the VFS and Zip modules. don't bother making -// an opaque struct - that'd have to be kept in sync with the real thing. -// - when Zip opens its archives via file_open, a handle isn't needed - -// the Zip module hides its File struct (required to close the file), -// and the Handle approach doesn't guard against some idiot calling -// close(our_fd_value) directly, either. - - -struct PosixFile -{ - int fd; - - // for reference counted memory-mapping - u8* mapping; - uint map_refs; -}; -cassert(sizeof(PosixFile) < FILE_OPAQUE_SIZE); - -int file_fd_from_PosixFile(File* f) -{ - const PosixFile* pf = (const PosixFile*)f->opaque; - return pf->fd; -} - - -LibError file_validate(const File* f) -{ - if(!f) - WARN_RETURN(ERR::INVALID_PARAM); - const PosixFile* pf = (PosixFile*)f->opaque; - if(pf->fd < 0) - WARN_RETURN(ERR::_1); - // mapped but refcount is invalid - else if((pf->mapping != 0) ^ (pf->map_refs != 0)) - WARN_RETURN(ERR::_2); - // note: don't check atom_fn - that complains at the end of - // file_open if flags & FILE_DONT_SET_FN and has no benefit, really. - - return INFO::OK; -} - - -LibError file_open(const char* P_fn, uint flags, File* f) -{ - // zero output param in case we fail below. - memset(f, 0, sizeof(*f)); - - if(flags > FILE_FLAG_ALL) - WARN_RETURN(ERR::INVALID_PARAM); - - char N_fn[PATH_MAX]; - RETURN_ERR(file_make_full_native_path(P_fn, N_fn)); - - // don't stat if opening for writing - the file may not exist yet - off_t size = 0; - - int oflag = O_RDONLY; - if(flags & FILE_WRITE) - oflag = O_WRONLY|O_CREAT|O_TRUNC; - // read access requested - else - { - // get file size - struct stat s; - if(stat(N_fn, &s) < 0) - WARN_RETURN(ERR::TNODE_NOT_FOUND); - size = s.st_size; - - // note: despite increased overhead, the AIO read method is still - // significantly faster, even with small files. - // we therefore don't automatically disable AIO. - // notes: - // - up to 32KB can be read by one SCSI request. - // - flags are stored below and will influence file_io. - //if(size <= 32*KiB) - // flags |= FILE_NO_AIO; - - // make sure is a regular file - if(!S_ISREG(s.st_mode)) - WARN_RETURN(ERR::TNODE_WRONG_TYPE); - } - -#if OS_WIN - if(flags & FILE_TEXT) - oflag |= O_TEXT_NP; - else - oflag |= O_BINARY_NP; - - // if AIO is disabled at user's behest, so inform wposix. - if(flags & FILE_NO_AIO) - oflag |= O_NO_AIO_NP; -#endif - - int fd = open(N_fn, oflag, S_IRWXO|S_IRWXU|S_IRWXG); - if(fd < 0) - WARN_RETURN(ERR::FILE_ACCESS); - - f->flags = flags; - f->size = size; - // see FILE_DONT_SET_FN decl. - if(!(flags & FILE_DONT_SET_FN)) - f->atom_fn = file_make_unique_fn_copy(P_fn); - PosixFile* pf = (PosixFile*)f->opaque; - pf->mapping = 0; - pf->map_refs = 0; - pf->fd = fd; - CHECK_FILE(f); - - return INFO::OK; -} - - -LibError file_close(File* f) -{ - CHECK_FILE(f); - PosixFile* pf = (PosixFile*)f->opaque; - - // make sure the mapping is actually freed, - // regardless of how many references remain. - if(pf->map_refs > 1) - pf->map_refs = 1; - if(pf->mapping) // only free if necessary (unmap complains if not mapped) - file_unmap(f); - - // return final file size (required by VFS after writing files). - // this is much easier than updating when writing, because we'd have - // to add accounting code to both (sync and async) paths. - f->size = lseek(pf->fd, 0, SEEK_END); - - // (check fd to avoid BoundsChecker warning about invalid close() param) - if(pf->fd != -1) - { - close(pf->fd); - pf->fd = -1; - } - - // wipe out any cached blocks. this is necessary to cover the (rare) case - // of file cache contents predating the file write. - if(f->flags & FILE_WRITE) - file_cache_invalidate(f->atom_fn); - - return INFO::OK; -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// memory mapping -// -/////////////////////////////////////////////////////////////////////////////// - - -// no significance aside from preventing uint overflow. -static const uint MAX_MAP_REFS = 255; - - -// map the entire file into memory. if already currently mapped, -// return the previous mapping (reference-counted). -// output parameters are zeroed on failure. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -// -// rationale: reference counting is required for zip_map: several -// Zip "mappings" each reference one ZArchive's actual file mapping. -// implement it here so that we also get refcounting for normal files. -LibError file_map(File* f, u8*& p, size_t& size) -{ - p = 0; - size = 0; - - CHECK_FILE(f); - PosixFile* pf = (PosixFile*)f->opaque; - - const int prot = (f->flags & FILE_WRITE)? PROT_WRITE : PROT_READ; - - // already mapped - increase refcount and return previous mapping. - if(pf->mapping) - { - // prevent overflow; if we have this many refs, should find out why. - if(pf->map_refs >= MAX_MAP_REFS) - WARN_RETURN(ERR::LIMIT); - pf->map_refs++; - goto have_mapping; - } - - // don't allow mapping zero-length files (doesn't make sense, - // and BoundsChecker warns about wposix mmap failing). - // then again, don't complain, because this might happen when mounting - // a dir containing empty files; each is opened as a Zip file. - if(f->size == 0) - return ERR::FAIL; // NOWARN - - errno = 0; - pf->mapping = (u8*)mmap(0, f->size, prot, MAP_PRIVATE, pf->fd, (off_t)0); - if(pf->mapping == MAP_FAILED) - return LibError_from_errno(); - - pf->map_refs = 1; - -have_mapping: - p = pf->mapping; - size = f->size; - return INFO::OK; -} - - -// decrement the reference count for the mapping belonging to file . -// fail if there are no references; remove the mapping if the count reaches 0. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -LibError file_unmap(File* f) -{ - CHECK_FILE(f); - PosixFile* pf = (PosixFile*)f->opaque; - - // file is not currently mapped - if(pf->map_refs == 0) - WARN_RETURN(ERR::FILE_NOT_MAPPED); - - // still more than one reference remaining - done. - if(--pf->map_refs > 0) - return INFO::OK; - - // no more references: remove the mapping - u8* p = pf->mapping; - pf->mapping = 0; - // don't clear f->size - the file is still open. - - errno = 0; - int ret = munmap(p, f->size); - return LibError_from_posix(ret); -} - - -LibError file_init() -{ - path_init(); - file_cache_init(); - file_io_init(); - - // convenience - file_sector_size = sys_max_sector_size(); - - return INFO::OK; -} - -LibError file_shutdown() -{ - file_stats_dump(); - path_shutdown(); - file_io_shutdown(); - return INFO::OK; -} diff --git a/source/lib/res/file/file.h b/source/lib/res/file/file.h deleted file mode 100644 index 7aa544790a..0000000000 --- a/source/lib/res/file/file.h +++ /dev/null @@ -1,357 +0,0 @@ -/** - * ========================================================================= - * File : file.h - * Project : 0 A.D. - * Description : file layer on top of POSIX. avoids the need for - * : absolute paths and provides fast I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_FILE -#define INCLUDED_FILE - -#include "lib/posix/posix_filesystem.h" // struct stat - - -namespace ERR -{ - const LibError FILE_ACCESS = -110000; - const LibError FILE_NOT_MAPPED = -110001; - const LibError DIR_END = -110002; -} - -extern LibError file_init(); - -// used by vfs_redirector to call various file objects' methods. -struct FileProvider_VTbl; - - -// -// path conversion functions (native <--> portable), -// for external libraries that require the real filename. -// -// replaces '/' with platform's directory separator and vice versa. -// verifies path length < PATH_MAX (otherwise return ERR::PATH_LENGTH). -// - -// relative paths (relative to root dir) -extern LibError file_make_native_path(const char* path, char* n_path); -extern LibError file_make_portable_path(const char* n_path, char* path); - -// as above, but with full native paths (portable paths are always relative). -// prepends current directory, resp. makes sure it matches the given path. -extern LibError file_make_full_native_path(const char* path, char* n_full_path); -extern LibError file_make_full_portable_path(const char* n_full_path, char* path); - - -// establish the root directory from , which is treated as -// relative to the executable's directory (determined via argv[0]). -// all relative file paths passed to this module will be based from -// this root dir. -// -// example: executable in "$install_dir/system"; desired root dir is -// "$install_dir/data" => rel_path = "../data". -// -// argv[0] is necessary because the current directory is unknown at startup -// (e.g. it isn't set when invoked via batch file), and this is the -// easiest portable way to find our install directory. -// -// can only be called once, by design (see below). rel_path is trusted. -extern LibError file_set_root_dir(const char* argv0, const char* rel_path); - - -// allocate a copy of P_fn in our string pool. strings are equal iff -// their addresses are equal, thus allowing fast comparison. -// -// if the (generous) filename storage is full, 0 is returned. -// this is not ever expected to happen; callers need not check the -// return value because a warning is raised anyway. -extern const char* file_make_unique_fn_copy(const char* P_fn); - -extern const char* file_get_random_name(); - - -// -// directory -// - -const size_t DIR_ITERATOR_OPAQUE_SIZE = 40; - -// layer on top of POSIX opendir/readdir/closedir that handles -// portable -> native path conversion, ignores non-file/directory entries, -// and additionally returns the file status (size and mtime). - -// directory state initialized by dir_open. -// rationale: some private storage apart from opendir's DIR* is required -// to support stat(). we prefer having the caller reserve room (on the stack) -// rather than allocating dynamically (less efficient or more complicated). -// -// this is an opaque struct to avoid exposing our internals and insulate -// user code against changes; we verify at compile-time that the -// public/private definitions match. -// note: cannot just typedef to DirIterator_ because other modules -// instantiate this. -struct DirIterator -{ - // safety check - used to verify correct calling of dir_filtered_next_ent - const char* filter; - // .. has filter been assigned? this flag is necessary because - // there are no "invalid" filter values we can use. - uint filter_latched : 1; - - const FileProvider_VTbl* type; - - char opaque[DIR_ITERATOR_OPAQUE_SIZE]; -}; - -class TFile; - -// information about a directory entry filled by dir_next_ent. -struct DirEnt -{ - // we want to keep this as small as possible because - // file_enum allocates one copy for each file in the directory. - - // store only required stat fields (in VC's order of decl) - off_t size; - time_t mtime; - - // name (not including path!) of this entry. - // valid until a subsequent dir_next_ent or dir_close call for the - // current dir state. - // rationale: we don't want to return a pointer to a copy because - // users would have to free it (won't happen). - const char* name; - - const TFile* tf; -}; - -// return [bool] indicating whether the given DirEnt* (filled by -// dir_next_ent) represents a directory. -#define DIRENT_IS_DIR(p_ent) ((p_ent)->size == -1) - -// prepare to iterate (once) over entries in the given directory. -// if INFO::OK is returned, is ready for subsequent dir_next_ent calls and -// must be freed via dir_close. -extern LibError dir_open(const char* P_path, DirIterator* d); - -// return ERR::DIR_END if all entries have already been returned once, -// another negative error code, or INFO::OK on success, in which case -// describes the next (order is unspecified) directory entry. -extern LibError dir_next_ent(DirIterator* d, DirEnt* ent); - -// indicate the directory iterator is no longer needed; all resources it -// held are freed. -extern LibError dir_close(DirIterator* d); - - -extern bool dir_exists(const char* P_path); -extern LibError dir_create(const char* P_path); -extern LibError dir_delete(const char* P_path); - - -#ifdef __cplusplus - -typedef std::vector DirEnts; -typedef DirEnts::iterator DirEntIt; -typedef DirEnts::const_iterator DirEntCIt; - -// enumerate all directory entries in ; add to container and -// then sort it by filename. -extern LibError file_get_sorted_dirents(const char* P_path, DirEnts& dirents); - -#endif // #ifdef __cplusplus - - -// called by file_enum for each entry in the directory. -// name doesn't include path! -// return INFO::CB_CONTINUE to continue calling; anything else will cause -// file_enum to abort and immediately return that value. -typedef LibError (*FileCB)(const char* name, const struct stat* s, uintptr_t memento, const uintptr_t user); - -// call for each file and subdirectory in (alphabetical order), -// passing the entry name (not full path!), stat info, and . -// -// first builds a list of entries (sorted) and remembers if an error occurred. -// if returns non-zero, abort immediately and return that; otherwise, -// return first error encountered while listing files, or 0 on success. -extern LibError file_enum(const char* dir, FileCB cb, uintptr_t user); - -// chosen for semi-nice 48 byte total struct File size. -// each implementation checks if this is enough. -const size_t FILE_OPAQUE_SIZE = 52; - -// represents an open file of any type (OS, archive, VFS). -// contains common fields and opaque storage for type-specific fields. -// -// this cannot merely be added in a separate VFS layer: it would want to -// share some common fields, which either requires this approach -// (one publically visible struct with space for private storage), or -// a common struct layout / embedding a FileCommon struct at -// the beginning. the latter is a bit messy since fields must be accessed -// as e.g. af->fc.flags. one shared struct also makes for a common -// interface. -struct File -{ - uint flags; - off_t size; - - // copy of the filename that is uniquely identified by its address. - // used as key for file cache. - // NOTE: not set by file_open! (because the path passed there is - // a native path; it has no use within VFS and would only - // unnecessarily clutter the filename storage) - const char* atom_fn; - - // can be 0 if not currently in use; otherwise, points to - // the file provider's vtbl. - const FileProvider_VTbl* type; - - // storage for the provider-specific fields. - // the implementations cast this to their e.g. PosixFile struct. - // - // note: when doing so, there's no need to verify type - if - // vfs_io dispatches to afile_read, then the File.type must obviously - // have been "archive". - // if users call the e.g. archive.h methods directly, we assume they - // know what they're doing and don't check that. - u8 opaque[FILE_OPAQUE_SIZE]; -}; - -// note: these are all set during file_open and cannot be changed thereafter. -enum FileFlags -{ - // IO: - // ------------------------------------------------------------------------ - - // write-only access; otherwise, read only. - // - // unless FILE_NO_AIO is set, data that is to be written must be - // aligned and padded to a multiple of file_sector_size bytes; - // this requirement avoids the need for align buffers. - // - // note: only allowing either reads or writes simplifies file cache - // coherency (need only invalidate when closing a FILE_WRITE file). - FILE_WRITE = 0x01, - - // translate newlines: convert from/to native representation when - // reading/writing. this is useful if files we create need to be - // edited externally - e.g. Notepad requires \r\n. - // caveats: - // - FILE_NO_AIO must be set; translation is done by OS read()/write(). - // - not supported by POSIX, so this currently only has meaning on Win32. - FILE_TEXT = 0x02, - - // skip the aio path and use the OS-provided synchronous blocking - // read()/write() calls. this avoids the need for buffer alignment - // set out below, so it's useful for writing small text files. - FILE_NO_AIO = 0x04, - - // caching: - // ------------------------------------------------------------------------ - - // do not add the (entire) contents of this file to the cache. - // this flag should be specified when the data is cached at a higher - // level (e.g. OpenGL textures) to avoid wasting previous cache space. - FILE_CACHED_AT_HIGHER_LEVEL = 0x10, - - // enable caching individual blocks read from a file. the block cache - // is small, organized as LRU and incurs some copying overhead, so it - // should only be enabled when needed. this is the case for archives, - // where the cache absorbs overhead of block-aligning all IOs. - FILE_CACHE_BLOCK = 0x20, - - // notify us that the file buffer returned by file_io will not be - // freed immediately (i.e. before the next allocation). - // allocation policy may differ and a warning is suppressed. - FILE_LONG_LIVED = 0x40, - - // misc: - // ------------------------------------------------------------------------ - - // instruct file_open not to set FileCommon.atom_fn. - // this is a slight optimization used by VFS code: file_open - // would store the portable name, which is only used when calling - // the OS's open(); this would unnecessarily waste atom_fn memory. - // - // note: other file.cpp functions require atom_fn to be set, - // so this behavior is only triggered via flag (caller is - // promising they will set atom_fn). - FILE_DONT_SET_FN = 0x80, - - // (only relevant for VFS) file will be written into the - // appropriate subdirectory of the mount point established by - // vfs_set_write_target. see documentation there. - FILE_WRITE_TO_TARGET = FILE_WRITE|0x100, - - // sum of all flags above. used when validating flag parameters. - FILE_FLAG_ALL = 0x1FF -}; - - -// get file information. output param is zeroed on error. -extern LibError file_stat(const char* path, struct stat*); - -// does the given file exist? (implemented via file_stat) -extern bool file_exists(const char* fn); - -// permanently delete the file. be very careful with this! -extern LibError file_delete(const char* fn); - -// is ignored here. -// rationale: all file providers' open() routines should ideally take the -// same parameters. since afile_open requires archive Handle and -// memento, we need some way of passing them; TFile is sufficient -// (via vfs_tree accessor methods). -extern LibError file_open(const char* fn, uint flags, File* f); - -// note: final file size is calculated and returned in f->size. -// see implementation for rationale. -extern LibError file_close(File* f); - -extern LibError file_validate(const File* f); -#define CHECK_FILE(f) RETURN_ERR(file_validate(f)) - - -// remove all blocks loaded from the file . used when reloading the file. -extern LibError file_cache_invalidate(const char* fn); - - -#include "file_io.h" - - -// -// memory mapping -// - -// useful for files that are too large to be loaded into memory, -// or if only (non-sequential) portions of a file are needed at a time. - - -// map the entire file into memory. if already currently mapped, -// return the previous mapping (reference-counted). -// output parameters are zeroed on failure. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -// -// rationale: reference counting is required for zip_map: several -// Zip "mappings" each reference one ZArchive's actual file mapping. -// implement it here so that we also get refcounting for normal files. -extern LibError file_map(File* f, u8*& p, size_t& size); - -// decrement the reference count for the mapping belonging to file . -// fail if there are no references; remove the mapping if the count reaches 0. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -extern LibError file_unmap(File* f); - - -extern LibError file_shutdown(); - -#endif // #ifndef INCLUDED_FILE diff --git a/source/lib/res/file/file_cache.cpp b/source/lib/res/file/file_cache.cpp deleted file mode 100644 index b4172e3431..0000000000 --- a/source/lib/res/file/file_cache.cpp +++ /dev/null @@ -1,1295 +0,0 @@ -/** - * ========================================================================= - * File : file_cache.cpp - * Project : 0 A.D. - * Description : cache for entire files and I/O blocks. also allocates - * : file buffers, allowing zero-copy I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "file_cache.h" - -#include - -#include "lib/posix/posix_mman.h" -#include "lib/allocators.h" -#include "lib/bits.h" -#include "lib/byte_order.h" -#include "lib/cache_adt.h" -#include "file_internal.h" - -//----------------------------------------------------------------------------- - -// block cache: intended to cache raw compressed data, since files aren't aligned -// in the archive; alignment code would force a read of the whole block, -// which would be a slowdown unless we keep them in memory. -// -// keep out of async code (although extra work for sync: must not issue/wait -// if was cached) to simplify things. disadvantage: problems if same block -// is issued twice, before the first call completes (via wait_io). -// that won't happen though unless we have threaded file_ios => -// rare enough not to worry about performance. -// -// since sync code allocates the (temp) buffer, it's guaranteed -// to remain valid. -// - -class BlockMgr -{ - static const size_t MAX_BLOCKS = 32; - enum BlockStatus - { - BS_PENDING, - BS_COMPLETE, - BS_INVALID - }; - struct Block - { - BlockId id; - // initialized in BlockMgr ctor and remains valid - void* mem; - BlockStatus status; - int refs; - - Block() - : id(block_cache_make_id(0, 0)), status(BS_INVALID), refs(0) {} - }; - // access pattern is usually ring buffer, but in rare cases we - // need to skip over locked items, even though they are the oldest. - Block blocks[MAX_BLOCKS]; - uint oldest_block; - - // use Pool to allocate mem for all blocks because it guarantees - // page alignment (required for IO) and obviates manually aligning. - Pool pool; - -public: - BlockMgr() - : blocks(), oldest_block(0) - { - (void)pool_create(&pool, MAX_BLOCKS*FILE_BLOCK_SIZE, FILE_BLOCK_SIZE); - for(Block* b = blocks; b < blocks+MAX_BLOCKS; b++) - { - b->mem = pool_alloc(&pool, 0); - debug_assert(b->mem); // shouldn't ever fail - } - } - - void shutdown() - { - (void)pool_destroy(&pool); - } - - void* alloc(BlockId id) - { - Block* b; - for(b = blocks; b < blocks+MAX_BLOCKS; b++) - { - if(block_eq(b->id, id)) - debug_warn("allocating block that is already in list"); - } - - for(size_t i = 0; i < MAX_BLOCKS; i++) - { - b = &blocks[oldest_block]; - oldest_block = (oldest_block+1)%MAX_BLOCKS; - - // normal case: oldest item can be reused - if(b->status != BS_PENDING && b->refs == 0) - goto have_block; - - // wacky special case: oldest item is currently locked. - // skip it and reuse the next. - // - // to see when this can happen, consider IO depth = 4. - // let the Block at blocks[oldest_block] contain data that - // an IO wants. the 2nd and 3rd blocks are not in cache and - // happen to be taken from near the end of blocks[]. - // attempting to issue block #4 fails because its buffer would - // want the first slot (which is locked since the its IO - // is still pending). - if(b->status == BS_COMPLETE && b->refs > 0) - continue; - - debug_warn("status and/or refs have unexpected values"); - } - - debug_warn("all blocks are locked"); - return 0; -have_block: - - b->id = id; - b->status = BS_PENDING; - return b->mem; - } - - void mark_completed(BlockId id) - { - for(Block* b = blocks; b < blocks+MAX_BLOCKS; b++) - { - if(block_eq(b->id, id)) - { - debug_assert(b->status == BS_PENDING); - b->status = BS_COMPLETE; - return; - } - } - debug_warn("mark_completed: block not found, but ought still to be in cache"); - } - - void* find(BlockId id) - { - // linear search is ok, since we only keep a few blocks. - for(Block* b = blocks; b < blocks+MAX_BLOCKS; b++) - { - if(block_eq(b->id, id)) - { - if(b->status == BS_COMPLETE) - { - debug_assert(b->refs >= 0); - b->refs++; - return b->mem; - } - - debug_warn("block referenced while still in progress"); - return 0; - } - } - return 0; // not found - } - - void release(BlockId id) - { - for(Block* b = blocks; b < blocks+MAX_BLOCKS; b++) - { - if(block_eq(b->id, id)) - { - b->refs--; - debug_assert(b->refs >= 0); - return; - } - } - debug_warn("release: block not found, but ought still to be in cache"); - } - - void invalidate(const char* atom_fn) - { - for(Block* b = blocks; b < blocks+MAX_BLOCKS; b++) - { - if(b->id.atom_fn == atom_fn) - { - if(b->refs) - debug_warn("invalidating block that is currently in-use"); - b->status = BS_INVALID; - } - } - } -}; -static BlockMgr block_mgr; - - -bool block_eq(BlockId b1, BlockId b2) -{ - return b1.atom_fn == b2.atom_fn && b1.block_num == b2.block_num; -} - -// create an id for use with the cache that uniquely identifies -// the block from the file starting at . -BlockId block_cache_make_id(const char* atom_fn, const off_t ofs) -{ - // is guaranteed to be unique (see file_make_unique_fn_copy). - // block_num should always fit in 32 bits (assuming maximum file size - // = 2^32 * FILE_BLOCK_SIZE ~= 2^48 -- plenty). we don't bother - // checking this. - const u32 block_num = (u32)(ofs / FILE_BLOCK_SIZE); - BlockId id = { atom_fn, block_num }; - return id; -} - -void* block_cache_alloc(BlockId id) -{ - return block_mgr.alloc(id); -} - -void block_cache_mark_completed(BlockId id) -{ - block_mgr.mark_completed(id); -} - -void* block_cache_find(BlockId id) -{ - void* ret = block_mgr.find(id); - stats_block_cache(ret? CR_HIT : CR_MISS); - return ret; -} - -void block_cache_release(BlockId id) -{ - return block_mgr.release(id); -} - - -//----------------------------------------------------------------------------- - -// >= file_sector_size or else waio will have to realign. -// chosen as exactly 1 page: this allows write-protecting file buffers -// without worrying about their (non-page-aligned) borders. -// internal fragmentation is considerable but acceptable. -static const size_t BUF_ALIGN = 4*KiB; - -/* -CacheAllocator - -the biggest worry of a file cache is fragmentation. there are 2 -basic approaches to combat this: -1) 'defragment' periodically - move blocks around to increase - size of available 'holes'. -2) prevent fragmentation from occurring at all via - deliberate alloc/free policy. - -file_io returns cache blocks directly to the user (zero-copy IO), -so only currently unreferenced blocks can be moved (while holding a -lock, to boot). it is believed that this would severely hamper -defragmentation; we therefore go with the latter approach. - -basic insight is: fragmentation occurs when a block is freed whose -neighbors are not free (thus preventing coalescing). this can be -prevented by allocating objects of similar lifetimes together. -typical workloads (uniform access frequency) already show such behavior: -the Landlord cache manager evicts files in an LRU manner, which matches -the allocation policy. - -references: -"The Memory Fragmentation Problem - Solved?" (Johnstone and Wilson) -"Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson) - -policy: -- allocation: use all available mem first, then look at freelist -- freelist: good fit, address-ordered, always split blocks -- free: immediately coalesce -mechanism: -- coalesce: boundary tags in freed memory with magic value -- freelist: 2**n segregated doubly-linked, address-ordered -*/ -static const size_t MAX_CACHE_SIZE = 96*MiB; - -class CacheAllocator -{ -public: - CacheAllocator() - : bitmap(0), freelists() - { - // (safe to call this from ctor as of 2006-02-02) - (void)pool_create(&pool, MAX_CACHE_SIZE, 0); - } - - void shutdown() - { - (void)pool_destroy(&pool); - } - - void* alloc(size_t size) - { - // determine actual size to allocate - // .. better not be more than MAX_CACHE_SIZE - file_buf_alloc will - // fail because no amount of freeing up existing allocations - // would make enough room. therefore, check for this here - // (should never happen). - debug_assert(size < MAX_CACHE_SIZE); - // .. safely handle 0 byte allocations. according to C/C++ tradition, - // we allocate a unique address, which ends up wasting 1 page. - if(!size) - size = 1; - // .. each allocation must be aligned to BUF_ALIGN, so - // we round up all sizes to that. - const size_t size_pa = round_up(size, BUF_ALIGN); - const uint size_class = size_class_of(size_pa); - - void* p; - - // try to reuse a freed entry - p = alloc_from_class(size_class, size_pa); - if(p) - goto success; - - // grab more space from pool - p = pool_alloc(&pool, size_pa); - if(p) - goto success; - - // last resort: split a larger element - p = alloc_from_larger_class(size_class, size_pa); - if(p) - goto success; - - // failed - can no longer expand and nothing big enough was - // found in freelists. - // file cache will decide which elements are least valuable, - // free() those and call us again. - return 0; - -success: -#ifndef NDEBUG - alloc_checker.notify_alloc(p, size); -#endif - stats_notify_alloc(size_pa); - return p; - } - - // rationale: don't call this "free" because that would run afoul of the - // memory tracker's redirection macro and require #include "lib/nommgr.h". - void dealloc(void* p, size_t size) - { -#ifndef NDEBUG - alloc_checker.notify_free(p, size); -#endif - - const size_t size_pa = round_up(size, BUF_ALIGN); - // make sure entire (aligned!) range is within pool. - if(!pool_contains(&pool, p) || !pool_contains(&pool, (u8*)p+size_pa-1)) - { - debug_warn("invalid pointer"); - return; - } - - // (re)allow writes - // - // note: unfortunately we cannot unmap this buffer's memory - // (to make sure it is not used) because we write a header/footer - // into it to support coalescing. - (void)mprotect(p, size_pa, PROT_READ|PROT_WRITE); - - coalesce_and_free(p, size_pa); - - stats_notify_free(size_pa); - } - - // make given range read-only via MMU. - // write access is restored when buffer is freed. - // - // p and size are the exact (non-padded) values as in dealloc. - void make_read_only(void* p, size_t size) - { - // bail to avoid mprotect failing - if(!size) - return; - - const size_t size_pa = round_up(size, BUF_ALIGN); - (void)mprotect(p, size_pa, PROT_READ); - } - - // free all allocations and reset state to how it was just after - // (the first and only) init() call. - void reset() - { -#ifndef NDEBUG - alloc_checker.notify_clear(); -#endif - - pool_free_all(&pool); - bitmap = 0; - memset(freelists, 0, sizeof(freelists)); - stats_reset(); - } - -private: -#ifndef NDEBUG - AllocatorChecker alloc_checker; -#endif - - Pool pool; - - //------------------------------------------------------------------------- - // boundary tags for coalescing - static const u32 HEADER_ID = FOURCC('C','M','A','H'); - static const u32 FOOTER_ID = FOURCC('C','M','A','F'); - static const u32 MAGIC = FOURCC('\xFF','\x55','\xAA','\x01'); - struct Header - { - Header* prev; - Header* next; - size_t size_pa; - u32 id; - u32 magic; - }; - // we could use struct Header for Footer as well, but keeping them - // separate and different can avoid coding errors (e.g. mustn't pass a - // Footer to freelist_remove!) - struct Footer - { - // note: deliberately reordered fields for safety - u32 magic; - u32 id; - size_t size_pa; - }; - // must be enough room to stash Header+Footer within the freed allocation. - cassert(BUF_ALIGN >= sizeof(Header)+sizeof(Footer)); - - // expected_id identifies the tag type (either HEADER_ID or - // FOOTER_ID). returns whether the given id, magic and size_pa - // values are consistent with such a tag. - // - // note: these magic values are all that differentiates tags from - // user data. this isn't 100% reliable, but we can't insert extra - // boundary tags because the memory must remain aligned. - bool is_valid_tag(u32 expected_id, u32 id, u32 magic, size_t size_pa) const - { - if(id != expected_id || magic != MAGIC) - return false; - debug_assert(size_pa % BUF_ALIGN == 0); - debug_assert(size_pa <= MAX_CACHE_SIZE); - return true; - } - - // add p to freelist; if its neighbor(s) are free, merges them all into - // one big region and frees that. - // notes: - // - correctly deals with p lying at start/end of pool. - // - p and size_pa are trusted: [p, p+size_pa) lies within the pool. - void coalesce_and_free(void* p, size_t size_pa) - { - // CAVEAT: Header and Footer are wiped out by freelist_remove - - // must use them before that. - - // expand (p, size_pa) to include previous allocation if it's free. - // (unless p is at start of pool region) - if(p != pool.da.base) - { - const Footer* footer = (const Footer*)((u8*)p-sizeof(Footer)); - if(is_valid_tag(FOOTER_ID, footer->id, footer->magic, footer->size_pa)) - { - (u8*&)p -= footer->size_pa; - size_pa += footer->size_pa; - Header* header = (Header*)p; - freelist_remove(header); - } - } - - // expand size_pa to include following memory if it was allocated - // and is currently free. - // (unless it starts beyond end of currently committed region) - Header* header = (Header*)((u8*)p+size_pa); - if((u8*)header < pool.da.base+pool.da.cur_size) - { - if(is_valid_tag(HEADER_ID, header->id, header->magic, header->size_pa)) - { - size_pa += header->size_pa; - freelist_remove(header); - } - } - - freelist_add(p, size_pa); - } - - //------------------------------------------------------------------------- - // freelist - - // segregated, i.e. one list per size class. - // note: we store Header nodes instead of just a pointer to head of - // list - this wastes a bit of mem but greatly simplifies list insertion. - Header freelists[sizeof(uintptr_t)*CHAR_BIT]; - - // bit i set iff size class i's freelist is not empty. - // in conjunction with ls1, this allows finding a non-empty list in O(1). - uintptr_t bitmap; - - // "size class" i (>= 0) contains allocations of size (2**(i-1), 2**i] - // except for i=0, which corresponds to size=1. - static uint size_class_of(size_t size_pa) - { - return ceil_log2((uint)size_pa); - } - - // value of LSB 1-bit. - static uint ls1(uint x) - { - return (x & -(int)x); - } - - void freelist_add(void* p, size_t size_pa) - { - debug_assert((uintptr_t)p % BUF_ALIGN == 0); - debug_assert(size_pa % BUF_ALIGN == 0); - const uint size_class = size_class_of(size_pa); - - // write header and footer into the freed mem - // (its prev and next link fields will be set below) - Header* header = (Header*)p; - header->id = HEADER_ID; - header->magic = MAGIC; - header->size_pa = size_pa; - Footer* footer = (Footer*)((u8*)p+size_pa-sizeof(Footer)); - footer->id = FOOTER_ID; - footer->magic = MAGIC; - footer->size_pa = size_pa; - - Header* prev = &freelists[size_class]; - // find node after which to insert (address ordered freelist) - while(prev->next && header <= prev->next) - prev = prev->next; - - header->next = prev->next; - header->prev = prev; - if(prev->next) - prev->next->prev = header; - prev->next = header; - - bitmap |= BIT(size_class); - } - - void freelist_remove(Header* header) - { - debug_assert((uintptr_t)header % BUF_ALIGN == 0); - - Footer* footer = (Footer*)((u8*)header+header->size_pa-sizeof(Footer)); - debug_assert(is_valid_tag(HEADER_ID, header->id, header->magic, header->size_pa)); - debug_assert(is_valid_tag(FOOTER_ID, footer->id, footer->magic, footer->size_pa)); - debug_assert(header->size_pa == footer->size_pa); - const uint size_class = size_class_of(header->size_pa); - - header->prev->next = header->next; - if(header->next) - header->next->prev = header->prev; - - // if freelist is now empty, clear bit in bitmap. - if(!freelists[size_class].next) - bitmap &= ~BIT(size_class); - - // wipe out header and footer to prevent accidental reuse - memset(header, 0xEE, sizeof(Header)); - memset(footer, 0xEE, sizeof(Footer)); - } - - // returns 0 if nothing big enough is in size_class's freelist. - void* alloc_from_class(uint size_class, size_t size_pa) - { - // return first suitable entry in (address-ordered) list - for(Header* cur = freelists[size_class].next; cur; cur = cur->next) - { - if(cur->size_pa >= size_pa) - { - void* p = cur; - const size_t remnant_pa = cur->size_pa - size_pa; - - freelist_remove(cur); - - if(remnant_pa) - freelist_add((u8*)p+size_pa, remnant_pa); - - return p; - } - } - - return 0; - } - - // returns 0 if there is no big enough entry in any freelist. - void* alloc_from_larger_class(uint start_size_class, size_t size_pa) - { - uint classes_left = bitmap; - // .. strip off all smaller classes - classes_left &= (~0 << start_size_class); - - // for each non-empty freelist (loop doesn't incur overhead for - // empty freelists) - while(classes_left) - { - const uint class_size = ls1(classes_left); - classes_left &= ~class_size; // remove from classes_left - const uint size_class = size_class_of(class_size); - - // .. try to alloc - void* p = alloc_from_class(size_class, size_pa); - if(p) - return p; - } - - // apparently all classes above start_size_class are empty, - // or the above would have succeeded. - debug_assert(bitmap < BIT(start_size_class+1)); - return 0; - } - - //------------------------------------------------------------------------- - // stats and validation - size_t allocated_size_total_pa, free_size_total_pa; - - void stats_notify_alloc(size_t size_pa) { allocated_size_total_pa += size_pa; } - void stats_notify_free(size_t size_pa) { free_size_total_pa += size_pa; } - void stats_reset() { allocated_size_total_pa = free_size_total_pa = 0; } - - void self_check() const - { - debug_assert(allocated_size_total_pa+free_size_total_pa == pool.da.cur_size); - - // make sure freelists contain exactly free_size_total_pa bytes - size_t freelist_size_total_pa = 0; - uint classes_left = bitmap; - while(classes_left) - { - const uint class_size = ls1(classes_left); - classes_left &= ~class_size; // remove from classes_left - const uint size_class = size_class_of(class_size); - for(const Header* p = &freelists[size_class]; p; p = p->next) - freelist_size_total_pa += p->size_pa; - } - debug_assert(free_size_total_pa == freelist_size_total_pa); - } -}; // CacheAllocator - -static CacheAllocator cache_allocator; - -//----------------------------------------------------------------------------- - -/* -list of FileIOBufs currently held by the application. - -note: "currently held" means between a file_buf_alloc/file_buf_retrieve -and file_buf_free. -additionally, the buffer may be stored in file_cache if file_cache_add -was called; it remains there until evicted in favor of another buffer. - -rationale: users are strongly encouraged to access buffers as follows: -"alloc, use, free; alloc next..". this means only a few (typically one) are -active at a time. a list of these is more efficient to go through (O(1)) -than having to scan file_cache for the buffer (O(N)). - -see also discussion at declaration of FileIOBuf. -*/ -class ExtantBufMgr -{ -public: - ExtantBufMgr() - : extant_bufs(), epoch(1) {} - - // return index of ExtantBuf that contains , or -1. - ssize_t find(FileIOBuf buf) const - { - debug_assert(buf != 0); - for(size_t i = 0; i < extant_bufs.size(); i++) - { - const ExtantBuf& eb = extant_bufs[i]; - if(matches(eb, buf)) - return (ssize_t)i; - } - - return -1; // not found - } - - // add given buffer to extant list. - // long_lived indicates if this buffer will not be freed immediately - // (more precisely: before allocating the next buffer); see FB_LONG_LIVED. - // note: reuses a previous extant_bufs[] slot if one is unused. - void add(FileIOBuf buf, size_t size, const char* atom_fn, uint fb_flags) - { - // cache_allocator also does this; we need to follow suit so that - // matches() won't fail due to zero-length size. - if(!size) - size = 1; - - // don't do was-immediately-freed check for long_lived buffers. - const bool long_lived = (fb_flags & FB_LONG_LIVED) != 0; - const uint this_epoch = long_lived? 0 : epoch++; - - debug_assert(buf != 0); - // look for holes in array and reuse those - for(size_t i = 0; i < extant_bufs.size(); i++) - { - ExtantBuf& eb = extant_bufs[i]; - if(eb.atom_fn == atom_fn) - debug_warn("already exists!"); - // slot currently empty - if(!eb.buf) - { - debug_assert(eb.refs == 0); - eb.refs = 1; - eb.buf = buf; - eb.size = size; - eb.fb_flags = fb_flags; - eb.atom_fn = atom_fn; - eb.epoch = this_epoch; - return; - } - } - // add another entry - extant_bufs.push_back(ExtantBuf(buf, size, fb_flags, atom_fn, this_epoch)); - } - - // indicate that a reference has been taken for ; - // parameters are the same as for add(). - void add_ref(FileIOBuf buf, size_t size, const char* atom_fn, bool long_lived) - { - ssize_t idx = find(buf); - // this buf was already on the extant list - if(idx != -1) - extant_bufs[idx].refs++; - // it was in cache and someone is 'reactivating' it, i.e. moving it - // to the extant list. - else - add(buf, size, atom_fn, long_lived); - } - - // return atom_fn that was passed when add()-ing this buf, or 0 if - // it's not on extant list. - const char* get_owner_filename(FileIOBuf buf) - { - ssize_t idx = find(buf); - if(idx != -1) - return extant_bufs[idx].atom_fn; - else - return 0; - } - - // return false and warn if buf is not on extant list; otherwise, - // pass back its size/owner filename and decrement reference count. - // the return value indicates whether it reached 0, i.e. was - // actually removed from the extant list. - bool find_and_remove(FileIOBuf buf, size_t& size, const char*& atom_fn) - { - ssize_t idx = find(buf); - if(idx == -1) - { - debug_warn("buf is not on extant list! double free?"); - return false; - } - - ExtantBuf& eb = extant_bufs[idx]; - size = eb.size; - atom_fn = eb.atom_fn; - - if(eb.epoch != 0 && eb.epoch != epoch-1) - debug_warn("buf not released immediately"); - epoch++; - - bool actually_removed = false; - // no more references - if(--eb.refs == 0) - { - // mark slot in extant_bufs[] as reusable - memset(&eb, 0, sizeof(eb)); - - actually_removed = true; - } - - return actually_removed; - } - - // wipe out the entire list without freeing any FileIOBuf. - // only meant to be used in file_cache_reset: since the allocator - // is completely reset, there's no need to free outstanding items first. - void clear() - { - extant_bufs.clear(); - } - - // if buf is not in extant list, complain; otherwise, mark it as - // coming from the file . - // this is needed in the following case: uncompressed reads from archive - // boil down to a file_io of the archive file. the buffer is therefore - // tagged with the archive filename instead of the desired filename. - // afile_read sets things right by calling this. - void replace_owner(FileIOBuf buf, const char* atom_fn) - { - ssize_t idx = find(buf); - if(idx != -1) - extant_bufs[idx].atom_fn = atom_fn; - else - debug_warn("to-be-replaced buf not found"); - } - - // display list of all extant buffers in debug outut. - // meant to be called at exit, at which time any remaining buffers - // must apparently have been leaked. - void display_all_remaining() - { - debug_printf("Leaked FileIOBufs:\n"); - for(size_t i = 0; i < extant_bufs.size(); i++) - { - ExtantBuf& eb = extant_bufs[i]; - if(eb.buf) - debug_printf(" %p (0x%08x) %s\n", eb.buf, eb.size, eb.atom_fn); - } - debug_printf("--------\n"); - } - -private: - struct ExtantBuf - { - // treat as user-visible padded buffer, although it may already be - // the correct exact_buf. - // rationale: file_cache_retrieve gets padded_buf from file_cache - // and then calls add_ref. if not already in extant list, that - // would be added, whereas file_buf_alloc's add() would specify - // the exact_buf. assuming it's padded_buf is safe because - // exact_buf_oracle can be used to get exact_buf from that. - FileIOBuf buf; - - // treat as user-visible size, although it may already be the - // correct exact_size. - // rationale: this would also be available via TFile, but we want - // users to be able to allocate file buffers (and they don't know tf). - // therefore, we store this separately. - size_t size; - - // FileBufFlags - uint fb_flags; - - // which file was this buffer taken from? - // we search for given atom_fn as part of file_cache_retrieve - // (since we are responsible for already extant bufs). - // also useful for tracking down buf 'leaks' (i.e. someone - // forgetting to call file_buf_free). - const char* atom_fn; - - // active references, i.e. how many times file_buf_free must be - // called until this buffer is freed and removed from extant list. - uint refs; - - // used to check if this buffer was freed immediately - // (before allocating the next). that is the desired behavior - // because it avoids fragmentation and leaks. - uint epoch; - - ExtantBuf(FileIOBuf buf_, size_t size_, uint fb_flags_, const char* atom_fn_, uint epoch_) - : buf(buf_), size(size_), fb_flags(fb_flags_), atom_fn(atom_fn_), refs(1), epoch(epoch_) {} - }; - - std::vector extant_bufs; - - // see if buf (which may be padded) falls within eb's buffer. - // this is necessary for file_buf_free; we do not know the size - // of buffer to free until after find_and_remove, so exact_buf_oracle - // cannot be used. - bool matches(const ExtantBuf& eb, FileIOBuf buf) const - { - return (eb.buf <= buf && buf < eb.buf+eb.size); - } - - uint epoch; -}; // ExtantBufMgr -static ExtantBufMgr extant_bufs; - -//----------------------------------------------------------------------------- - -// HACK: key type is really const char*, but the file_cache's STL (hash_)map -// stupidly assumes that is a "string". (comparison can be done via -// pointer compare, due to atom_fn mechanism) we define as void* to avoid -// this behavior - it breaks the (const char*)1 self-test hack and is -// inefficient. -static Cache file_cache; - -/* -mapping of padded_buf to the original exact_buf and exact_size. - -rationale: cache stores the user-visible (padded) buffer, but we need -to pass the original to cache_allocator. -since not all buffers end up padded (only happens if reading -uncompressed files from archive), it is more efficient to only -store bookkeeping information for those who need it (rather than -maintaining a complete list of allocs in cache_allocator). - -storing both padded and exact buf/size in a FileIOBuf struct is not really -an option: that begs the question how users initialize it, and can't -well be stored in Cache. -*/ -class ExactBufOracle -{ -public: - typedef std::pair BufAndSize; - - // associate padded_buf with exact_buf and exact_size; - // these can later be retrieved via get(). - // should only be called if necessary, i.e. they are not equal. - // assumes and verifies that the association didn't already exist - // (otherwise it's a bug, because it's removed when buf is freed) - void add(FileIOBuf exact_buf, size_t exact_size, FileIOBuf padded_buf) - { - debug_assert((uintptr_t)exact_buf % BUF_ALIGN == 0); - debug_assert(exact_buf <= padded_buf); - - std::pair ret; - const BufAndSize item = std::make_pair(exact_buf, exact_size); - ret = padded2exact.insert(std::make_pair(padded_buf, item)); - // make sure it wasn't already in the map - debug_assert(ret.second == true); - } - - // return exact_buf and exact_size that were associated with . - // can optionally remove that association afterwards (slightly more - // efficient than a separate remove() call). - BufAndSize get(FileIOBuf padded_buf, size_t size, bool remove_afterwards = false) - { - Padded2Exact::iterator it = padded2exact.find(padded_buf); - - BufAndSize ret; - // not found => must already be exact_buf. will be verified below. - if(it == padded2exact.end()) - ret = std::make_pair(padded_buf, size); - else - { - ret = it->second; - - // something must be different, else it shouldn't have been - // added anyway. - // actually, no: file_io may have had to register these values - // (since its user_size != size), but they may match what - // caller passed us. - //debug_assert(ret.first != padded_buf || ret.second != size); - - if(remove_afterwards) - padded2exact.erase(it); - } - - // exact_buf must be aligned, or something is wrong. - debug_assert((uintptr_t)ret.first % BUF_ALIGN == 0); - return ret; - } - - // remove all associations. this is intended only for use in - // file_cache_reset. - void clear() - { - padded2exact.clear(); - } - -private: - typedef std::map Padded2Exact; - Padded2Exact padded2exact; -}; -static ExactBufOracle exact_buf_oracle; - -// referenced by cache_alloc -static void free_padded_buf(FileIOBuf padded_buf, size_t size, bool from_heap = false); - -static void cache_free(FileIOBuf exact_buf, size_t exact_size) -{ - cache_allocator.dealloc((void*)exact_buf, exact_size); -} - -static FileIOBuf cache_alloc(size_t size) -{ - uint attempts = 0; - for(;;) - { - FileIOBuf buf = (FileIOBuf)cache_allocator.alloc(size); - if(buf) - return buf; - - // remove least valuable entry from cache and free its buffer. - FileIOBuf discarded_buf; size_t size; - bool removed = file_cache.remove_least_valuable(&discarded_buf, &size); - // only false if cache is empty, which can't be the case because - // allocation failed. - debug_assert(removed); - - // discarded_buf may be the least valuable entry in cache, but if - // still in use (i.e. extant), it must not actually be freed yet! - if(extant_bufs.find(discarded_buf) == -1) - { - free_padded_buf(discarded_buf, size); - - // optional: this iteration doesn't really count because no - // memory was actually freed. helps prevent infinite loop - // warning without having to raise the limit really high. - attempts--; - } - - // note: this may seem hefty, but 300 is known to be reached. - // (after building archive, file cache is full; attempting to - // allocate ~4MB while only freeing small blocks scattered over - // the entire cache can take a while) - if(++attempts > 500) - debug_warn("possible infinite loop: failed to make room in cache"); - } - - UNREACHABLE; -} - - -// translate to the exact buffer and free it. -// convenience function used by file_buf_alloc and file_buf_free. -static void free_padded_buf(FileIOBuf padded_buf, size_t size, bool from_heap) -{ - const bool remove_afterwards = true; - ExactBufOracle::BufAndSize exact = exact_buf_oracle.get(padded_buf, size, remove_afterwards); - FileIOBuf exact_buf = exact.first; size_t exact_size = exact.second; - - if(from_heap) - page_aligned_free((void*)exact_buf, exact_size); - else - cache_free(exact_buf, exact_size); -} - - -// allocate a new buffer of bytes (possibly more due to internal -// fragmentation). never returns 0. -// : owner filename (buffer is intended to be used for data from -// this file). -// : see FileBufFlags. -FileIOBuf file_buf_alloc(size_t size, const char* atom_fn, uint fb_flags) -{ - const bool should_update_stats = (fb_flags & FB_NO_STATS) == 0; - const bool from_heap = (fb_flags & FB_FROM_HEAP) != 0; - - FileIOBuf buf; - if(from_heap) - { - buf = (FileIOBuf)page_aligned_alloc(size); - if(!buf) - WARN_ERR(ERR::NO_MEM); - } - else - buf = cache_alloc(size); - - extant_bufs.add(buf, size, atom_fn, fb_flags); - - if(should_update_stats) - stats_buf_alloc(size, round_up(size, BUF_ALIGN)); - return buf; -} - - -// mark as no longer needed. if its reference count drops to 0, -// it will be removed from the extant list. if it had been added to the -// cache, it remains there until evicted in favor of another buffer. -LibError file_buf_free(FileIOBuf buf, uint fb_flags) -{ - const bool should_update_stats = (fb_flags & FB_NO_STATS) == 0; - const bool from_heap = (fb_flags & FB_FROM_HEAP) != 0; - - if(!buf) - return INFO::OK; - - size_t size; const char* atom_fn; - bool actually_removed = extant_bufs.find_and_remove(buf, size, atom_fn); - if(actually_removed) - { - // avoid any potential confusion and some overhead by skipping the - // retrieve step (not needed anyway). - if(from_heap) - goto free_immediately; - - { - FileIOBuf buf_in_cache; - // it's still in cache - leave its buffer intact. - if(file_cache.retrieve(atom_fn, buf_in_cache, 0, false)) - { - // sanity checks: what's in cache must match what we have. - // note: don't compare actual_size with cached size - they are - // usually different. - debug_assert(buf_in_cache == buf); - } - // buf is not in cache - needs to be freed immediately. - else - { -free_immediately: - // note: extant_bufs cannot be relied upon to store and return - // exact_buf - see definition of ExtantBuf.buf. - // we have to use exact_buf_oracle, which is a bit slow, but hey. - free_padded_buf(buf, size, from_heap); - } - } - } - - if(should_update_stats) - stats_buf_free(); - trace_notify_free(atom_fn, size); - - return INFO::OK; -} - - -// inform us that the buffer address will be increased by -bytes. -// this happens when reading uncompressed files from archive: they -// start at unaligned offsets and file_io rounds offset down to -// next block boundary. the buffer therefore starts with padding, which -// is skipped so the user only sees their data. -// we make note of the new buffer address so that it can be freed correctly -// by passing the new padded buffer. -void file_buf_add_padding(FileIOBuf exact_buf, size_t exact_size, size_t padding) -{ - debug_assert(padding < FILE_BLOCK_SIZE); - FileIOBuf padded_buf = exact_buf + padding; - exact_buf_oracle.add(exact_buf, exact_size, padded_buf); -} - - -// if buf is not in extant list, complain; otherwise, mark it as -// coming from the file . -// this is needed in the following case: uncompressed reads from archive -// boil down to a file_io of the archive file. the buffer is therefore -// tagged with the archive filename instead of the desired filename. -// afile_read sets things right by calling this. -LibError file_buf_set_real_fn(FileIOBuf buf, const char* atom_fn) -{ - // note: removing and reinserting would be easiest, but would - // mess up the epoch field. - extant_bufs.replace_owner(buf, atom_fn); - return INFO::OK; -} - - -// if file_cache_add-ing the given buffer, would it be added? -// this is referenced by trace_entry_causes_io; see explanation there. -bool file_cache_would_add(size_t size, const char* UNUSED(atom_fn), - uint file_flags) -{ - // caller is saying this file shouldn't be cached here. - if(file_flags & FILE_CACHED_AT_HIGHER_LEVEL) - return false; - - // refuse to cache 0-length files (it would have no benefit and - // causes problems due to divide-by-0). - if(size == 0) - return false; - - return true; -} - - -// "give" to the cache, specifying its size and owner filename. -// since this data may be shared among users of the cache, it is made -// read-only (via MMU) to make sure no one can corrupt/change it. -// -// note: the reference added by file_buf_alloc still exists! it must -// still be file_buf_free-d after calling this. -LibError file_cache_add(FileIOBuf buf, size_t size, const char* atom_fn, - uint file_flags) -{ - debug_assert(buf); - - if(!file_cache_would_add(size, atom_fn, file_flags)) - return INFO::SKIPPED; - - // assign cost - uint cost = 1; - - ExactBufOracle::BufAndSize bas = exact_buf_oracle.get(buf, size); - FileIOBuf exact_buf = bas.first; size_t exact_size = bas.second; - cache_allocator.make_read_only((void*)exact_buf, exact_size); - - file_cache.add(atom_fn, buf, size, cost); - - return INFO::OK; -} - - - - -// check if the contents of the file are in file cache. -// if not, return 0; otherwise, return buffer address and optionally -// pass back its size. -// -// note: does not call stats_cache because it does not know the file size -// in case of cache miss! doing so is left to the caller. -FileIOBuf file_cache_retrieve(const char* atom_fn, size_t* psize, uint fb_flags) -{ - // note: do not query extant_bufs - reusing that doesn't make sense - // (why would someone issue a second IO for the entire file while - // still referencing the previous instance?) - - const bool long_lived = (fb_flags & FB_LONG_LIVED) != 0; - const bool should_account = (fb_flags & FB_NO_ACCOUNTING) == 0; - const bool should_update_stats = (fb_flags & FB_NO_STATS) == 0; - - FileIOBuf buf; - const bool should_refill_credit = should_account; - if(!file_cache.retrieve(atom_fn, buf, psize, should_refill_credit)) - return 0; - - if(should_account) - extant_bufs.add_ref(buf, *psize, atom_fn, long_lived); - - if(should_update_stats) - stats_buf_ref(); - - return buf; -} - - -// invalidate all data loaded from the file . this ensures the next -// load of this file gets the (presumably new) contents of the file, -// not previous stale cache contents. -// call after hotloading code detects file has been changed. -LibError file_cache_invalidate(const char* P_fn) -{ - const char* atom_fn = file_make_unique_fn_copy(P_fn); - - // note: what if the file has an extant buffer? - // this *could* conceivably happen during hotloading if a file is - // saved right when the engine wants to access it (unlikely but not - // impossible). - // what we'll do is just let them continue as if nothing had happened; - // invalidating is only meant to make sure that the reload's IO - // will load the new data (not stale stuff from cache). - // => nothing needs to be done. - - // mark all blocks from the file as invalid - block_mgr.invalidate(atom_fn); - - // file was cached: remove it and free that memory - FileIOBuf cached_buf; size_t size; - if(file_cache.retrieve(atom_fn, cached_buf, &size)) - { - file_cache.remove(atom_fn); - free_padded_buf(cached_buf, size); - } - - return INFO::OK; -} - - -// reset entire state of the file cache to what it was after initialization. -// that means completely emptying the extant list and cache. -// used after simulating cache operation, which fills the cache with -// invalid data. -void file_cache_reset() -{ - // just wipe out extant list and cache without freeing the bufs - - // cache allocator is completely reset below. - - extant_bufs.clear(); - - // note: do not loop until file_cache.empty - there may still be - // some items pending eviction even though cache is "empty". - FileIOBuf discarded_buf; size_t size; - while(file_cache.remove_least_valuable(&discarded_buf, &size)) - { - } - - cache_allocator.reset(); - exact_buf_oracle.clear(); -} - - - -void file_cache_init() -{ -} - - -void file_cache_shutdown() -{ - extant_bufs.display_all_remaining(); - cache_allocator.shutdown(); - block_mgr.shutdown(); -} - - -// for self test - -void* file_cache_allocator_alloc(size_t size) -{ - return cache_allocator.alloc(size); -} -void file_cache_allocator_free(void* p, size_t size) -{ - return cache_allocator.dealloc(p, size); -} -void file_cache_allocator_reset() -{ - cache_allocator.reset(); -} diff --git a/source/lib/res/file/file_cache.h b/source/lib/res/file/file_cache.h deleted file mode 100644 index b4139d1769..0000000000 --- a/source/lib/res/file/file_cache.h +++ /dev/null @@ -1,107 +0,0 @@ -/** - * ========================================================================= - * File : file_cache.h - * Project : 0 A.D. - * Description : cache for entire files and I/O blocks. also allocates - * : file buffers, allowing zero-copy I/O. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_FILE_CACHE -#define INCLUDED_FILE_CACHE - -#include "file.h" // FileIOBuf - -struct BlockId -{ - const char* atom_fn; - u32 block_num; -}; - -extern bool block_eq(BlockId b1, BlockId b2); - -// create an id for use with the cache that uniquely identifies -// the block from the file starting at . -extern BlockId block_cache_make_id(const char* atom_fn, const off_t ofs); - -extern void* block_cache_alloc(BlockId id); - -extern void block_cache_mark_completed(BlockId id); - -extern void* block_cache_find(BlockId id); -extern void block_cache_release(BlockId id); - - - - -// interpret file_io parameters (pbuf, size, flags, cb) and allocate a -// file buffer if necessary. -// called by file_io and afile_read. -extern LibError file_io_get_buf(FileIOBuf* pbuf, size_t size, - const char* atom_fn, uint flags, FileIOCB cb); - -// inform us that the buffer address will be increased by -bytes. -// this happens when reading uncompressed files from archive: they -// start at unaligned offsets and file_io rounds offset down to -// next block boundary. the buffer therefore starts with padding, which -// is skipped so the user only sees their data. -// we make note of the new buffer address so that it can be freed correctly -// by passing the new padded buffer. -extern void file_buf_add_padding(FileIOBuf exact_buf, size_t exact_size, size_t padding); - -// if buf is not in extant list, complain; otherwise, mark it as -// coming from the file . -// this is needed in the following case: uncompressed reads from archive -// boil down to a file_io of the archive file. the buffer is therefore -// tagged with the archive filename instead of the desired filename. -// afile_read sets things right by calling this. -extern LibError file_buf_set_real_fn(FileIOBuf buf, const char* atom_fn); - -// if file_cache_add-ing the given buffer, would it be added? -// this is referenced by trace_entry_causes_io; see explanation there. -extern bool file_cache_would_add(size_t size, const char* atom_fn, - uint file_flags); - -// "give" to the cache, specifying its size and owner filename. -// since this data may be shared among users of the cache, it is made -// read-only (via MMU) to make sure no one can corrupt/change it. -// -// note: the reference added by file_buf_alloc still exists! it must -// still be file_buf_free-d after calling this. -extern LibError file_cache_add(FileIOBuf buf, size_t size, - const char* atom_fn, uint file_flags); - - - -// check if the contents of the file are in file cache. -// if not, return 0; otherwise, return buffer address and optionally -// pass back its size. -// -// note: does not call stats_cache because it does not know the file size -// in case of cache miss! doing so is left to the caller. -extern FileIOBuf file_cache_retrieve(const char* atom_fn, size_t* psize, uint fb_flags = 0); - -// invalidate all data loaded from the file . this ensures the next -// load of this file gets the (presumably new) contents of the file, -// not previous stale cache contents. -// call after hotloading code detects file has been changed. -extern LibError file_cache_invalidate(const char* P_fn); - -// reset entire state of the file cache to what it was after initialization. -// that means completely emptying the extant list and cache. -// used after simulating cache operation, which fills the cache with -// invalid data. -extern void file_cache_reset(); - -extern void file_cache_init(); -extern void file_cache_shutdown(); - - -// test access mechanism -extern void* file_cache_allocator_alloc(size_t size); -extern void file_cache_allocator_free(void* p, size_t size); -extern void file_cache_allocator_reset(); - -#endif // #ifndef INCLUDED_FILE_CACHE diff --git a/source/lib/res/file/file_internal.h b/source/lib/res/file/file_internal.h deleted file mode 100644 index c756702756..0000000000 --- a/source/lib/res/file/file_internal.h +++ /dev/null @@ -1,75 +0,0 @@ -/** - * ========================================================================= - * File : file_internal.h - * Project : 0 A.D. - * Description : master (private) header for all file code. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "lib/path_util.h" -#include "path.h" - -#include "file.h" -#include "file_cache.h" -#include "file_io.h" - -#include "file_stats.h" // must come after file and file_cache - -#include "archive/compression.h" -#include "archive/zip.h" -#include "archive/archive.h" -#include "archive/archive_builder.h" -#include "archive/trace.h" -#include "archive/vfs_optimizer.h" - -#include "vfs.h" -#include "vfs_mount.h" -#include "vfs_tree.h" -#include "vfs_redirector.h" - -const size_t AIO_SECTOR_SIZE = 512; - -// block := power-of-two sized chunk of a file. -// all transfers are expanded to naturally aligned, whole blocks -// (this makes caching parts of files feasible; it is also much faster -// for some aio implementations, e.g. wposix). -// -// this is not exposed to users because it's an implementation detail and -// they shouldn't care. -// -// measurements show this value to yield best read throughput. -const size_t FILE_BLOCK_SIZE = 32*KiB; - -// helper routine used by functions that call back to a FileIOCB. -// -// bytes_processed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE } -// note: don't abort if = 0: zip callback may not actually -// output anything if passed very little data. -extern LibError file_io_call_back(const u8* block, size_t size, - FileIOCB cb, uintptr_t cbData, size_t& bytes_processed); - - -// retrieve the next (order is unspecified) dir entry matching . -// return 0 on success, ERR::DIR_END if no matching entry was found, -// or a negative error code on failure. -// filter values: -// - 0: anything; -// - "/": any subdirectory; -// - "/|": any subdirectory, or as below with ; -// - : any file whose name matches; ? and * wildcards are allowed. -// -// note that the directory entries are only scanned once; after the -// end is reached (-> ERR::DIR_END returned), no further entries can -// be retrieved, even if filter changes (which shouldn't happen - see impl). -// -// rationale: we do not sort directory entries alphabetically here. -// most callers don't need it and the overhead is considerable -// (we'd have to store all entries in a vector). it is left up to -// higher-level code such as VfsUtil. -extern LibError dir_filtered_next_ent(DirIterator* di, DirEnt* ent, const char* filter); - -// returns file descriptor (int) given File (assumed to represent PosixFile). -// this avoids the need for declaring PosixFile here for file_io's use. -extern int file_fd_from_PosixFile(File* f); diff --git a/source/lib/res/file/file_io.cpp b/source/lib/res/file/file_io.cpp deleted file mode 100644 index 4e7751dc71..0000000000 --- a/source/lib/res/file/file_io.cpp +++ /dev/null @@ -1,656 +0,0 @@ -/** - * ========================================================================= - * File : file_io.cpp - * Project : 0 A.D. - * Description : provide fast I/O via POSIX aio and splitting into blocks. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "file_io.h" - -#include -#include - -#include "lib/posix/posix_aio.h" -#include "lib/bits.h" -#include "lib/allocators.h" -#include "lib/adts.h" -#include "file_internal.h" - - -ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO); -ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1); - - -//----------------------------------------------------------------------------- -// async I/O -//----------------------------------------------------------------------------- - -struct PosixFileIo -{ - void* cb; // aiocb -}; -cassert(sizeof(PosixFileIo) <= FILE_IO_OPAQUE_SIZE); - -// we don't do any caching or alignment here - this is just a thin AIO wrapper. -// rationale: -// - aligning the transfer isn't possible here since we have no control -// over the buffer, i.e. we cannot read more data than requested. -// instead, this is done in file_io. -// - transfer sizes here are arbitrary (viz. not block-aligned); -// that means the cache would have to handle this or also split them up -// into blocks, which is redundant (already done by file_io). -// - if caching here, we'd also have to handle "forwarding" (i.e. -// desired block has been issued but isn't yet complete). again, it -// is easier to let the synchronous file_io manager handle this. -// - finally, file_io knows more about whether the block should be cached -// (e.g. whether another block request will follow), but we don't -// currently make use of this. -// -// disadvantages: -// - streamed data will always be read from disk. no problem, because -// such data (e.g. music, long speech) is unlikely to be used again soon. -// - prefetching (issuing the next few blocks from archive/file during -// idle time to satisfy potential future IOs) requires extra buffers; -// this is a bit more complicated than just using the cache as storage. - -// FileIO must reference an aiocb, which is used to pass IO params to the OS. -// unfortunately it is 144 bytes on Linux - too much to put in FileIO, -// since that is stored in a 'resource control block' (see h_mgr.h). -// we therefore allocate dynamically, but via suballocator to avoid -// hitting the heap on every IO. -class AiocbAllocator -{ - Pool pool; -public: - void init() - { - (void)pool_create(&pool, 32*sizeof(aiocb), sizeof(aiocb)); - } - void shutdown() - { - (void)pool_destroy(&pool); - } - aiocb* alloc() - { - return (aiocb*)pool_alloc(&pool, 0); - } - // weird name to avoid trouble with mem tracker macros - // (renaming is less annoying than #include "lib/nommgr.h") - void free_(void* cb) - { - pool_free(&pool, cb); - } -}; -static AiocbAllocator aiocb_allocator; - - -// starts transferring to/from the given buffer. -// no attempt is made at aligning or padding the transfer. -LibError file_io_issue(File* f, off_t ofs, size_t size, u8* p, FileIo* io) -{ - debug_printf("FILE| issue ofs=0x%X size=0x%X\n", ofs, size); - - // zero output param in case we fail below. - memset(io, 0, sizeof(FileIo)); - - // check params - CHECK_FILE(f); - if(!size || !p || !io) - WARN_RETURN(ERR::INVALID_PARAM); - const bool is_write = (f->flags & FILE_WRITE) != 0; - - PosixFileIo* pio = (PosixFileIo*)io; - - // note: cutting off at EOF is necessary to avoid transfer errors, - // but makes size no longer sector-aligned, which would force - // waio to realign (slow). we want to pad back to sector boundaries - // afterwards (to avoid realignment), but that is not possible here - // since we have no control over the buffer (there might not be - // enough room in it). hence, do cut-off in IOManager. - // - // example: 200-byte file. IOManager issues (large) blocks; - // that ends up way beyond EOF, so ReadFile fails. - // limiting size to 200 bytes works, but causes waio to pad the - // transfer and use align buffer (slow). - // rounding up to 512 bytes avoids realignment and does not fail - // (apparently since NTFS files are sector-padded anyway?) - - // (we can't store the whole aiocb directly - glibc's version is - // 144 bytes large) - aiocb* cb = aiocb_allocator.alloc(); - pio->cb = cb; - if(!cb) - WARN_RETURN(ERR::NO_MEM); - memset(cb, 0, sizeof(*cb)); - - // send off async read/write request - cb->aio_lio_opcode = is_write? LIO_WRITE : LIO_READ; - cb->aio_buf = (volatile void*)p; - cb->aio_fildes = file_fd_from_PosixFile(f); - cb->aio_offset = ofs; - cb->aio_nbytes = size; - int err = lio_listio(LIO_NOWAIT, &cb, 1, (struct sigevent*)0); - if(err < 0) - { - debug_printf("lio_listio: %d, %d[%s]\n", err, errno, strerror(errno)); - (void)file_io_discard(io); - return LibError_from_errno(); - } - - const BlockId disk_pos = block_cache_make_id(f->atom_fn, ofs); - stats_io_check_seek(disk_pos.atom_fn, disk_pos.block_num); - - return INFO::OK; -} - - -// indicates if the IO referenced by has completed. -// return value: 0 if pending, 1 if complete, < 0 on error. -int file_io_has_completed(FileIo* io) -{ - PosixFileIo* pio = (PosixFileIo*)io; - aiocb* cb = (aiocb*)pio->cb; - int ret = aio_error(cb); - if(ret == EINPROGRESS) - return 0; - if(ret == 0) - return 1; - - WARN_RETURN(ERR::FAIL); -} - - -LibError file_io_wait(FileIo* io, u8*& p, size_t& size) -{ - PosixFileIo* pio = (PosixFileIo*)io; -// debug_printf("FILE| wait io=%p\n", io); - - // zero output params in case something (e.g. H_DEREF) fails. - p = 0; - size = 0; - - aiocb* cb = (aiocb*)pio->cb; - - // wait for transfer to complete. - const aiocb** cbs = (const aiocb**)&cb; // pass in an "array" - while(aio_error(cb) == EINPROGRESS) - aio_suspend(cbs, 1, (timespec*)0); // wait indefinitely - - // query number of bytes transferred (-1 if the transfer failed) - const ssize_t bytes_transferred = aio_return(cb); -// debug_printf("FILE| bytes_transferred=%d aio_nbytes=%u\n", bytes_transferred, cb->aio_nbytes); - - // see if actual transfer count matches requested size. - // note: most callers clamp to EOF but round back up to sector size - // (see explanation in file_io_issue). - debug_assert(bytes_transferred >= (ssize_t)(cb->aio_nbytes-AIO_SECTOR_SIZE)); - - p = (u8*)cb->aio_buf; // cast from volatile void* - size = bytes_transferred; - return INFO::OK; -} - - -LibError file_io_discard(FileIo* io) -{ - PosixFileIo* pio = (PosixFileIo*)io; - memset(pio->cb, 0, sizeof(aiocb)); // prevent further use. - aiocb_allocator.free_(pio->cb); - pio->cb = 0; - return INFO::OK; -} - - -LibError file_io_validate(const FileIo* io) -{ - PosixFileIo* pio = (PosixFileIo*)io; - const aiocb* cb = (const aiocb*)pio->cb; - // >= 0x100 is not necessarily bogus, but suspicious. - // this also catches negative values. - if((uint)cb->aio_fildes >= 0x100) - WARN_RETURN(ERR::_1); - if(debug_is_pointer_bogus((void*)cb->aio_buf)) - WARN_RETURN(ERR::_2); - if(cb->aio_lio_opcode != LIO_WRITE && cb->aio_lio_opcode != LIO_READ && cb->aio_lio_opcode != LIO_NOP) - WARN_RETURN(ERR::_3); - // all other aiocb fields have no invariants we could check. - return INFO::OK; -} - - -//----------------------------------------------------------------------------- -// sync I/O -//----------------------------------------------------------------------------- - -// set from sys_max_sector_size(); see documentation there. -size_t file_sector_size; - - -// the underlying aio implementation likes buffer and offset to be -// sector-aligned; if not, the transfer goes through an align buffer, -// and requires an extra cpu_memcpy. -// -// if the user specifies an unaligned buffer, there's not much we can -// do - we can't assume the buffer contains padding. therefore, -// callers should let us allocate the buffer if possible. -// -// if ofs misalign = buffer, only the first and last blocks will need -// to be copied by aio, since we read up to the next block boundary. -// otherwise, everything will have to be copied; at least we split -// the read into blocks, so aio's buffer won't have to cover the -// whole file. - - -// helper routine used by functions that call back to a FileIOCB. -// -// bytes_processed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE } -// note: don't abort if = 0: zip callback may not actually -// output anything if passed very little data. -LibError file_io_call_back(const u8* block, size_t size, - FileIOCB cb, uintptr_t cbData, size_t& bytes_processed) -{ - if(cb) - { - stats_cb_start(); - LibError ret = cb(cbData, block, size, &bytes_processed); - stats_cb_finish(); - - // failed - reset byte count in case callback didn't - if(ret != INFO::OK && ret != INFO::CB_CONTINUE) - bytes_processed = 0; - - CHECK_ERR(ret); // user might not have raised a warning; make sure - return ret; - } - // no callback to process data: raw = actual - else - { - bytes_processed = size; - return INFO::CB_CONTINUE; - } -} - - -// interpret file_io parameters (pbuf, size, flags, cb) and allocate a -// file buffer if necessary. -// called by file_io and afile_read. -LibError file_io_get_buf(FileIOBuf* pbuf, size_t size, - const char* atom_fn, uint file_flags, FileIOCB cb) -{ - // decode *pbuf - exactly one of these is true - const bool temp = (pbuf == FILE_BUF_TEMP); - const bool alloc = !temp && (*pbuf == FILE_BUF_ALLOC); - const bool user = !temp && !alloc; - - const bool is_write = (file_flags & FILE_WRITE) != 0; - const uint fb_flags = (file_flags & FILE_LONG_LIVED)? FB_LONG_LIVED : 0; - - // reading into temp buffers - ok. - if(!is_write && temp && cb != 0) - return INFO::OK; - - // reading and want buffer allocated. - if(!is_write && alloc) - { - *pbuf = file_buf_alloc(size, atom_fn, fb_flags); - if(!*pbuf) // very unlikely (size totally bogus or cache hosed) - WARN_RETURN(ERR::NO_MEM); - return INFO::OK; - } - - // writing from user-specified buffer - ok - if(is_write && user) - return INFO::OK; - - WARN_RETURN(ERR::INVALID_PARAM); -} - - -class IOManager -{ - File* f; - bool is_write; - bool no_aio; - - FileIOCB cb; - uintptr_t cbData; - - off_t start_ofs; - FileIOBuf* pbuf; - - size_t user_size; - size_t ofs_misalign; - size_t size; - - // (useful, raw data: possibly compressed, but doesn't count padding) - size_t total_issued; - size_t total_transferred; - // if callback, sum of what it reports; otherwise, = total_transferred - // this is what we'll return. - size_t total_processed; - - - struct IOSlot - { - FileIo io; - - const void* cached_block; - - - BlockId block_id; - // needed so that we can add the block to the cache when - // its IO is complete. if we add it when issuing, we'd no longer be - // thread-safe: someone else might find it in the cache before its - // transfer has completed. don't want to add an "is_complete" flag, - // because that'd be hard to update (on every wait_io). - - - void* temp_buf; - - IOSlot() - { - reset(); - } - void reset() - { - memset(&io, 0, sizeof(io)); - cached_block = 0; - memset(&block_id, 0, sizeof(block_id)); - temp_buf = 0; - } - }; - static const uint MAX_PENDING_IOS = 4; - //RingBuf queue; - std::deque queue; - - // stop issuing and processing as soon as this changes - LibError err; - - - ssize_t lowio() - { - const int fd = file_fd_from_PosixFile(f); - lseek(fd, start_ofs, SEEK_SET); - - // emulate temp buffers - we take care of allocating and freeing. - u8* dst; - boost::shared_ptr dstMem; - if(pbuf == FILE_BUF_TEMP) - { - dstMem.reset((u8*)page_aligned_alloc(size), PageAlignedDeleter(size)); - dst = dstMem.get(); - } - else - dst = (u8*)*pbuf; // WARNING: FileIOBuf is nominally const; if that's ever enforced, this may need to change. - - const ssize_t total_transferred = is_write? write(fd, dst, size) : read(fd, dst, size); - if(total_transferred < 0) - return LibError_from_errno(); - - size_t total_processed; - LibError ret = file_io_call_back(dst, total_transferred, cb, cbData, total_processed); - RETURN_ERR(ret); - return (ssize_t)total_processed; - } - - - // align and pad the IO to FILE_BLOCK_SIZE - // (reduces work for AIO implementation). - LibError prepare() - { - ofs_misalign = 0; - size = user_size; - - if(!is_write && !no_aio) - { - // note: we go to the trouble of aligning the first block (instead of - // just reading up to the next block and letting aio realign it), - // so that it can be taken from the cache. - // this is not possible if we don't allocate the buffer because - // extra space must be added for the padding. - - ofs_misalign = start_ofs % FILE_BLOCK_SIZE; - start_ofs -= (off_t)ofs_misalign; - size = round_up(ofs_misalign + user_size, FILE_BLOCK_SIZE); - - // but cut off at EOF (necessary to prevent IO error). - const off_t bytes_left = f->size - start_ofs; - if(bytes_left < 0) - WARN_RETURN(ERR::IO_EOF); - size = std::min(size, (size_t)bytes_left); - - // and round back up to sector size. - // see rationale in file_io_issue. - size = round_up(size, AIO_SECTOR_SIZE); - } - - RETURN_ERR(file_io_get_buf(pbuf, size, f->atom_fn, f->flags, cb)); - - return INFO::OK; - } - - void issue(IOSlot& slot) - { - const off_t ofs = start_ofs+(off_t)total_issued; - // for both reads and writes, do not issue beyond end of file/data - const size_t issue_size = std::min(FILE_BLOCK_SIZE, size - total_issued); -// try to grab whole blocks (so we can put them in the cache). -// any excess data (can only be within first or last) is -// discarded in wait(). - - // check if in cache - slot.block_id = block_cache_make_id(f->atom_fn, ofs); - slot.cached_block = block_cache_find(slot.block_id); - if(!slot.cached_block) - { - void* buf; - - // if using buffer, set position in it; otherwise, use temp buffer - if(pbuf == FILE_BUF_TEMP) - buf = slot.temp_buf = block_cache_alloc(slot.block_id); - else - buf = (char*)*pbuf + total_issued; - - LibError ret = file_io_issue(f, ofs, issue_size, (u8*)buf, &slot.io); - // transfer failed - loop will now terminate after - // waiting for all pending transfers to complete. - if(ret != INFO::OK) - err = ret; - } - - total_issued += issue_size; - } - - void wait(IOSlot& slot, u8*& block, size_t& block_size) - { - // get completed block address/size - if(slot.cached_block) - { - block = (u8*)slot.cached_block; - block_size = FILE_BLOCK_SIZE; - } - // .. wasn't in cache; it was issued, so wait for it - else - { - LibError ret = file_io_wait(&slot.io, block, block_size); - if(ret < 0) - err = ret; - } - - // special forwarding path: copy into block cache from - // user's buffer. this necessary to efficiently support direct - // IO of uncompressed files in archives. - // note: must occur before skipping padding below. - if(!slot.cached_block && pbuf != FILE_BUF_TEMP && f->flags & FILE_CACHE_BLOCK) - { - slot.temp_buf = block_cache_alloc(slot.block_id); - cpu_memcpy(slot.temp_buf, block, block_size); - // block_cache_mark_completed will be called in process() - } - - // first time; skip past padding - if(total_transferred == 0) - { - block = (u8*)block + ofs_misalign; - block_size -= ofs_misalign; - } - - // last time: don't include trailing padding - if(total_transferred + block_size > user_size) - block_size = user_size - total_transferred; - - // we have useable data from a previous temp buffer, - // but it needs to be copied into the user's buffer - if(slot.cached_block && pbuf != FILE_BUF_TEMP) - cpu_memcpy((char*)*pbuf+ofs_misalign+total_transferred, block, block_size); - - total_transferred += block_size; - } - - void process(IOSlot& slot, u8* block, size_t block_size, FileIOCB cb, uintptr_t cbData) - { - if(err == INFO::CB_CONTINUE) - { - size_t bytes_processed; - err = file_io_call_back(block, block_size, cb, cbData, bytes_processed); - if(err == INFO::CB_CONTINUE || err == INFO::OK) - total_processed += bytes_processed; - // else: processing failed. - // loop will now terminate after waiting for all - // pending transfers to complete. - } - - if(slot.cached_block) - block_cache_release(slot.block_id); - else - { - file_io_discard(&slot.io); - - if(slot.temp_buf) - block_cache_mark_completed(slot.block_id); - } - } - - - ssize_t aio() - { -again: - { - // data remaining to transfer, and no error: - // start transferring next block. - if(total_issued < size && err == INFO::CB_CONTINUE && queue.size() < MAX_PENDING_IOS) - { - queue.push_back(IOSlot()); - IOSlot& slot = queue.back(); - issue(slot); - goto again; - } - - // IO pending: wait for it to complete, and process it. - if(!queue.empty()) - { - IOSlot& slot = queue.front(); - u8* block; size_t block_size; - wait(slot, block, block_size); - process(slot, block, block_size, cb, cbData); - queue.pop_front(); - goto again; - } - } - // (all issued OR error) AND no pending transfers - done. - - debug_assert(total_issued >= total_transferred && total_transferred >= user_size); - return (ssize_t)total_processed; - } - -public: - IOManager(File* f_, off_t ofs_, size_t size_, FileIOBuf* pbuf_, - FileIOCB cb_, uintptr_t cbData_) - { - f = f_; - is_write = (f->flags & FILE_WRITE ) != 0; - no_aio = (f->flags & FILE_NO_AIO) != 0; - - cb = cb_; - cbData = cbData_; - - start_ofs = ofs_; - user_size = size_; - pbuf = pbuf_; - - total_issued = 0; - total_transferred = 0; - total_processed = 0; - err = INFO::CB_CONTINUE; - } - - // now we read the file in 64 KiB chunks, N-buffered. - // if reading from Zip, inflate while reading the next block. - ssize_t run() - { - RETURN_ERR(prepare()); - - const FileIOImplentation fi = no_aio? FI_LOWIO : FI_AIO; - const FileOp fo = is_write? FO_WRITE : FO_READ; - double start_time = 0.0; - stats_io_sync_start(&start_time); - ssize_t bytes_transferred = no_aio? lowio() : aio(); - stats_io_sync_finish(fi, fo, bytes_transferred, &start_time); - - // we allocated the memory: skip any leading padding - if(pbuf != FILE_BUF_TEMP && !is_write) - { - FileIOBuf org_buf = *pbuf; - *pbuf = (u8*)org_buf + ofs_misalign; - if(ofs_misalign || size != user_size) - file_buf_add_padding(org_buf, size, ofs_misalign); - } - - if(err != INFO::CB_CONTINUE && err != INFO::OK) - return (ssize_t)err; - return bytes_transferred; - } - -}; // IOManager - - -// transfer bytes, starting at , to/from the given file. -// (read or write access was chosen at file-open time). -// -// if non-NULL, is called for each block transferred, passing . -// it returns how much data was actually transferred, or a negative error -// code (in which case we abort the transfer and return that value). -// the callback mechanism is useful for user progress notification or -// processing data while waiting for the next I/O to complete -// (quasi-parallel, without the complexity of threads). -// -// return number of bytes transferred (see above), or a negative error code. -ssize_t file_io(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, - FileIOCB cb, uintptr_t cbData) // optional -{ - CHECK_FILE(f); - - // note: do not update stats/trace here: this includes Zip IOs, - // which shouldn't be reported. - - IOManager mgr(f, ofs, size, pbuf, cb, cbData); - return mgr.run(); -} - - - - -void file_io_init() -{ - aiocb_allocator.init(); -} - - -void file_io_shutdown() -{ - aiocb_allocator.shutdown(); -} diff --git a/source/lib/res/file/file_io.h b/source/lib/res/file/file_io.h deleted file mode 100644 index 8ee7fd916e..0000000000 --- a/source/lib/res/file/file_io.h +++ /dev/null @@ -1,149 +0,0 @@ -/** - * ========================================================================= - * File : file_io.h - * Project : 0 A.D. - * Description : provide fast I/O via POSIX aio and splitting into blocks. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_FILE_IO -#define INCLUDED_FILE_IO - -struct FileProvider_VTbl; -struct File; - - -namespace ERR -{ - const LibError IO = -110100; - const LibError IO_EOF = -110101; -} - - -extern void file_io_init(); -extern void file_io_shutdown(); - - -// -// asynchronous IO -// - -// this is a thin wrapper on top of the system AIO calls. -// IOs are carried out exactly as requested - there is no caching or -// alignment done here. rationale: see source. - -// again chosen for nice alignment; each user checks if big enough. -const size_t FILE_IO_OPAQUE_SIZE = 28; - -struct FileIo -{ - const FileProvider_VTbl* type; - u8 opaque[FILE_IO_OPAQUE_SIZE]; -}; - -// queue the IO; it begins after the previous ones (if any) complete. -// -// rationale: this interface is more convenient than implicitly advancing a -// file pointer because archive.cpp often accesses random offsets. -extern LibError file_io_issue(File* f, off_t ofs, size_t size, u8* buf, FileIo* io); - -// indicates if the given IO has completed. -// return value: 0 if pending, 1 if complete, < 0 on error. -extern int file_io_has_completed(FileIo* io); - -// wait for the given IO to complete. passes back its buffer and size. -extern LibError file_io_wait(FileIo* io, u8*& p, size_t& size); - -// indicates the IO's buffer is no longer needed and frees that memory. -extern LibError file_io_discard(FileIo* io); - -extern LibError file_io_validate(const FileIo* io); - - -// -// synchronous IO -// - -extern size_t file_sector_size; - -// called by file_io after a block IO has completed. -// *bytes_processed must be set; file_io will return the sum of these values. -// example: when reading compressed data and decompressing in the callback, -// indicate #bytes decompressed. -// return value: INFO::CB_CONTINUE to continue calling; anything else: -// abort immediately and return that. -// note: in situations where the entire IO is not split into blocks -// (e.g. when reading from cache or not using AIO), this is still called but -// for the entire IO. we do not split into fake blocks because it is -// advantageous (e.g. for decompressors) to have all data at once, if available -// anyway. -typedef LibError (*FileIOCB)(uintptr_t cbData, const u8* block, size_t size, size_t* bytes_processed); - - -typedef const u8* FileIOBuf; - -FileIOBuf* const FILE_BUF_TEMP = (FileIOBuf*)1; -const FileIOBuf FILE_BUF_ALLOC = (FileIOBuf)2; - - -enum FileBufFlags -{ - // indicates the buffer will not be freed immediately - // (i.e. before the next buffer alloc) as it normally should. - // this flag serves to suppress a warning and better avoid fragmentation. - // caller sets this when FILE_LONG_LIVED is specified. - // - // also used by file_cache_retrieve because it may have to - // 'reactivate' the buffer (transfer from cache to extant list), - // which requires knowing whether the buffer is long-lived or not. - FB_LONG_LIVED = 1, - - // statistics (e.g. # buffer allocs) should not be updated. - // (useful for simulation, e.g. trace_entry_causes_io) - FB_NO_STATS = 2, - - // file_cache_retrieve should not update item credit. - // (useful when just looking up buffer given atom_fn) - FB_NO_ACCOUNTING = 4, - - // memory will be allocated from the heap, not the (limited) file cache. - // this makes sense for write buffers that are never used again, - // because we avoid having to displace some other cached items. - FB_FROM_HEAP = 8 -}; - -// allocate a new buffer of bytes (possibly more due to internal -// fragmentation). never returns 0. -// : owner filename (buffer is intended to be used for data from -// this file). -extern FileIOBuf file_buf_alloc(size_t size, const char* atom_fn, uint fb_flags = 0); - -// mark as no longer needed. if its reference count drops to 0, -// it will be removed from the extant list. if it had been added to the -// cache, it remains there until evicted in favor of another buffer. -extern LibError file_buf_free(FileIOBuf buf, uint fb_flags = 0); - - -// transfer bytes, starting at , to/from the given file. -// (read or write access was chosen at file-open time). -// -// if non-NULL, is called for each block transferred, passing . -// it returns how much data was actually transferred, or a negative error -// code (in which case we abort the transfer and return that value). -// the callback mechanism is useful for user progress notification or -// processing data while waiting for the next I/O to complete -// (quasi-parallel, without the complexity of threads). -// -// return number of bytes transferred (see above), or a negative error code. -extern ssize_t file_io(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb = 0, uintptr_t cbData = 0); - -extern ssize_t file_read_from_cache(const char* atom_fn, off_t ofs, size_t size, - FileIOBuf* pbuf, FileIOCB cb, uintptr_t cbData); - - -extern LibError file_io_get_buf(FileIOBuf* pbuf, size_t size, - const char* atom_fn, uint file_flags, FileIOCB cb); - -#endif // #ifndef INCLUDED_FILE_IO diff --git a/source/lib/res/file/file_stats.cpp b/source/lib/res/file/file_stats.cpp deleted file mode 100644 index 7d167d2cc0..0000000000 --- a/source/lib/res/file/file_stats.cpp +++ /dev/null @@ -1,349 +0,0 @@ -/** - * ========================================================================= - * File : file_stats.cpp - * Project : 0 A.D. - * Description : gathers statistics from all file modules. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "file_stats.h" - -#include - -#include "lib/timer.h" -#include "file_internal.h" - - -typedef std::set AtomFnSet; -typedef std::pair PairIB; - -// vfs -static uint vfs_files; -static size_t vfs_size_total; -static double vfs_init_elapsed_time; - -// file -static uint unique_names; -static size_t unique_name_len_total; -static uint open_files_cur, open_files_max; // total = opened_files.size() -static double opened_file_size_total; -static AtomFnSet opened_files; - -// file_buf -static uint extant_bufs_cur, extant_bufs_max, extant_bufs_total; -static double buf_user_size_total, buf_padded_size_total; - -// file_io -static uint user_ios; -static double user_io_size_total; -static double io_actual_size_total[FI_MAX_IDX][2]; -static double io_elapsed_time[FI_MAX_IDX][2]; -static double io_process_time_total; -static uint io_seeks; - -// file_cache -static uint cache_count[2]; -static double cache_size_total[2]; -static AtomFnSet ever_cached_files; -static uint conflict_misses; -static double conflict_miss_size_total; -static uint block_cache_count[2]; - -// archive builder -static uint ab_connection_attempts; // total number of trace entries -static uint ab_repeated_connections; // how many of these were not unique - - -// convenience functions for measuring elapsed time in an interval. -// by exposing start/finish calls, we avoid callers from querying -// timestamps when stats are disabled. -static double start_time; -static void timer_start(double* start_time_storage = &start_time) -{ - // make sure no measurement is currently active - // (since start_time is shared static storage) - debug_assert(*start_time_storage == 0.0); - *start_time_storage = get_time(); -} -static double timer_reset(double* start_time_storage = &start_time) -{ - double elapsed = get_time() - *start_time_storage; - *start_time_storage = 0.0; - return elapsed; -} - -//----------------------------------------------------------------------------- - -// -// vfs -// - -void stats_vfs_file_add(size_t file_size) -{ - vfs_files++; - vfs_size_total += file_size; -} - -void stats_vfs_file_remove(size_t file_size) -{ - vfs_files--; - vfs_size_total -= file_size; -} - - -void stats_vfs_init_start() -{ - timer_start(); -} - -void stats_vfs_init_finish() -{ - vfs_init_elapsed_time += timer_reset(); -} - - -// -// file -// - -void stats_unique_name(size_t name_len) -{ - unique_names++; - unique_name_len_total += name_len; -} - - -void stats_open(const char* atom_fn, size_t file_size) -{ - open_files_cur++; - open_files_max = std::max(open_files_max, open_files_cur); - - PairIB ret = opened_files.insert(atom_fn); - // hadn't been opened yet - if(ret.second) - opened_file_size_total += file_size; -} - -void stats_close() -{ - debug_assert(open_files_cur > 0); - open_files_cur--; -} - - -// -// file_buf -// - -void stats_buf_alloc(size_t user_size, size_t padded_size) -{ - extant_bufs_cur++; - extant_bufs_max = std::max(extant_bufs_max, extant_bufs_cur); - extant_bufs_total++; - - buf_user_size_total += user_size; - buf_padded_size_total += padded_size; -} - -void stats_buf_free() -{ - debug_assert(extant_bufs_cur > 0); - extant_bufs_cur--; -} - -void stats_buf_ref() -{ - extant_bufs_cur++; -} - - -// -// file_io -// - -void stats_io_user_request(size_t user_size) -{ - user_ios++; - user_io_size_total += user_size; -} - -// these bracket file_io's IOManager::run and measure effective throughput. -// note: cannot be called from aio issue/finish because IOManager's -// decompression may cause us to miss the exact end of IO, thus throwing off -// throughput measurements. -void stats_io_sync_start(double* start_time_storage) -{ - timer_start(start_time_storage); -} - -void stats_io_sync_finish(FileIOImplentation fi, FileOp fo, ssize_t user_size, double* start_time_storage) -{ - debug_assert(fi < FI_MAX_IDX); - debug_assert(fo == FO_READ || FO_WRITE); - - // ignore IOs that failed (nothing we can do) - if(user_size > 0) - { - io_actual_size_total[fi][fo] += user_size; - io_elapsed_time[fi][fo] += timer_reset(start_time_storage); - } -} - - -void stats_io_check_seek(const char* atom_fn, u32 block_num) -{ - static const char* last_atom_fn; - static u32 last_block_num; - - // makes debugging ("why are there seeks") a bit nicer by suppressing - // the first (bogus) seek. - if(!last_atom_fn) - goto dont_count_first_seek; - - if(atom_fn != last_atom_fn || // different file OR - block_num != last_block_num+1) // nonsequential - io_seeks++; - -dont_count_first_seek: - last_atom_fn = atom_fn; - last_block_num = block_num; -} - - -void stats_cb_start() -{ - timer_start(); -} - -void stats_cb_finish() -{ - io_process_time_total += timer_reset(); -} - - -// -// file_cache -// - -void stats_cache(CacheRet cr, size_t size, const char* atom_fn) -{ - debug_assert(cr == CR_HIT || cr == CR_MISS); - - if(cr == CR_MISS) - { - PairIB ret = ever_cached_files.insert(atom_fn); - if(!ret.second) // was already cached once - { - conflict_miss_size_total += size; - conflict_misses++; - } - } - - cache_count[cr]++; - cache_size_total[cr] += size; -} - -void stats_block_cache(CacheRet cr) -{ - debug_assert(cr == CR_HIT || cr == CR_MISS); - block_cache_count[cr]++; -} - - -// -// archive builder -// - -void stats_ab_connection(bool already_exists) -{ - ab_connection_attempts++; - if(already_exists) - ab_repeated_connections++; -} - - -//----------------------------------------------------------------------------- - -template int percent(T num, T divisor) -{ - if(!divisor) - return 0; - return (int)(100*num / divisor); -} - -void file_stats_dump() -{ - if(!debug_filter_allows("FILE_STATS|")) - return; - - const double KB = 1e3; const double MB = 1e6; const double ms = 1e-3; - - debug_printf("--------------------------------------------------------------------------------\n"); - debug_printf("File statistics:\n"); - - // note: we split the reports into several debug_printfs for clarity; - // this is necessary anyway due to fixed-size buffer. - - debug_printf( - "\nvfs:\n" - "Total files: %u (%g MB)\n" - "Init/mount time: %g ms\n", - vfs_files, vfs_size_total/MB, - vfs_init_elapsed_time/ms - ); - - debug_printf( - "\nfile:\n" - "Total names: %u (%u KB)\n" - "Accessed files: %u (%g MB) -- %u%% of data set\n" - "Max. concurrent: %u; leaked: %u.\n", - unique_names, unique_name_len_total/1000, - opened_files.size(), opened_file_size_total/MB, percent(opened_files.size(), (size_t)vfs_files), - open_files_max, open_files_cur - ); - - debug_printf( - "\nfile_buf:\n" - "Total buffers used: %u (%g MB)\n" - "Max concurrent: %u; leaked: %u\n" - "Internal fragmentation: %d%%\n", - extant_bufs_total, buf_user_size_total/MB, - extant_bufs_max, extant_bufs_cur, - percent(buf_padded_size_total-buf_user_size_total, buf_user_size_total) - ); - - debug_printf( - "\nfile_io:\n" - "Total user load requests: %u (%g MB)\n" - "IO thoughput [MB/s; 0=never happened]:\n" - " lowio: R=%.3g, W=%.3g\n" - " aio: R=%.3g, W=%.3g\n" - "Average size = %g KB; seeks: %u; total callback time: %g ms\n" - "Total data actually read from disk = %g MB\n", - user_ios, user_io_size_total/MB, -#define THROUGHPUT(impl, op) (io_elapsed_time[impl][op] == 0.0)? 0.0 : (io_actual_size_total[impl][op] / io_elapsed_time[impl][op] / MB) - THROUGHPUT(FI_LOWIO, FO_READ), THROUGHPUT(FI_LOWIO, FO_WRITE), - THROUGHPUT(FI_AIO , FO_READ), THROUGHPUT(FI_AIO , FO_WRITE), - user_io_size_total/user_ios/KB, io_seeks, io_process_time_total/ms, - (io_actual_size_total[FI_LOWIO][FO_READ]+io_actual_size_total[FI_AIO][FO_READ])/MB - ); - - debug_printf( - "\nfile_cache:\n" - "Hits: %u (%g MB); misses %u (%g MB); ratio: %u%%\n" - "Percent of requested bytes satisfied by cache: %u%%; non-compulsory misses: %u (%u%% of misses)\n" - "Block hits: %u; misses: %u; ratio: %u%%\n", - cache_count[CR_HIT], cache_size_total[CR_HIT]/MB, cache_count[CR_MISS], cache_size_total[CR_MISS]/MB, percent(cache_count[CR_HIT], cache_count[CR_HIT]+cache_count[CR_MISS]), - percent(cache_size_total[CR_HIT], cache_size_total[CR_HIT]+cache_size_total[CR_MISS]), conflict_misses, percent(conflict_misses, cache_count[CR_MISS]), - block_cache_count[CR_HIT], block_cache_count[CR_MISS], percent(block_cache_count[CR_HIT], block_cache_count[CR_HIT]+block_cache_count[CR_MISS]) - ); - - debug_printf( - "\nvfs_optimizer:\n" - "Total trace entries: %u; repeated connections: %u; unique files: %u\n", - ab_connection_attempts, ab_repeated_connections, ab_connection_attempts-ab_repeated_connections - ); -} diff --git a/source/lib/res/file/file_stats.h b/source/lib/res/file/file_stats.h deleted file mode 100644 index 586aa71b46..0000000000 --- a/source/lib/res/file/file_stats.h +++ /dev/null @@ -1,82 +0,0 @@ -/** - * ========================================================================= - * File : file_stats.h - * Project : 0 A.D. - * Description : gathers statistics from all file modules. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_FILE_STATS -#define INCLUDED_FILE_STATS - -#define FILE_STATS_ENABLED 1 - - -enum FileIOImplentation { FI_LOWIO, FI_AIO, FI_BCACHE, FI_MAX_IDX }; -enum FileOp { FO_READ, FO_WRITE }; -enum CacheRet { CR_HIT, CR_MISS }; - - -#if FILE_STATS_ENABLED - -// vfs -extern void stats_vfs_file_add(size_t file_size); -extern void stats_vfs_file_remove(size_t file_size); -extern void stats_vfs_init_start(); -extern void stats_vfs_init_finish(); - -// file -extern void stats_unique_name(size_t name_len); -extern void stats_open(const char* atom_fn, size_t file_size); -extern void stats_close(); - -// file_buf -extern void stats_buf_alloc(size_t user_size, size_t padded_size); -extern void stats_buf_free(); -extern void stats_buf_ref(); - -// file_io -extern void stats_io_user_request(size_t user_size); -extern void stats_io_sync_start(double* start_time_storage); -extern void stats_io_sync_finish(FileIOImplentation fi, FileOp fo, ssize_t user_size, double* start_time_storage); -extern void stats_io_check_seek(const char* atom_fn, u32 block_num); -extern void stats_cb_start(); -extern void stats_cb_finish(); - -// file_cache -extern void stats_cache(CacheRet cr, size_t size, const char* atom_fn); -extern void stats_block_cache(CacheRet cr); - -// archive builder -extern void stats_ab_connection(bool already_exists); - -extern void file_stats_dump(); - -#else - -#define stats_vfs_file_add(file_size) -#define stats_vfs_file_remove(file_size) -#define stats_vfs_init_start() -#define stats_vfs_init_finish() -#define stats_unique_name(name_len) -#define stats_open(atom_fn, file_size) -#define stats_close() -#define stats_buf_alloc(user_size, padded_size) -#define stats_buf_free() -#define stats_buf_ref() -#define stats_io_user_request(user_size) -#define stats_io_sync_start(disk_pos, start_time_storage) -#define stats_io_sync_finish(fi, fo, user_size, start_time_storage) -#define stats_io_check_seek(atom_fn, block_num) -#define stats_cb_start() -#define stats_cb_finish() -#define stats_cache(cr, size, atom_fn) -#define stats_block_cache(cr) -#define stats_ab_connection(already_exists) -#define file_stats_dump() - -#endif - -#endif // #ifndef INCLUDED_FILE_STATS diff --git a/source/lib/res/file/file_util.cpp b/source/lib/res/file/file_util.cpp deleted file mode 100644 index d12169717a..0000000000 --- a/source/lib/res/file/file_util.cpp +++ /dev/null @@ -1,319 +0,0 @@ -/** - * ========================================================================= - * File : file_util.cpp - * Project : 0 A.D. - * Description : utility functions for file and path handling modules - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" - -#include - -#include "lib/regex.h" -#include "file_internal.h" - - -static bool dirent_less(const DirEnt& d1, const DirEnt& d2) -{ - return strcmp(d1.name, d2.name) < 0; -} - -// enumerate all directory entries in ; add to container and -// then sort it by filename. -LibError file_get_sorted_dirents(const char* P_path, DirEnts& dirents) -{ - DirIterator d; - RETURN_ERR(dir_open(P_path, &d)); - - dirents.reserve(50); // preallocate for efficiency - - DirEnt ent; - for(;;) - { - LibError ret = dir_next_ent(&d, &ent); - if(ret == ERR::DIR_END) - break; - RETURN_ERR(ret); - - ent.name = file_make_unique_fn_copy(ent.name); - dirents.push_back(ent); - } - - std::sort(dirents.begin(), dirents.end(), dirent_less); - - (void)dir_close(&d); - return INFO::OK; -} - - -// call for each file and subdirectory in (alphabetical order), -// passing the entry name (not full path!), stat info, and . -// -// first builds a list of entries (sorted) and remembers if an error occurred. -// if returns non-zero, abort immediately and return that; otherwise, -// return first error encountered while listing files, or 0 on success. -// -// rationale: -// this makes file_enum and zip_enum slightly incompatible, since zip_enum -// returns the full path. that's necessary because VFS zip_cb -// has no other way of determining what VFS dir a Zip file is in, -// since zip_enum enumerates all files in the archive (not only those -// in a given dir). no big deal though, since add_ent has to -// special-case Zip files anyway. -// the advantage here is simplicity, and sparing callbacks the trouble -// of converting from/to native path (we just give 'em the dirent name). -LibError file_enum(const char* P_path, const FileCB cb, const uintptr_t user) -{ - LibError stat_err = INFO::OK; // first error encountered by stat() - LibError cb_err = INFO::OK; // first error returned by cb - - DirEnts dirents; - RETURN_ERR(file_get_sorted_dirents(P_path, dirents)); - - // call back for each entry (now sorted); - // first, expand each DirEnt to full struct stat (we store as such to - // reduce memory use and therefore speed up sorting) - struct stat s; - memset(&s, 0, sizeof(s)); - // .. not needed for plain files (OS opens them; memento doesn't help) - const uintptr_t memento = 0; - for(DirEntCIt it = dirents.begin(); it != dirents.end(); ++it) - { - const DirEnt& dirent = *it; - s.st_mode = (dirent.size == -1)? S_IFDIR : S_IFREG; - s.st_size = dirent.size; - s.st_mtime = dirent.mtime; - LibError ret = cb(dirent.name, &s, memento, user); - if(ret != INFO::CB_CONTINUE) - { - cb_err = ret; // first error (since we now abort) - break; - } - } - - if(cb_err != INFO::OK) - return cb_err; - return stat_err; -} - - - -// retrieve the next (order is unspecified) dir entry matching . -// return 0 on success, ERR::DIR_END if no matching entry was found, -// or a negative error code on failure. -// filter values: -// - 0: anything; -// - "/": any subdirectory; -// - "/|": any subdirectory, or as below with ; -// - : any file whose name matches; ? and * wildcards are allowed. -// -// note that the directory entries are only scanned once; after the -// end is reached (-> ERR::DIR_END returned), no further entries can -// be retrieved, even if filter changes (which shouldn't happen - see impl). -// -// rationale: we do not sort directory entries alphabetically here. -// most callers don't need it and the overhead is considerable -// (we'd have to store all entries in a vector). it is left up to -// higher-level code such as VfsUtil. -LibError dir_filtered_next_ent(DirIterator* di, DirEnt* ent, const char* filter) -{ - // warn if scanning the directory twice with different filters - // (this used to work with dir/file because they were stored separately). - // it is imaginable that someone will want to change it, but until - // there's a good reason, leave this check in. note: only comparing - // pointers isn't 100% certain, but it's safe enough and easy. - if(!di->filter_latched) - { - di->filter = filter; - di->filter_latched = 1; - } - if(di->filter != filter) - debug_warn("filter has changed for this directory. are you scanning it twice?"); - - bool want_dir = true; - if(filter) - { - // directory - if(filter[0] == '/') - { - // .. and also files - if(filter[1] == '|') - filter += 2; - } - // file only - else - want_dir = false; - } - - // loop until ent matches what is requested, or end of directory. - for(;;) - { - RETURN_ERR(xdir_next_ent(di, ent)); - - if(DIRENT_IS_DIR(ent)) - { - if(want_dir) - break; - } - else - { - // (note: filter = 0 matches anything) - if(match_wildcard(ent->name, filter)) - break; - } - } - - return INFO::OK; -} - - -// call for each entry matching (see vfs_next_dirent) in -// directory ; if flags & VFS_DIR_RECURSIVE, entries in -// subdirectories are also returned. -// -// note: EnumDirEntsCB path and ent are only valid during the callback. -LibError vfs_dir_enum(const char* start_path, uint flags, const char* user_filter, - DirEnumCB cb, uintptr_t cbData) -{ - debug_assert((flags & ~(VFS_DIR_RECURSIVE)) == 0); - const bool recursive = (flags & VFS_DIR_RECURSIVE) != 0; - - char filter_buf[PATH_MAX]; - const char* filter = user_filter; - bool user_filter_wants_dirs = true; - if(user_filter) - { - if(user_filter[0] != '/') - user_filter_wants_dirs = false; - - // we need subdirectories and the caller hasn't already requested them - if(recursive && !user_filter_wants_dirs) - { - snprintf(filter_buf, sizeof(filter_buf), "/|%s", user_filter); - filter = filter_buf; - } - } - - - // note: FIFO queue instead of recursion is much more efficient - // (less stack usage; avoids seeks by reading all entries in a - // directory consecutively) - - std::queue dir_queue; - dir_queue.push(file_make_unique_fn_copy(start_path)); - - // for each directory: - do - { - // get current directory path from queue - // note: can't refer to the queue contents - those are invalidated - // as soon as a directory is pushed onto it. - PathPackage pp; - (void)path_package_set_dir(&pp, dir_queue.front()); - dir_queue.pop(); - - Handle hdir = vfs_dir_open(pp.path); - if(hdir <= 0) - { - debug_warn("vfs_open_dir failed"); - continue; - } - - // for each entry (file, subdir) in directory: - DirEnt ent; - while(vfs_dir_next_ent(hdir, &ent, filter) == 0) - { - // build complete path (DirEnt only stores entry name) - (void)path_package_append_file(&pp, ent.name); - const char* atom_path = file_make_unique_fn_copy(pp.path); - - if(DIRENT_IS_DIR(&ent)) - { - if(recursive) - dir_queue.push(atom_path); - - if(user_filter_wants_dirs) - cb(atom_path, &ent, cbData); - } - else - cb(atom_path, &ent, cbData); - } - - vfs_dir_close(hdir); - } - while(!dir_queue.empty()); - - return INFO::OK; -} - - -// fill V_next_fn (which must be big enough for PATH_MAX chars) with -// the next numbered filename according to the pattern defined by V_fn_fmt. -// must be initially zeroed (e.g. by defining as static) and passed -// each time. -// if (default), the paths are treated as VFS paths; otherwise, -// file.cpp's functions are used. this is necessary because one of -// our callers needs a filename for VFS archive files. -// -// this function is useful when creating new files which are not to -// overwrite the previous ones, e.g. screenshots. -// example for V_fn_fmt: "screenshots/screenshot%04d.png". -void next_numbered_filename(const char* fn_fmt, - NextNumberedFilenameInfo* nfi, char* next_fn, bool use_vfs) -{ - // (first call only:) scan directory and set next_num according to - // highest matching filename found. this avoids filling "holes" in - // the number series due to deleted files, which could be confusing. - // example: add 1st and 2nd; [exit] delete 1st; [restart] - // add 3rd -> without this measure it would get number 1, not 3. - if(nfi->next_num == 0) - { - char dir[PATH_MAX]; - path_dir_only(fn_fmt, dir); - const char* name_fmt = path_name_only(fn_fmt); - - int max_num = -1; int num; - DirEnt ent; - - if(use_vfs) - { - Handle hd = vfs_dir_open(dir); - if(hd > 0) - { - while(vfs_dir_next_ent(hd, &ent, 0) == INFO::OK) - { - if(!DIRENT_IS_DIR(&ent) && sscanf(ent.name, name_fmt, &num) == 1) - max_num = std::max(num, max_num); - } - (void)vfs_dir_close(hd); - } - } - else - { - DirIterator it; - if(dir_open(dir, &it) == INFO::OK) - { - while(dir_next_ent(&it, &ent) == INFO::OK) - if(!DIRENT_IS_DIR(&ent) && sscanf(ent.name, name_fmt, &num) == 1) - max_num = std::max(num, max_num); - (void)dir_close(&it); - } - } - - nfi->next_num = max_num+1; - } - - bool (*exists)(const char* fn) = use_vfs? vfs_exists : file_exists; - - // now increment number until that file doesn't yet exist. - // this is fairly slow, but typically only happens once due - // to scan loop above. (we still need to provide for looping since - // someone may have added files in the meantime) - // binary search isn't expected to improve things. - do - snprintf(next_fn, PATH_MAX, fn_fmt, nfi->next_num++); - while(exists(next_fn)); -} diff --git a/source/lib/res/file/path.cpp b/source/lib/res/file/path.cpp deleted file mode 100644 index 8518a773da..0000000000 --- a/source/lib/res/file/path.cpp +++ /dev/null @@ -1,313 +0,0 @@ -/** - * ========================================================================= - * File : path.cpp - * Project : 0 A.D. - * Description : helper functions for VFS paths. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "path.h" - -#include - -#include "lib/posix/posix_filesystem.h" -#include "lib/adts.h" -#include "lib/rand.h" -#include "lib/allocators.h" -#include "lib/sysdep/sysdep.h" -#include "lib/module_init.h" -#include "file_internal.h" - - -ERROR_ASSOCIATE(ERR::ROOT_DIR_ALREADY_SET, "Attempting to set FS root dir more than once", -1); - - -// path types: -// p_*: posix (e.g. mount object name or for open()) -// v_*: vfs (e.g. mount point) -// fn : filename only (e.g. from readdir) -// dir_name: directory only, no path (e.g. subdir name) -// -// all paths must be relative (no leading '/'); components are separated -// by '/'; no ':', '\\', "." or ".." allowed; root dir is "". -// -// grammar: -// path ::= dir*file? -// dir ::= name/ -// file ::= name -// name ::= [^/] - -enum Conversion -{ - TO_NATIVE, - TO_PORTABLE -}; - -static LibError convert_path(char* dst, const char* src, Conversion conv = TO_NATIVE) -{ - // SYS_DIR_SEP is assumed to be a single character! - - const char* s = src; - char* d = dst; - - char from = SYS_DIR_SEP, to = '/'; - if(conv == TO_NATIVE) - from = '/', to = SYS_DIR_SEP; - - size_t len = 0; - - for(;;) - { - len++; - if(len >= PATH_MAX) - WARN_RETURN(ERR::PATH_LENGTH); - - char c = *s++; - - if(c == from) - c = to; - - *d++ = c; - - // end of string - done - if(c == '\0') - return INFO::OK; - } -} - - -// set by file_set_root_dir -static char n_root_dir[PATH_MAX]; -static size_t n_root_dir_len; - - -// return the native equivalent of the given relative portable path -// (i.e. convert all '/' to the platform's directory separator) -// makes sure length < PATH_MAX. -LibError file_make_native_path(const char* path, char* n_path) -{ - return convert_path(n_path, path, TO_NATIVE); -} - -// return the portable equivalent of the given relative native path -// (i.e. convert the platform's directory separators to '/') -// makes sure length < PATH_MAX. -LibError file_make_portable_path(const char* n_path, char* path) -{ - return convert_path(path, n_path, TO_PORTABLE); -} - - -// return the native equivalent of the given portable path -// (i.e. convert all '/' to the platform's directory separator). -// also prepends current directory => n_full_path is absolute. -// makes sure length < PATH_MAX. -LibError file_make_full_native_path(const char* path, char* n_full_path) -{ - debug_assert(path != n_full_path); // doesn't work in-place - - strcpy_s(n_full_path, PATH_MAX, n_root_dir); - return convert_path(n_full_path+n_root_dir_len, path, TO_NATIVE); -} - -// return the portable equivalent of the given relative native path -// (i.e. convert the platform's directory separators to '/') -// n_full_path is absolute; if it doesn't match the current dir, fail. -// (note: portable paths are always relative to the file root dir). -// makes sure length < PATH_MAX. -LibError file_make_full_portable_path(const char* n_full_path, char* path) -{ - debug_assert(path != n_full_path); // doesn't work in-place - - if(strncmp(n_full_path, n_root_dir, n_root_dir_len) != 0) - WARN_RETURN(ERR::TNODE_NOT_FOUND); - return convert_path(path, n_full_path+n_root_dir_len, TO_PORTABLE); -} - - -// security check: only allow attempting to chdir once, so that malicious -// code cannot circumvent the VFS checks that disallow access to anything -// above the current directory (set here). -// this routine is called early at startup, so any subsequent attempts -// are likely bogus. -// we provide for resetting this from the self-test to allow clean -// re-init of the individual tests. -static bool root_dir_established; - - -// establish the root directory from , which is treated as -// relative to the executable's directory (determined via argv[0]). -// all relative file paths passed to this module will be based from -// this root dir. -// -// example: executable in "$install_dir/system"; desired root dir is -// "$install_dir/data" => rel_path = "../data". -// -// argv[0] is necessary because the current directory is unknown at startup -// (e.g. it isn't set when invoked via batch file), and this is the -// easiest portable way to find our install directory. -// -// can only be called once, by design (see below). rel_path is trusted. -LibError file_set_root_dir(const char* argv0, const char* rel_path) -{ - if(root_dir_established) - WARN_RETURN(ERR::ROOT_DIR_ALREADY_SET); - root_dir_established = true; - - // get full path to executable - char n_path[PATH_MAX]; - // .. first try safe, but system-dependent version - if(sys_get_executable_name(n_path, PATH_MAX) < 0) - { - // .. failed; use argv[0] - if(!realpath(argv0, n_path)) - return LibError_from_errno(); - } - - // make sure it's valid - if(access(n_path, X_OK) < 0) - return LibError_from_errno(); - - // strip executable name, append rel_path, convert to native - char* start_of_fn = (char*)path_name_only(n_path); - RETURN_ERR(file_make_native_path(rel_path, start_of_fn)); - - // get actual root dir - previous n_path may include ".." - // (slight optimization, speeds up path lookup) - if(!realpath(n_path, n_root_dir)) - return LibError_from_errno(); - - // .. append SYS_DIR_SEP to simplify code that uses n_root_dir - n_root_dir_len = strlen(n_root_dir)+1; // +1 for trailing SYS_DIR_SEP - debug_assert((n_root_dir_len+1) < sizeof(n_root_dir)); // Just checking - n_root_dir[n_root_dir_len-1] = SYS_DIR_SEP; - // You might think that n_root_dir is already 0-terminated, since it's - // static - but that might not be true after calling file_reset_root_dir! - n_root_dir[n_root_dir_len] = 0; - - return INFO::OK; -} - - -void path_reset_root_dir() -{ - // see comment at root_dir_established. - debug_assert(root_dir_established); - n_root_dir[0] = '\0'; - n_root_dir_len = 0; - root_dir_established = false; -} - - -//----------------------------------------------------------------------------- -// storage for path strings -//----------------------------------------------------------------------------- - -// rationale: we want a constant-time IsAtomFn(string pointer) lookup: -// this avoids any overhead of calling file_make_unique_fn_copy on -// already-atomized strings. that requires allocating from one contiguous -// arena, which is also more memory-efficient than the heap (no headers). -static Pool atom_pool; - -typedef DynHashTbl AtomMap; -static AtomMap atom_map; - -bool path_is_atom_fn(const char* fn) -{ - return pool_contains(&atom_pool, (void*)fn); -} - -// allocate a copy of P_fn in our string pool. strings are equal iff -// their addresses are equal, thus allowing fast comparison. -// -// if the (generous) filename storage is full, 0 is returned. -// this is not ever expected to happen; callers need not check the -// return value because a warning is raised anyway. -const char* file_make_unique_fn_copy(const char* P_fn) -{ - // early out: if already an atom, return immediately. - if(path_is_atom_fn(P_fn)) - return P_fn; - - const size_t fn_len = strlen(P_fn); - const char* unique_fn; - - // check if already allocated; return existing copy if so. - // - // rationale: the entire storage could be done via container, - // rather than simply using it as a lookup mapping. - // however, DynHashTbl together with Pool (see above) is more efficient. - unique_fn = atom_map.find(P_fn); - if(unique_fn) - return unique_fn; - - unique_fn = (const char*)pool_alloc(&atom_pool, fn_len+1); - if(!unique_fn) - { - DEBUG_WARN_ERR(ERR::NO_MEM); - return 0; - } - cpu_memcpy((void*)unique_fn, P_fn, fn_len); - ((char*)unique_fn)[fn_len] = '\0'; - - atom_map.insert(unique_fn, unique_fn); - - stats_unique_name(fn_len); - return unique_fn; -} - - -static ModuleInitState initState; - -void path_init() -{ - if(!ModuleShouldInitialize(&initState)) - return; - - pool_create(&atom_pool, 8*MiB, POOL_VARIABLE_ALLOCS); -} - -void path_shutdown() -{ - if(!ModuleShouldShutdown(&initState)) - return; - - atom_map.clear(); - (void)pool_destroy(&atom_pool); -} - - -const char* file_get_random_name() -{ - // there had better be names in atom_pool, else this will fail. - debug_assert(atom_pool.da.pos != 0); - -again: - const size_t start_ofs = (size_t)rand(0, (uint)atom_pool.da.pos); - - // scan back to start of string (don't scan ahead; this must - // work even if atom_pool only contains one entry). - const char* start = (const char*)atom_pool.da.base+start_ofs; - for(size_t i = 0; i < start_ofs; i++) - { - if(*start == '\0') - break; - start--; - } - - // skip past the '\0' we found. loop is needed because there may be - // several if we land in padding (due to pool alignment). - size_t chars_left = atom_pool.da.pos - start_ofs; - for(; *start == '\0'; start++) - { - // we had landed in padding at the end of the buffer. - if(chars_left-- == 0) - goto again; - } - - const char* next_name = start; - return next_name; -} diff --git a/source/lib/res/file/path.h b/source/lib/res/file/path.h deleted file mode 100644 index 9033312cf7..0000000000 --- a/source/lib/res/file/path.h +++ /dev/null @@ -1,63 +0,0 @@ -/** - * ========================================================================= - * File : path.h - * Project : 0 A.D. - * Description : helper functions for VFS paths. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_PATH -#define INCLUDED_PATH - - -namespace ERR -{ - const LibError ROOT_DIR_ALREADY_SET = -110200; -} - - -#define VFS_PATH_IS_DIR(path) (*path == '\0' || path[strlen(path)-1] == '/') - -struct NextNumberedFilenameInfo -{ - int next_num; -}; - -// fill V_next_fn (which must be big enough for PATH_MAX chars) with -// the next numbered filename according to the pattern defined by V_fn_fmt. -// must be initially zeroed (e.g. by defining as static) and passed -// each time. -// if (default), the paths are treated as VFS paths; otherwise, -// file.cpp's functions are used. this is necessary because one of -// our callers needs a filename for VFS archive files. -// -// this function is useful when creating new files which are not to -// overwrite the previous ones, e.g. screenshots. -// example for V_fn_fmt: "screenshots/screenshot%04d.png". -extern void next_numbered_filename(const char* V_fn_fmt, - NextNumberedFilenameInfo* nfi, char* V_next_fn, bool use_vfs = true); - - -extern bool path_is_atom_fn(const char* fn); - -extern const char* file_get_random_name(); - - -/** - * reset root directory that was previously established via file_set_root_dir. - * - * this function avoids the security complaint that would be raised if - * file_set_root_dir is called twice; it is provided for the - * legitimate application of a self-test setUp()/tearDown(). - **/ -extern void path_reset_root_dir(); - -// note: other functions are declared directly in the public file.h header. - - -extern void path_init(); -extern void path_shutdown(); - -#endif // #ifndef INCLUDED_PATH diff --git a/source/lib/res/file/tests/test_file_cache.h b/source/lib/res/file/tests/test_file_cache.h deleted file mode 100644 index 82407dd0c1..0000000000 --- a/source/lib/res/file/tests/test_file_cache.h +++ /dev/null @@ -1,51 +0,0 @@ -#include "lib/self_test.h" - -#include "lib/res/file/file_cache.h" -#include "lib/rand.h" - -class TestFileCache : public CxxTest::TestSuite -{ - enum { TEST_ALLOC_TOTAL = 100*1000*1000 }; -public: - void test_cache_allocator() - { - // allocated address -> its size - typedef std::map AllocMap; - AllocMap allocations; - - // put allocator through its paces by allocating several times - // its capacity (this ensures memory is reused) - srand(1); - size_t total_size_used = 0; - while(total_size_used < TEST_ALLOC_TOTAL) - { - size_t size = rand(1, TEST_ALLOC_TOTAL/16); - total_size_used += size; - void* p; - // until successful alloc: - for(;;) - { - p = file_cache_allocator_alloc(size); - if(p) - break; - // out of room - remove a previous allocation - // .. choose one at random - size_t chosen_idx = (size_t)rand(0, (uint)allocations.size()); - AllocMap::iterator it = allocations.begin(); - for(; chosen_idx != 0; chosen_idx--) - ++it; - file_cache_allocator_free(it->first, it->second); - allocations.erase(it); - } - - // must not already have been allocated - TS_ASSERT_EQUALS(allocations.find(p), allocations.end()); - allocations[p] = size; - } - - // reset to virginal state - // note: even though everything has now been freed, this is - // necessary since the freelists may be a bit scattered already. - file_cache_allocator_reset(); - } -}; diff --git a/source/lib/res/file/tests/test_path.h b/source/lib/res/file/tests/test_path.h deleted file mode 100644 index a9ba0d85fa..0000000000 --- a/source/lib/res/file/tests/test_path.h +++ /dev/null @@ -1,68 +0,0 @@ -#include "lib/self_test.h" - -#include "lib/self_test.h" -#include "lib/res/file/path.h" -#include "lib/res/file/file.h" - -class TestPath : public CxxTest::TestSuite -{ -public: - void test_conversion() - { - char N_path[PATH_MAX] = {0}; - TS_ASSERT_OK(file_make_native_path("a/b/c", N_path)); -#if OS_WIN - TS_ASSERT_STR_EQUALS(N_path, "a\\b\\c"); -#else - TS_ASSERT_STR_EQUALS(N_path, "a/b/c"); -#endif - - char P_path[PATH_MAX] = {0}; - TS_ASSERT_OK(file_make_portable_path("a\\b\\c", P_path)); -#if OS_WIN - TS_ASSERT_STR_EQUALS(P_path, "a/b/c"); -#else - // sounds strange, but correct: on non-Windows, \\ didn't - // get recognized as separators and weren't converted. - TS_ASSERT_STR_EQUALS(P_path, "a\\b\\c"); -#endif - - } - - // file_make_full_*_path is left untested (hard to do so) - - void test_atom() - { - path_init(); - - // file_make_unique_fn_copy - - // .. return same address for same string? - const char* atom1 = file_make_unique_fn_copy("a/bc/def"); - const char* atom2 = file_make_unique_fn_copy("a/bc/def"); - TS_ASSERT_EQUALS(atom1, atom2); - - // .. early out (already in pool) check works? - const char* atom3 = file_make_unique_fn_copy(atom1); - TS_ASSERT_EQUALS(atom3, atom1); - - - // path_is_atom_fn - // is it reported as in pool? - TS_ASSERT(path_is_atom_fn(atom1)); - - // file_get_random_name - // see if the atom added above eventually comes out when a - // random one is returned from the pool. - int tries_left; - for(tries_left = 1000; tries_left != 0; tries_left--) - { - const char* random_name = file_get_random_name(); - if(random_name == atom1) - break; - } - TS_ASSERT(tries_left != 0); - - path_shutdown(); - } -}; diff --git a/source/lib/res/file/vfs.cpp b/source/lib/res/file/vfs.cpp deleted file mode 100644 index 3aff308cf0..0000000000 --- a/source/lib/res/file/vfs.cpp +++ /dev/null @@ -1,758 +0,0 @@ -/** - * ========================================================================= - * File : vfs.cpp - * Project : 0 A.D. - * Description : Handle-based wrapper on top of the vfs_mount API. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "vfs.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lib/adts.h" -#include "lib/timer.h" -#include "lib/res/res.h" -#include "lib/sysdep/dir_watch.h" -#include "file_internal.h" -#include "lib/module_init.h" - -// not safe to call before main! - - -// pathnames are case-insensitive. -// implementation: -// when mounting, we get the exact filenames as reported by the OS; -// we allow open requests with mixed case to match those, -// but still use the correct case when passing to other libraries -// (e.g. the actual open() syscall, called via file_open). -// rationale: -// necessary, because some exporters output .EXT uppercase extensions -// and it's unreasonable to expect that users will always get it right. - - -// rationale for no forcibly-close support: -// issue: -// we might want to edit files while the game has them open. -// usual case: edit file, notify engine that it should be reloaded. -// here: need to tell the engine to stop what it's doing and close the file; -// only then can the artist write to the file, and trigger a reload. -// -// work involved: -// since closing a file with pending aios results in undefined -// behavior on Win32, we would have to keep track of all aios from each file, -// and cancel them. we'd also need to notify the higher level resource user -// that its read was cancelled, as opposed to failing due to read errors -// (which might cause the game to terminate). -// -// this is just more work than benefit. cases where the game holds on to files -// are rare: -// - streaming music (artist can use regular commands to stop the current -// track, or all music) -// - if the engine happens to be reading that file at the moment (expected -// to happen only during loading, and these are usually one-shot anway, -// i.e. it'll be done soon) -// - bug (someone didn't close a file - tough luck, and should be fixed -// instead of hacking around it). -// - archives (these remain open. allowing reload would mean we'd have to keep -// track of all files from an archive, and reload them all. another hassle. -// anyway, if files are to be changed in-game, then change the plain-file -// version - that's what they're for). - - -/////////////////////////////////////////////////////////////////////////////// -// -// directory -// -/////////////////////////////////////////////////////////////////////////////// - - -struct VDir -{ - DirIterator di; - uint di_valid : 1; // will be closed iff == 1 -}; - -H_TYPE_DEFINE(VDir); - -static void VDir_init(VDir* UNUSED(vd), va_list UNUSED(args)) -{ -} - -static void VDir_dtor(VDir* vd) -{ - // note: DirIterator has no way of checking if it's valid; - // we must therefore only free it if reload() succeeded. - if(vd->di_valid) - { - xdir_close(&vd->di); - vd->di_valid = 0; - } -} - -static LibError VDir_reload(VDir* vd, const char* V_dir_path, Handle UNUSED(hvd)) -{ - debug_assert(VFS_PATH_IS_DIR(V_dir_path)); - - RETURN_ERR(xdir_open(V_dir_path, &vd->di)); - vd->di_valid = 1; - return INFO::OK; -} - -static LibError VDir_validate(const VDir* vd) -{ - // note: is mostly opaque and cannot be validated. - if(vd->di.filter && !isprint(vd->di.filter[0])) - WARN_RETURN(ERR::_1); - return INFO::OK; -} - -static LibError VDir_to_string(const VDir* vd, char* buf) -{ - const char* filter = vd->di.filter; - if(!vd->di.filter_latched) - filter = "?"; - if(!filter) - filter = "*"; - snprintf(buf, H_STRING_LEN, "(\"%s\")", filter); - return INFO::OK; -} - - -// open a directory for reading its entries via vfs_next_dirent. -// V_dir must end in '/' to indicate it's a directory path. -Handle vfs_dir_open(const char* V_dir_path) -{ - // must disallow handle caching because this object is not - // copy-equivalent (since the iterator is advanced by each user). - return h_alloc(H_VDir, V_dir_path, RES_NO_CACHE); -} - - -// close the handle to a directory. -LibError vfs_dir_close(Handle& hd) -{ - return h_free(hd, H_VDir); -} - - -// retrieve the next (order is unspecified) dir entry matching . -// return 0 on success, ERR::DIR_END if no matching entry was found, -// or a negative error code on failure. -// filter values: -// - 0: anything; -// - "/": any subdirectory; -// - "/|": any subdirectory, or as below with ; -// - : any file whose name matches; ? and * wildcards are allowed. -// -// note that the directory entries are only scanned once; after the -// end is reached (-> ERR::DIR_END returned), no further entries can -// be retrieved, even if filter changes (which shouldn't happen - see impl). -// -// rationale: we do not sort directory entries alphabetically here. -// most callers don't need it and the overhead is considerable -// (we'd have to store all entries in a vector). it is left up to -// higher-level code such as VfsUtil. -LibError vfs_dir_next_ent(const Handle hd, DirEnt* ent, const char* filter) -{ - H_DEREF(hd, VDir, vd); - return dir_filtered_next_ent(&vd->di, ent, filter); -} - - - -/////////////////////////////////////////////////////////////////////////////// -// -// file -// -/////////////////////////////////////////////////////////////////////////////// - - -// return actual path to the specified file: -// "/fn" or "/fn". -LibError vfs_realpath(const char* V_path, char* realpath) -{ - TFile* tf; - CHECK_ERR(tree_lookup(V_path, &tf)); - - const char* atom_fn = tfile_get_atom_fn(tf); - const Mount* m = tfile_get_mount(tf); - return mount_realpath(atom_fn, m, realpath); -} - - - -// does the specified file exist? return false on error. -// useful because a "file not found" warning is not raised, unlike vfs_stat. -bool vfs_exists(const char* V_fn) -{ - TFile* tf; - return (tree_lookup(V_fn, &tf) == 0); -} - - -// get file status (mode, size, mtime). output param is zeroed on error. -LibError vfs_stat(const char* V_path, struct stat* s) -{ - memset(s, 0, sizeof(*s)); - - TFile* tf; - CHECK_ERR(tree_lookup(V_path, &tf)); - - return tree_stat(tf, s); -} - - -//----------------------------------------------------------------------------- - -struct VFile -{ - File f; - - // current file pointer. this is necessary because file.cpp's interface - // requires passing an offset for every VIo; see file_io_issue. - off_t ofs; - - // pointer to VFS file info storage; used to update size/mtime - // after a newly written file is closed. - TFile* tf; - - uint is_valid : 1; - - // be aware when adding fields that this struct is quite large, - // and may require increasing the control block size limit. -}; - -H_TYPE_DEFINE(VFile); - -static void VFile_init(VFile* vf, va_list args) -{ - vf->f.flags = va_arg(args, int); -} - -static void VFile_dtor(VFile* vf) -{ - // note: checking if reload() succeeded is unnecessary because - // xfile_close and mem_free_h safely handle 0-initialized data. - WARN_ERR(xfile_close(&vf->f)); - - // update file state in VFS tree - // (must be done after close, since that calculates the size) - if(vf->f.flags & FILE_WRITE) - tree_update_file(vf->tf, vf->f.size, time(0)); // can't fail - - if(vf->is_valid) - stats_close(); -} - -static LibError VFile_reload(VFile* vf, const char* V_path, Handle) -{ - const uint flags = vf->f.flags; - - // we're done if file is already open. need to check this because - // reload order (e.g. if resource opens a file) is unspecified. - if(xfile_is_open(&vf->f)) - return INFO::OK; - - TFile* tf; - uint lf = (flags & FILE_WRITE)? LF_CREATE_MISSING : 0; - LibError err = tree_lookup(V_path, &tf, lf); - if(err < 0) - { - // don't CHECK_ERR - this happens often and the dialog is annoying - debug_printf("lookup failed for %s\n", V_path); - return err; - } - - // careful! FILE_WRITE_TO_TARGET consists of 2 bits; they must both be - // set (one of them is FILE_WRITE, which can be set independently). - // this is a bit ugly but better than requiring users to write - // FILE_WRITE|FILE_WRITE_TO_TARGET. - if((flags & FILE_WRITE_TO_TARGET) == FILE_WRITE_TO_TARGET) - RETURN_ERR(set_mount_to_write_target(tf)); - - RETURN_ERR(xfile_open(V_path, flags, tf, &vf->f)); - - stats_open(vf->f.atom_fn, vf->f.size); - vf->is_valid = 1; - vf->tf = tf; - - return INFO::OK; -} - -static LibError VFile_validate(const VFile* vf) -{ - // doesn't have any invariant we can check. - RETURN_ERR(xfile_validate(&vf->f)); - return INFO::OK; -} - -static LibError VFile_to_string(const VFile* UNUSED(vf), char* buf) -{ - strcpy(buf, ""); // safe - return INFO::OK; -} - - -// return the size of an already opened file, or a negative error code. -ssize_t vfs_size(Handle hf) -{ - H_DEREF(hf, VFile, vf); - return vf->f.size; -} - - -// open the file for synchronous or asynchronous VIo. write access is -// requested via FILE_WRITE flag, and is not possible for files in archives. -// file_flags: default 0 -// -// on failure, a debug_warn is generated and a negative error code returned. -Handle vfs_open(const char* V_fn, uint file_flags) -{ - // keeping files open doesn't make sense in most cases (because the - // file is used to load resources, which are cached at a higher level). - uint res_flags = RES_NO_CACHE; - - // res_flags is for h_alloc and file_flags goes to VFile_init. - // h_alloc already complains on error. - return h_alloc(H_VFile, V_fn, res_flags, file_flags); -} - - -// close the handle to a file. -LibError vfs_close(Handle& hf) -{ - // h_free already complains on error. - return h_free(hf, H_VFile); -} - - -// transfer the next bytes to/from the given file. -// (read or write access was chosen at file-open time). -// -// if non-NULL, is called for each block transferred, passing . -// it returns how much data was actually transferred, or a negative error -// code (in which case we abort the transfer and return that value). -// the callback mechanism is useful for user progress notification or -// processing data while waiting for the next I/O to complete -// (quasi-parallel, without the complexity of threads). -// -// p (value-return) indicates the buffer mode: -// - *p == 0: read into buffer we allocate; set *p. -// caller should mem_free it when no longer needed. -// - *p != 0: read into or write into the buffer *p. -// - p == 0: only read into temp buffers. useful if the callback -// is responsible for processing/copying the transferred blocks. -// since only temp buffers can be added to the cache, -// this is the preferred read method. -// -// return number of bytes transferred (see above), or a negative error code. -ssize_t vfs_io(const Handle hf, const size_t size, FileIOBuf* pbuf, - FileIOCB cb, uintptr_t cbData) -{ - debug_printf("VFS| io: size=%d\n", size); - - H_DEREF(hf, VFile, vf); - File* f = &vf->f; - - stats_io_user_request(size); - trace_notify_io(f->atom_fn, size, f->flags); - - off_t ofs = vf->ofs; - vf->ofs += (off_t)size; - - ssize_t nbytes = xfile_io(&vf->f, ofs, size, pbuf, cb, cbData); - RETURN_ERR(nbytes); - return nbytes; -} - - -// load the entire file into memory. -// p and size are filled with address/size of buffer (0 on failure). -// flags influences IO mode and is typically 0. -// when the file contents are no longer needed, call file_buf_free(buf). -LibError vfs_load(const char* V_fn, FileIOBuf& buf, size_t& size, - uint file_flags, FileIOCB cb, uintptr_t cbData) // all default 0 -{ - debug_printf("VFS| load: V_fn=%s\n", V_fn); - - const char* atom_fn = file_make_unique_fn_copy(V_fn); - const uint fb_flags = (file_flags & FILE_LONG_LIVED)? FB_LONG_LIVED : 0; - buf = file_cache_retrieve(atom_fn, &size, fb_flags); - if(buf) - { - // we want to skip the below code (especially vfs_open) for - // efficiency. that includes stats/trace accounting, though, - // so duplicate that here: - stats_cache(CR_HIT, size, atom_fn); - stats_io_user_request(size); - trace_notify_io(atom_fn, size, file_flags); - - size_t actual_size; - LibError ret = file_io_call_back(buf, size, cb, cbData, actual_size); - if(ret < 0) - file_buf_free(buf); - // we don't care if the cb has "had enough" or whether it would - // accept more data - this is all it gets and we need to - // translate return value to avoid confusing callers. - if(ret == INFO::CB_CONTINUE) - ret = INFO::OK; - size = actual_size; - return ret; - } - - buf = 0; size = 0; // initialize in case something below fails - - Handle hf = vfs_open(atom_fn, file_flags); - H_DEREF(hf, VFile, vf); - - size = vf->f.size; - - buf = FILE_BUF_ALLOC; - ssize_t nread = vfs_io(hf, size, &buf, cb, cbData); - // IO failed - if(nread < 0) - { - file_buf_free(buf); - (void)vfs_close(hf); - buf = 0, size = 0; // make sure they are zeroed - return (LibError)nread; - } - - debug_assert(nread == (ssize_t)size); - - (void)file_cache_add(buf, size, atom_fn, file_flags); - stats_cache(CR_MISS, size, atom_fn); - - (void)vfs_close(hf); - return INFO::OK; -} - - -// caveat: pads file to next max(4kb, sector_size) boundary -// (due to limitation of Win32 FILE_FLAG_NO_BUFFERING I/O). -// if that's a problem, specify FILE_NO_AIO when opening. -ssize_t vfs_store(const char* V_fn, const u8* p, const size_t size, uint flags /* default 0 */) -{ - Handle hf = vfs_open(V_fn, flags|FILE_WRITE); - H_DEREF(hf, VFile, vf); - FileIOBuf buf = (FileIOBuf)p; - const ssize_t ret = vfs_io(hf, size, &buf); - WARN_ERR(vfs_close(hf)); - return ret; -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// asynchronous I/O -// -/////////////////////////////////////////////////////////////////////////////// - -// we don't support forcibly closing files => don't need to keep track of -// all IOs pending for each file. too much work, little benefit. -struct VIo -{ - Handle hf; - size_t size; - u8* buf; - - FileIo io; -}; - -H_TYPE_DEFINE(VIo); - -static void VIo_init(VIo* vio, va_list args) -{ - vio->hf = va_arg(args, Handle); - vio->size = va_arg(args, size_t); - vio->buf = va_arg(args, u8*); -} - -static void VIo_dtor(VIo* vio) -{ - // note: checking if reload() succeeded is unnecessary because - // xfile_io_discard safely handles 0-initialized data. - WARN_ERR(xfile_io_discard(&vio->io)); -} - -// we don't support transparent read resume after file invalidation. -// if the file has changed, we'd risk returning inconsistent data. -// doesn't look possible without controlling the AIO implementation: -// when we cancel, we can't prevent the app from calling -// aio_result, which would terminate the read. -static LibError VIo_reload(VIo* vio, const char* UNUSED(fn), Handle UNUSED(h)) -{ - size_t size = vio->size; - u8* buf = vio->buf; - - H_DEREF(vio->hf, VFile, vf); - off_t ofs = vf->ofs; - vf->ofs += (off_t)size; - - return xfile_io_issue(&vf->f, ofs, size, buf, &vio->io); -} - -static LibError VIo_validate(const VIo* vio) -{ - if(vio->hf < 0) - WARN_RETURN(ERR::_21); - // doesn't have any invariant we can check. - if(debug_is_pointer_bogus(vio->buf)) - WARN_RETURN(ERR::_22); - return xfile_io_validate(&vio->io); -} - -static LibError VIo_to_string(const VIo* vio, char* buf) -{ - snprintf(buf, H_STRING_LEN, "buf=%p size=%d", vio->buf, vio->size); - return INFO::OK; -} - - - -// begin transferring bytes, starting at . get result -// with vfs_io_wait; when no longer needed, free via vfs_io_discard. -Handle vfs_io_issue(Handle hf, size_t size, u8* buf) -{ - const char* fn = 0; - uint flags = 0; - return h_alloc(H_VIo, fn, flags, hf, size, buf); -} - - -// finished with transfer - free its buffer (returned by vfs_io_wait) -LibError vfs_io_discard(Handle& hio) -{ - return h_free(hio, H_VIo); -} - - -// indicates if the VIo referenced by has completed. -// return value: 0 if pending, 1 if complete, < 0 on error. -int vfs_io_has_completed(Handle hio) -{ - H_DEREF(hio, VIo, vio); - return xfile_io_has_completed(&vio->io); -} - - -// wait until the transfer completes, and return its buffer. -// output parameters are zeroed on error. -LibError vfs_io_wait(Handle hio, u8*& p, size_t& size) -{ - H_DEREF(hio, VIo, vio); - return xfile_io_wait(&vio->io, p, size); -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// memory mapping -// -/////////////////////////////////////////////////////////////////////////////// - - -// map the entire (uncompressed!) file into memory. if currently -// already mapped, return the previous mapping (reference-counted). -// output parameters are zeroed on failure. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -LibError vfs_map(const Handle hf, const uint UNUSED(flags), u8*& p, size_t& size) -{ - p = 0; - size = 0; - // need to zero these here in case H_DEREF fails - - H_DEREF(hf, VFile, vf); - return xfile_map(&vf->f, p, size); -} - - -// decrement the reference count for the mapping belonging to file . -// fail if there are no references; remove the mapping if the count reaches 0. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -LibError vfs_unmap(const Handle hf) -{ - H_DEREF(hf, VFile, vf); - return xfile_unmap(&vf->f); -} - - -//----------------------------------------------------------------------------- -// hotloading -//----------------------------------------------------------------------------- - -// called by vfs_reload and vfs_reload_changed_files (which will already -// have rebuilt the VFS - doing so more than once a frame is unnecessary). -static LibError reload_without_rebuild(const char* fn) -{ - // invalidate this file's cached blocks to make sure its contents are - // loaded anew. - RETURN_ERR(file_cache_invalidate(fn)); - - RETURN_ERR(h_reload(fn)); - - return INFO::OK; -} - - -// called via console command. -LibError vfs_reload(const char* fn) -{ - // if currently maps to an archive, the VFS must switch - // over to using the loose file (that was presumably changed). - RETURN_ERR(mount_rebuild()); - - return reload_without_rebuild(fn); -} - -// array of reloads requested this frame (see 'do we really need to -// reload' below). go through gyrations to avoid heap allocs. -const size_t MAX_RELOADS_PER_FRAME = 12; -typedef char Path[PATH_MAX]; -typedef Path PathList[MAX_RELOADS_PER_FRAME]; - -// do we really need to reload? try to avoid the considerable cost of -// rebuilding VFS and scanning all Handles. -static bool can_ignore_reload(const char* V_path, PathList pending_reloads, uint num_pending) -{ - // note: be careful to avoid 'race conditions' depending on the - // timeframe in which notifications reach us. - // example: editor deletes a.tga; we are notified; reload is - // triggered but fails since the file isn't found; further - // notifications (e.g. renamed a.tmp to a.tga) come within x [ms] and - // are ignored due to a time limit. - // therefore, we can only check for multiple reload requests a frame; - // to that purpose, an array is built and duplicates ignored. - const char* ext = path_extension(V_path); - // .. directory change notification; ignore because we get - // per-file notifications anyway. (note: assume no extension => - // it's a directory). - if(ext[0] == '\0') - return true; - // .. compiled XML files the engine writes out by the hundreds; - // skipping them is a big performance gain. - if(!strcasecmp(ext, "xmb")) - return true; - // .. temp files, usually created when an editor saves a file - // (delete, create temp, rename temp); no need to reload those. - if(!strcasecmp(ext, "tmp")) - return true; - // .. more than one notification for a file; only reload once. - // note: this doesn't suffer from the 'reloaded too early' - // problem described above; if there's more than one - // request in the array, the file has since been written. - for(uint i = 0; i < num_pending; i++) - { - if(!strcmp(pending_reloads[i], V_path)) - return true; - } - - return false; -} - - -// get directory change notifications, and reload all affected files. -// must be called regularly (e.g. once a frame). this is much simpler -// than asynchronous notifications: everything would need to be thread-safe. -LibError vfs_reload_changed_files() -{ - PathList pending_reloads; - - uint num_pending = 0; - // process only as many notifications as we have room for; the others - // will be handled next frame. it's not imagineable that they'll pile up. - while(num_pending < MAX_RELOADS_PER_FRAME) - { - // get next notification - char N_path[PATH_MAX]; - LibError ret = dir_get_changed_file(N_path); - if(ret == ERR::AGAIN) // none available; done. - break; - RETURN_ERR(ret); - - // convert to VFS path - char P_path[PATH_MAX]; - RETURN_ERR(file_make_full_portable_path(N_path, P_path)); - char* V_path = pending_reloads[num_pending]; - RETURN_ERR(mount_make_vfs_path(P_path, V_path)); - - if(can_ignore_reload(V_path, pending_reloads, num_pending)) - continue; - - // path has already been written to pending_reloads, - // so just mark it valid. - num_pending++; - } - - // rebuild VFS, in case a file that has been changed is currently - // mounted from an archive (reloading would just grab the unchanged - // version in the archive). the rebuild sees differing mtimes and - // always choses the loose file version. only do this once - // (instead of per reload request) because it's slow (> 1s)! - if(num_pending != 0) - RETURN_ERR(mount_rebuild()); - - // now actually reload all files in the array we built - for(uint i = 0; i < num_pending; i++) - RETURN_ERR(reload_without_rebuild(pending_reloads[i])); - - return INFO::OK; -} - - -//----------------------------------------------------------------------------- - -void vfs_display() -{ - tree_display(); -} - -static ModuleInitState initState; - -// make the VFS tree ready for use. must be called before all other -// functions below, barring explicit mentions to the contrary. -// -// rationale: initialization could be done implicitly by calling this -// from all VFS APIs. we refrain from that and require the user to -// call this because a central point of initialization (file_set_root_dir) -// is necessary anyway and this way is simpler/easier to maintain. -void vfs_init() -{ - if(!ModuleShouldInitialize(&initState)) - return; - - h_mgr_init(); - - stats_vfs_init_start(); - mount_init(); - stats_vfs_init_finish(); -} - -void vfs_shutdown() -{ - if(!ModuleShouldShutdown(&initState)) - return; - - trace_shutdown(); - mount_shutdown(); - - h_mgr_shutdown(); -} diff --git a/source/lib/res/file/vfs.h b/source/lib/res/file/vfs.h deleted file mode 100644 index eda13be6b3..0000000000 --- a/source/lib/res/file/vfs.h +++ /dev/null @@ -1,458 +0,0 @@ -/** - * ========================================================================= - * File : vfs.h - * Project : 0 A.D. - * Description : Virtual File System API - allows transparent access to - * : files in archives and modding via multiple mount points. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -/* - -[KEEP IN SYNC WITH WIKI!] - -Introduction ------------- - -The VFS (Virtual File System) is a layer between the application and -file.cpp's API. Its main purpose is to decrease the cost of file access; -also provided for are "hotloading" and "modding" via overriding files -(explained below). -The interface is almost identical to that of file.cpp, except that -it works with Handles for safety (see h_mgr.h). - - -File Access Cost ----------------- - -Games typically encompass thousands of files. Such heavy loads expose -2 problems with current file systems: -- wasted disk space. An average of half a cluster (>= 1 sector, typically - 512 bytes) is lost per file due to internal fragmentation. -- lengthy file open times. Permissions checks and overhead added by - antivirus scanners combine to make these slow. Additionally, files are - typically not arranged in order of access, which induces costly - disk seeks. - -The solution is to put all files in archives: internal fragmentation is -eliminated since they are packed end-to-end; open is much faster; -seeks are avoided by arranging in order of access. For more information, -see 'Archive Details' below. - -Note that a good file system (Reiser3 comes close) could also deliver the -above. However, this code is available now on all platforms; there is -no disadvantage to using it and the other features remain. - - -Hotloading ----------- - -During development, artists and programmers typically follow a edit/ -see how it looks in-game/repeat methodology. Unfortunately, changes to a -file are not immediately noticed by the game; the usual workaround is to -restart the map (or worse, entire game) to make sure they are reloaded. -Since decreases in edit cycle time improve productivity, we want changes to -files to be picked up immediately. To that end, we support hotloading - -as soon as the OS reports changes, all Handle objects that ensued from that -file are reloaded. - -The VFS's part in this is registering "watches" that report changes to -any mounted real directory. Since the file notification backend -(currently SGI FAM and a Win32 port) cannot watch an entire directory tree, -we need to do so for every single directory. The VFS traverses each and -stores information anyway, so we do that here. - - -Modding -------- - -1) Motivation - -When users tweak game parameters or even create an entirely new game -principle with the same underlying engine, it is called modding. -As evidenced by the Counterstrike mod for Half-Life, this can greatly -prolong the life of a game. Additionally, since we started out as a -mod group, great value is placed on giving users all the tools to make -modding easy. - -2) Means - -The actual method of overriding game data is quite simple: a mod directory -is mounted into the file system with a higher priority than original data. -These files therefore temporarily (as long as the mod is active) replace the -originals. This allows multiple (non-overlapping!) mods to be active at the -same time and also makes switching between them easy. -The same mechanism is also used for patches to game data. - -3) Rationale - -Older games did not provide any support for modding other than -directly editing game data. Obviously this is risky and insufficient. -Requiring mods to provide a entire new copy of all game logic/scripts -would obviate support from the file system, but is too much work for the -modder (since all files would first have to be copied somewhere). -Allowing overriding individual files is much safer (since game data is -never touched) and easier (more fine-grained control for modders). - - -Patching --------- - -As mentioned above, patching is also done via mounting. -Alternatives would be to completely replace the game data archive -(infeasible due to size) or apply a binary patch (complicated and -brittle WRT versioning). We are therefore happy to use the -already existing mod mechanism. - -Note however that multiple patches do impact performance (despite -constant-time VFS path -> file location lookup) simply due to locality; -files are no longer arranged in order of access. Fortunately there is an -easy way to avoid this: simply run the archive builder script; all -patched files will be merged into the archive. However, be warned that -reverting to previous versions (e.g. to watch old replays) would no longer -be possible! This is because their changes have been 'baked into' the -main archive, whereas previously the patch could simply be deleted. - - -Mount Details -------------- - -"Mounting" is understood to mean populating a given VFS directory (the -"mount point") with the contents of e.g. a real directory or archive -(the "mounted object" - for a list of supported types, see enum MountType). - -It is important to note that the VFS is a full-fledged tree storing -information about each file, e.g. its last-modified time or actual location. -The advantage is that file open time does not increase with the number of -mounts, which is important because multiple patches and mods may be active. -This is in contrast to e.g. PhysicsFS, which just maintains a list of -mountings and scans it when opening each file. - -Each file object in the VFS tree stores its current location; there is no -way to access files of the same name but lower priority residing in other -mounted objects. For this reason, the entire VFS must be rebuilt (i.e. -repopulating all mount points) when a mounting is removed. Fortunately -this is rare and does not happen in-game; we optimize for the common case. - - -Archive Details ---------------- - -1) Rationale - -An open format (.zip) was chosen instead of a proprietary solution for the -following reasons: -- interoperability: anyone can view or add files without the need for - special tools, which is important for modding. -- less work: freely available decompression code (ZLib) eases implementation. -Disadvantages are efficiency (only adequate; an in-house format would offer -more potential for optimization) and lacking protection of data files. -Interoperability is a double-edged sword, since anyone can change critical -files or use game assets. However, obfuscating archive contents doesn't -solve anything, because the application needs to access them and a cracker -need only reverse-engineer that. Regardless, the application can call its -archives e.g. ".pk3" (as does Quake III) for minimal protection. - -2) Archive Builder - -Arranging archive contents in order of access was mentioned above. To that -end, the VFS can log all file open calls into a text file (one per line). -This is then processed by an archive builder script, which needs to -collect all files by VFS lookup rules, then add them to the archive in -the order specified in that file (all remaining files that weren't triggered -in the logging test run should be added thereafter). - -Note that the script need only be a simple frontend for e.g. infozip, and -that a plain user-created archive will work as well (advantage of using Zip); -this is just an optimization. - -3) Misc. Notes - -To ease development, files may additionally be stored in normal directories. -The VFS transparently provides access to the correct (newest) version. -This is to allow keeping data files in SCM - developers can get the latest -version without always having to update archives afterwards. - -One additional advantage of archives over loose files is that I/O throughput -is increased - since files are compressed, there is less to read from disk. -Decompression is free because it is done in parallel with IOs. - -*/ - -#ifndef INCLUDED_VFS -#define INCLUDED_VFS - -#include "../handle.h" // Handle def -#include "lib/posix/posix_filesystem.h" // struct stat -#include "file.h" // file open flags - -// upper bound on number of files; used as size of TNode pool and -// enables an optimization in the cache if it fits in 16 bits -// (each block stores a 16-bit ID instead of pointer to TNode). -// -1 allows for an "invalid/free" value. -// -// must be #define instead of const because we check whether it -// fits in 16-bits via #if. -#define VFS_MAX_FILES ((1u << 16) - 1) - -// make the VFS tree ready for use. must be called before all other -// functions below, barring explicit mentions to the contrary. -extern void vfs_init(); -extern void vfs_shutdown(void); - -// enable/disable logging each file open event - used by the archive builder. -// this should only be done when necessary for performance reasons and is -// typically triggered via command line param. safe to call before vfs_init. -extern void vfs_enable_file_listing(bool want_enabled); - -// write a representation of the VFS tree to stdout. -extern void vfs_display(void); - - -// -// paths -// - -// note: the VFS doesn't specify any path length restriction - -// internal filename storage is not fixed-length. -// for an an indication of how large fixed-size user buffers should be, -// use PATH_MAX. - -// VFS paths are of the form: "(dir/)*file?" -// in English: '/' as path separator; trailing '/' required for dir names; -// no leading '/', since "" is the root dir. - - -// -// mount -// - -enum VfsMountFlags -{ - // the directory being mounted (but not its subdirs! see impl) will be - // searched for archives, and their contents added. - // use only if necessary, since this is slow (we need to check if - // each file is an archive, which entails reading the header). - VFS_MOUNT_ARCHIVES = 1, - - // when mounting a directory, all directories beneath it are - // added recursively as well. - VFS_MOUNT_RECURSIVE = 2, - - // all real directories mounted during this operation will be watched - // for changes. this flag is provided to avoid watches in output-only - // directories, e.g. screenshots/ (only causes unnecessary overhead). - VFS_MOUNT_WATCH = 4, - - // anything mounted from here should be added to archive when - // building via vfs_optimizer. - VFS_MOUNT_ARCHIVABLE = 8 -}; - -// mount into the VFS at , -// which is created if it does not yet exist. -// files in that directory override the previous VFS contents if -// (ority) is not lower. -// all archives in are also mounted, in alphabetical order. -// -// flags determines extra actions to perform; see VfsMountFlags. -// -// P_real_dir = "." or "./" isn't allowed - see implementation for rationale. -extern LibError vfs_mount(const char* V_mount_point, const char* P_real_dir, uint flags = 0, uint pri = 0); - -// unmount a previously mounted item, and rebuild the VFS afterwards. -extern LibError vfs_unmount(const char* name); - -// set current "mod write directory" to P_target_dir, which must -// already have been mounted into the VFS. -// all files opened for writing with the FILE_WRITE_TO_TARGET flag set will -// be written into the appropriate subdirectory of this mount point. -// -// this allows e.g. the editor to write files that are already -// stored in archives, which are read-only. -extern LibError vfs_set_write_target(const char* P_target_dir); - - -// -// directory entry -// - -// open the directory for reading its entries via vfs_next_dirent. -// V_dir must end in '/' to indicate it's a directory path. -extern Handle vfs_dir_open(const char* V_dir_path); - -// close the handle to a directory. -// all DirEnt.name strings are now invalid. -extern LibError vfs_dir_close(Handle& hd); - -// retrieve the next (order is unspecified) dir entry matching . -// return 0 on success, ERR::DIR_END if no matching entry was found, -// or a negative error code on failure. -// filter values: -// - 0: anything; -// - "/": any subdirectory; -// - "/|": any subdirectory, or as below with ; -// - : any file whose name matches; ? and * wildcards are allowed. -// -// note that the directory entries are only scanned once; after the -// end is reached (-> ERR::DIR_END returned), no further entries can -// be retrieved, even if filter changes (which shouldn't happen - see impl). -// -// see also the definition of DirEnt in file.h. -// -// rationale: we do not sort directory entries alphabetically here. -// most callers don't need it and the overhead is considerable -// (we'd have to store all entries in a vector). it is left up to -// higher-level code such as VfsUtil. -extern LibError vfs_dir_next_ent(Handle hd, DirEnt* ent, const char* filter = 0); - - - -// called by EnumDirEnts for each entry in a directory (optionally those in -// its subdirectories as well), passing their complete path+name, the info -// that would be returned by vfs_next_dirent, and user-specified context. -// note: path and ent parameters are only valid during the callback. -typedef void (*DirEnumCB)(const char* path, const DirEnt* ent, uintptr_t cbData); - -enum DirEnumFlags -{ - VFS_DIR_RECURSIVE = 1 -}; - -// call for each entry matching (see vfs_next_dirent) in -// directory ; if flags & VFS_DIR_RECURSIVE, entries in -// subdirectories are also returned. -extern LibError vfs_dir_enum(const char* path, uint enum_flags, const char* filter, - DirEnumCB cb, uintptr_t cbData); - - -// -// file -// - -// return actual path to the specified file: -// "/fn" or "/fn". -extern LibError vfs_realpath(const char* fn, char* realpath); - -// does the specified file exist? return false on error. -// useful because a "file not found" warning is not raised, unlike vfs_stat. -extern bool vfs_exists(const char* fn); - -// get file status (size, mtime). output param is zeroed on error. -extern LibError vfs_stat(const char* fn, struct stat*); - -// return the size of an already opened file, or a negative error code. -extern ssize_t vfs_size(Handle hf); - -// open the file for synchronous or asynchronous IO. write access is -// requested via FILE_WRITE flag, and is not possible for files in archives. -// flags defined in file.h -extern Handle vfs_open(const char* fn, uint flags = 0); - -// close the handle to a file. -extern LibError vfs_close(Handle& h); - - -// -// asynchronous I/O -// - -// low-level file routines - no caching or alignment. - -// begin transferring bytes, starting at . get result -// with vfs_wait_read; when no longer needed, free via vfs_io_discard. -extern Handle vfs_io_issue(Handle hf, size_t size, u8* buf); - -// indicates if the given IO has completed. -// return value: 0 if pending, 1 if complete, < 0 on error. -extern int vfs_io_has_completed(Handle hio); - -// wait until the transfer completes, and return its buffer. -// output parameters are zeroed on error. -extern LibError vfs_io_wait(Handle hio, u8*& p, size_t& size); - -// finished with transfer - free its buffer (returned by vfs_wait_read). -extern LibError vfs_io_discard(Handle& hio); - - -// -// synchronous I/O -// - -// transfer the next bytes to/from the given file. -// (read or write access was chosen at file-open time). -// -// if non-NULL, is called for each block transferred, passing . -// it returns how much data was actually transferred, or a negative error -// code (in which case we abort the transfer and return that value). -// the callback mechanism is useful for user progress notification or -// processing data while waiting for the next I/O to complete -// (quasi-parallel, without the complexity of threads). -// -// p (value-return) indicates the buffer mode: -// - *p == 0: read into buffer we allocate; set *p. -// caller should mem_free it when no longer needed. -// - *p != 0: read into or write into the buffer *p. -// - p == 0: only read into temp buffers. useful if the callback -// is responsible for processing/copying the transferred blocks. -// since only temp buffers can be added to the cache, -// this is the preferred read method. -// -// return number of bytes transferred (see above), or a negative error code. -extern ssize_t vfs_io(Handle hf, size_t size, FileIOBuf* p, FileIOCB cb = 0, uintptr_t cbData = 0); - - -// convenience functions that replace vfs_open / vfs_io / vfs_close: - -// load the entire file into memory. -// p and size are filled with address/size of buffer (0 on failure). -// flags influences IO mode and is typically 0. -// when the file contents are no longer needed, call file_buf_free(buf). -extern LibError vfs_load(const char* V_fn, FileIOBuf& buf, size_t& size, -uint flags = 0, FileIOCB cb = 0, uintptr_t cbData = 0); - - -extern ssize_t vfs_store(const char* fn, const u8* p, size_t size, uint flags = 0); - - -// -// memory mapping -// - -// useful for files that are too large to be loaded into memory, -// or if only (non-sequential) portions of a file are needed at a time. -// -// this is of course only possible for uncompressed files - compressed files -// would have to be inflated sequentially, which defeats the point of mapping. - - -// map the entire (uncompressed!) file into memory. if currently -// already mapped, return the previous mapping (reference-counted). -// output parameters are zeroed on failure. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -extern LibError vfs_map(Handle hf, uint flags, u8*& p, size_t& size); - -// decrement the reference count for the mapping belonging to file . -// fail if there are no references; remove the mapping if the count reaches 0. -// -// the mapping will be removed (if still open) when its file is closed. -// however, map/unmap calls should still be paired so that the mapping -// may be removed when no longer needed. -extern LibError vfs_unmap(Handle hf); - - -// -// hotloading -// - -extern LibError vfs_reload(const char* fn); - -// this must be called from the main thread? (wdir_watch problem) -extern LibError vfs_reload_changed_files(void); - -#endif // #ifndef INCLUDED_VFS diff --git a/source/lib/res/file/vfs_mount.cpp b/source/lib/res/file/vfs_mount.cpp deleted file mode 100644 index 7acfb27190..0000000000 --- a/source/lib/res/file/vfs_mount.cpp +++ /dev/null @@ -1,904 +0,0 @@ -/** - * ========================================================================= - * File : vfs_mount.cpp - * Project : 0 A.D. - * Description : mounts files and archives into VFS; provides x_* API - * : that dispatches to file or archive implementation. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "vfs_mount.h" - -#include -#include -#include -#include -#include - -#include "lib/sysdep/dir_watch.h" -#include "lib/res/h_mgr.h" -#include "file_internal.h" - - -// we add/cancel directory watches from the VFS mount code for convenience - -// it iterates through all subdirectories anyway (*) and provides storage for -// a key to identify the watch (obviates separate TDir -> watch mapping). -// -// define this to strip out that code - removes .watch from struct TDir, -// and calls to res_watch_dir / res_cancel_watch. -// -// *: the add_watch code would need to iterate through subdirs and watch -// each one, because the monitor API (e.g. FAM) may only be able to -// watch single directories, instead of a whole subdirectory tree. -#define NO_DIR_WATCH - - -// location of a file: either archive or a real directory. -// not many instances => don't worry about efficiency. -struct Mount -{ - // mounting into this VFS directory; - // must end in '/' (unless if root td, i.e. "") - std::string V_mount_point; - - // real directory being mounted. - // if this Mount represents an archive, this is the real directory - // containing the Zip file (required so that this Mount is unmounted). - std::string P_name; - - Handle archive; - - uint pri; - - // see enum VfsMountFlags - uint flags; - - MountType type; - - Mount(const char* V_mount_point_, const char* P_name_, Handle archive_, uint flags_, uint pri_) - : V_mount_point(V_mount_point_), P_name(P_name_) - { - archive = archive_; - flags = flags_; - pri = pri_; - - if(archive > 0) - { - h_add_ref(archive); - type = MT_ARCHIVE; - } - else - type = MT_FILE; - } - - ~Mount() - { - if(archive > 0) // avoid h_mgr warning - archive_close(archive); - } - - Mount& operator=(const Mount& rhs) - { - V_mount_point = rhs.V_mount_point; - P_name = rhs.P_name; - archive = rhs.archive; - pri = rhs.pri; - flags = rhs.flags; - type = rhs.type; - - if(archive > 0) // avoid h_mgr warning - h_add_ref(archive); - - return *this; - } - - struct equal_to : public std::binary_function - { - bool operator()(const Mount& m, const char* P_name) const - { - return (m.P_name == P_name); - } - }; - -private: - Mount(); -}; - - - -char mount_get_type(const Mount* m) -{ - switch(m->type) - { - case MT_ARCHIVE: - return 'A'; - case MT_FILE: - return 'F'; - default: - return '?'; - } -} - - -Handle mount_get_archive(const Mount* m) -{ - return m->archive; -} - - -bool mount_is_archivable(const Mount* m) -{ - return (m->flags & VFS_MOUNT_ARCHIVES) != 0; -} - - -bool mount_should_replace(const Mount* m_old, const Mount* m_new, - size_t size_old, size_t size_new, time_t mtime_old, time_t mtime_new) -{ - // 1) "replace" if not yet associated with a Mount. - if(!m_old) - return true; - - // 2) keep old if new priority is lower. - if(m_new->pri < m_old->pri) - return false; - - // assume they're the same if size and last-modified time match. - // note: FAT timestamp only has 2 second resolution - const double mtime_diff = difftime(mtime_old, mtime_new); - const bool identical = (size_old == size_new) && - fabs(mtime_diff) <= 2.0; - - // 3) go with more efficient source (if files are identical) - // - // since priority is not less, we really ought to always go with m_new. - // however, there is one special case we handle for performance reasons: - // if the file contents are the same, prefer the more efficient source. - // note that priority doesn't automatically take care of this, - // especially if set incorrectly. - // - // note: see MountType for explanation of type > type2. - if(identical && m_old->type > m_new->type) - return false; - - // 4) don't replace "old" file if modified more recently than "new". - // (still provide for 2 sec. FAT tolerance - see above) - if(mtime_diff > 2.0) - return false; - - return true; -} - - -// given Mount and V_path, return its actual location (portable path). -// works for any type of path: file or directory. -LibError mount_realpath(const char* V_path, const Mount* m, char* P_real_path) -{ - const char* remove = m->V_mount_point.c_str(); - const char* replace = m->P_name.c_str(); // P_parent_path - CHECK_ERR(path_replace(P_real_path, V_path, remove, replace)); - - // if P_real_path ends with '/' (a remnant from V_path), strip - // it because that's not acceptable for portable paths. - const size_t P_len = strlen(P_real_path); - if(P_len != 0 && P_real_path[P_len-1] == '/') - P_real_path[P_len-1] = '\0'; - - return INFO::OK; -} - - -/////////////////////////////////////////////////////////////////////////////// -// -// populate the directory being mounted with files from real subdirectories -// and archives. -// -/////////////////////////////////////////////////////////////////////////////// - -static const Mount& add_mount(const char* V_mount_point, const char* P_real_path, Handle archive, - uint flags, uint pri); - -// passed through dirent_cb's afile_enum to afile_cb -struct ZipCBParams : boost::noncopyable -{ - // tree directory into which we are adding the archive's files - TDir* const td; - - // archive's location; assigned to all files added from here - const Mount* const m; - - // storage for directory lookup optimization (see below). - // held across one afile_enum's afile_cb calls. - const char* last_path; - TDir* last_td; - - ZipCBParams(TDir* dir_, const Mount* loc_) - : td(dir_), m(loc_) - { - last_path = 0; - last_td = 0; - } -}; - -// called by add_ent's afile_enum for each file in the archive. -// we get the full path, since that's what is stored in Zip archives. -// -// [total time 21ms, with ~2000 file's (includes add_file cost)] -static LibError afile_cb(const char* atom_fn, const struct stat* s, uintptr_t memento, uintptr_t user) -{ - CHECK_PATH(atom_fn); - - const char* name = path_name_only(atom_fn); - char path[PATH_MAX]; - path_dir_only(atom_fn, path); - const char* atom_path = file_make_unique_fn_copy(path); - - ZipCBParams* params = (ZipCBParams*)user; - TDir* td = params->td; - const Mount* m = params->m; - const char* last_path = params->last_path; - TDir* last_td = params->last_td; - - // into which directory should the file be inserted? - // naive approach: tree_lookup_dir the path (slow!) - // optimization: store the last file's path; if it's the same, - // use the directory we looked up last time (much faster!) - // .. same as last time - if(last_path == atom_path) - td = last_td; - // .. last != current: need to do lookup - else - { - // we have to create them if missing, since we can't rely on the - // archiver placing directories before subdirs or files that - // reference them (WinZip doesn't always). - // we also need to start at the mount point (td). - const uint flags = LF_CREATE_MISSING|LF_START_DIR; - CHECK_ERR(tree_lookup_dir(atom_path, &td, flags)); - - params->last_path = atom_path; - params->last_td = td; - } - - WARN_ERR(tree_add_file(td, name, m, s->st_size, s->st_mtime, memento)); - vfs_opt_notify_non_loose_file(atom_fn); - return INFO::CB_CONTINUE; -} - - -static bool archive_less(Handle hza1, Handle hza2) -{ - const char* fn1 = h_filename(hza1); - const char* fn2 = h_filename(hza2); - return strcmp(fn1, fn2) < 0; -} - -typedef std::vector Archives; -typedef Archives::const_iterator ArchiveCIt; - -// return value is INFO::OK iff archives != 0 and the file should not be -// added to VFS (e.g. because it is an archive). -static LibError enqueue_archive(const char* name, const char* P_archive_dir, Archives* archives) -{ - // caller doesn't want us to check if this is a Zip file. this is the - // case in all subdirectories of the mount point, since checking for all - // mounted files would be slow. see mount_dir_tree. - if(!archives) - return INFO::SKIPPED; - - // get complete path for archive_open. - // this doesn't (need to) work for subdirectories of the mounted td! - // we can't use mount_get_path because we don't have the VFS path. - char P_path[PATH_MAX]; - RETURN_ERR(path_append(P_path, P_archive_dir, name)); - - // just open the Zip file and see if it's valid. we don't bother - // checking the extension because archives won't necessarily be - // called .zip (e.g. Quake III .pk3). - Handle archive = archive_open(P_path); - // .. special case: is recognizable as a Zip file but is - // invalid and can't be opened. avoid adding it to - // archive list and/or VFS. - if(archive == ERR::CORRUPTED) - goto do_not_add_to_VFS_or_list; - RETURN_ERR(archive); - - archives->push_back(archive); - - // avoid also adding the archive file itself to VFS. - // (when caller sees INFO::OK, they skip the file) -do_not_add_to_VFS_or_list: - return INFO::OK; -} - -static LibError mount_archive(TDir* td, const Mount& m) -{ - ZipCBParams params(td, &m); - archive_enum(m.archive, afile_cb, (uintptr_t)¶ms); - return INFO::OK; -} - -static LibError mount_archives(TDir* td, Archives* archives, const Mount* mount) -{ - // VFS_MOUNT_ARCHIVES flag wasn't set, or no archives present - if(archives->empty()) - return INFO::OK; - - // load archives in alphabetical filename order to allow patches - std::sort(archives->begin(), archives->end(), archive_less); - - for(ArchiveCIt it = archives->begin(); it != archives->end(); ++it) - { - Handle hza = *it; - - // add this archive to the mount list (address is guaranteed to - // remain valid). - const Mount& m = add_mount(mount->V_mount_point.c_str(), mount->P_name.c_str(), hza, mount->flags, mount->pri); - - mount_archive(td, m); - } - - return INFO::OK; -} - - -//----------------------------------------------------------------------------- - -struct TDirAndPath -{ - TDir* td; - std::string path; - - TDirAndPath(TDir* d, const char* p) - : td(d), path(p) - { - } -}; - -typedef std::deque DirQueue; - - - -static LibError enqueue_dir(TDir* parent_td, const char* name, - const char* P_parent_path, DirQueue* dir_queue) -{ - // caller doesn't want us to enqueue subdirectories; bail. - if(!dir_queue) - return INFO::OK; - - // skip versioning system directories - this avoids cluttering the - // VFS with hundreds of irrelevant files. - // we don't do this for Zip files because it's harder (we'd have to - // strstr the entire path) and it is assumed the Zip file builder - // will take care of it. - if(!strcmp(name, "CVS") || !strcmp(name, ".svn")) - return INFO::OK; - - // prepend parent path to get complete pathname. - char P_path[PATH_MAX]; - CHECK_ERR(path_append(P_path, P_parent_path, name)); - - // create subdirectory.. - TDir* td; - CHECK_ERR(tree_add_dir(parent_td, name, &td)); - // .. and add it to the list of directories to visit. - dir_queue->push_back(TDirAndPath(td, P_path)); - return INFO::OK; -} - - - - -// called by TDir::addR's file_enum for each entry in a real directory. -// -// if called for a real directory, it is added to VFS. -// else if called for a loose file that is a valid archive (*), -// it is mounted (all of its files are added) -// else the file is added to VFS. -// -// * we only perform this check in the directory being mounted, -// i.e. passed in by tree_add_dir. to determine if a file is an archive, -// we have to open it and read the header, which is slow. -// can't just check extension, because it might not be .zip (e.g. Quake3 .pk3). - -// -// td - tree td into which the dirent is to be added -// m - real td's location; assigned to all files added from this mounting -// archives - if the dirent is an archive, its Mount is added here. - -static LibError add_ent(TDir* td, DirEnt* ent, const char* P_parent_path, const Mount* m, - DirQueue* dir_queue, Archives* archives) -{ - const char* name = ent->name; - - // it's a directory entry. - if(DIRENT_IS_DIR(ent)) - return enqueue_dir(td, name, P_parent_path, dir_queue); - // else: it's a file (dir_next_ent discards everything except for - // file and subdirectory entries). - - if(enqueue_archive(name, m->P_name.c_str(), archives) == INFO::OK) - // return value indicates this file shouldn't be added to VFS - // (see enqueue_archive) - return INFO::OK; - - // notify archive builder that this file could be archived but - // currently isn't; if there are too many of these, archive will be - // rebuilt. - // note: check if archivable to exclude stuff like screenshots - // from counting towards the threshold. - if(mount_is_archivable(m)) - { - // prepend parent path to get complete pathname. - char V_path[PATH_MAX]; - CHECK_ERR(path_append(V_path, tfile_get_atom_fn((TFile*)td), name)); - const char* atom_fn = file_make_unique_fn_copy(V_path); - - vfs_opt_notify_loose_file(atom_fn); - } - - // it's a regular data file; add it to the directory. - return tree_add_file(td, name, m, ent->size, ent->mtime, 0); -} - - -// note: full path is needed for the dir watch. -static LibError populate_dir(TDir* td, const char* P_path, const Mount* m, - DirQueue* dir_queue, Archives* archives, uint flags) -{ - LibError err; - - RealDir* rd = tree_get_real_dir(td); - RETURN_ERR(mount_attach_real_dir(rd, P_path, m, flags)); - - DirIterator d; - RETURN_ERR(dir_open(P_path, &d)); - - DirEnt ent; - for(;;) - { - // don't RETURN_ERR since we need to close d. - err = dir_next_ent(&d, &ent); - if(err != INFO::OK) - break; - - err = add_ent(td, &ent, P_path, m, dir_queue, archives); - WARN_ERR(err); - } - - WARN_ERR(dir_close(&d)); - return INFO::OK; -} - - -// actually mount the specified entry. split out of vfs_mount, -// because when invalidating (reloading) the VFS, we need to -// be able to mount without changing the mount list. -// add all loose files and subdirectories (recursive). -// also mounts all archives in P_real_path and adds to archives. - -// add the contents of directory to this TDir, -// marking the files' locations as . flags: see VfsMountFlags. -// -// note: we are only able to add archives found in the root directory, -// due to dirent_cb implementation. that's ok - we don't want to check -// every single file to see if it's an archive (slow!). -static LibError mount_dir_tree(TDir* td_start, const Mount& m) -{ - LibError err = INFO::OK; - - // add_ent fills these queues with dirs/archives if the corresponding - // flags are set. - DirQueue dir_queue; // don't preallocate (not supported by TDirAndPath) - Archives archives; - archives.reserve(8); // preallocate for efficiency. - - // instead of propagating flags down to add_dir, prevent recursing - // and adding archives by setting the destination pointers to 0 (easier). - DirQueue* const pdir_queue = (m.flags & VFS_MOUNT_RECURSIVE)? &dir_queue : 0; - Archives* parchives = (m.flags & VFS_MOUNT_ARCHIVES)? &archives : 0; - - // kickoff (less efficient than goto, but c_str reference requires - // pop to come at end of loop => this is easiest) - dir_queue.push_back(TDirAndPath(td_start, m.P_name.c_str())); - - do - { - TDir* const td = dir_queue.front().td; - const char* P_path = dir_queue.front().path.c_str(); - - LibError ret = populate_dir(td, P_path, &m, pdir_queue, parchives, m.flags); - if(err == INFO::OK) - err = ret; - - // prevent searching for archives in subdirectories (slow!). this - // is currently required by the implementation anyway. - parchives = 0; - - dir_queue.pop_front(); - // pop at end of loop, because we hold a c_str() reference. - } - while(!dir_queue.empty()); - - // do not pass parchives because that has been set to 0! - mount_archives(td_start, &archives, &m); - - return INFO::OK; -} - - - - - - - -// the VFS stores the location (archive or directory) of each file; -// this allows multiple search paths without having to check each one -// when opening a file (slow). -// -// one Mount is allocated for each archive or directory mounted. -// therefore, files only /point/ to a (possibly shared) Mount. -// if a file's location changes (e.g. after mounting a higher-priority -// directory), the VFS entry will point to the new Mount; the priority -// of both locations is unchanged. -// -// allocate via mnt_create, passing the location. do not free! -// we keep track of all Locs allocated; they are freed at exit, -// and by mount_unmount_all (useful when rebuilding the VFS). -// this is much easier and safer than walking the VFS tree and -// freeing every location we find. - - - - - - - -/////////////////////////////////////////////////////////////////////////////// -// -// mount list (allows multiple mountings, e.g. for mods) -// -/////////////////////////////////////////////////////////////////////////////// - -// every mounting results in at least one Mount (and possibly more, e.g. -// if the directory contains Zip archives, which each get a Mount). -// -// requirements for container: -// - must not invalidate iterators after insertion! -// (TFile holds a pointer to the Mount from which it was added) -// - must store items in order of insertion -// xxx - -typedef std::list Mounts; -typedef Mounts::iterator MountIt; -static Mounts mounts; - - -static const Mount& add_mount(const char* V_mount_point, const char* P_real_path, Handle hza, - uint flags, uint pri) -{ - mounts.push_back(Mount(V_mount_point, P_real_path, hza, flags, pri)); - return mounts.back(); -} - - -// note: this is not a member function of Mount to avoid having to -// forward-declare mount_archive, mount_dir_tree. -static LibError remount(const Mount& m) -{ - TDir* td; - CHECK_ERR(tree_add_path(m.V_mount_point.c_str(), &m, &td)); - - switch(m.type) - { - case MT_ARCHIVE: - return mount_archive(td, m); - case MT_FILE: - return mount_dir_tree(td, m); - default: - WARN_RETURN(ERR::MOUNT_INVALID_TYPE); - } -} - -static void mount_unmount_all(void) -{ - mounts.clear(); -} - -static inline void remount_all() -{ - std::for_each(mounts.begin(), mounts.end(), remount); -} - - - - - -// mount into the VFS at , -// which is created if it does not yet exist. -// files in that directory override the previous VFS contents if -// (ority) is not lower. -// all archives in are also mounted, in alphabetical order. -// -// flags determines extra actions to perform; see VfsMountFlags. -// -// P_real_path = "." or "./" isn't allowed - see implementation for rationale. -LibError vfs_mount(const char* V_mount_point, const char* P_real_path, uint flags, uint pri) -{ - // make sure caller didn't forget the required trailing '/'. - debug_assert(VFS_PATH_IS_DIR(V_mount_point)); - - // make sure it's not already mounted, i.e. in mounts. - // also prevents mounting a parent directory of a previously mounted - // directory, or vice versa. example: mount $install/data and then - // $install/data/mods/official - mods/official would also be accessible - // from the first mount point - bad. - // no matter if it's an archive - still shouldn't be a "subpath". - for(MountIt it = mounts.begin(); it != mounts.end(); ++it) - { - if(path_is_subpath(P_real_path, it->P_name.c_str())) - WARN_RETURN(ERR::ALREADY_MOUNTED); - } - - // disallow "." because "./" isn't supported on Windows. - // it would also create a loophole for the parent td check above. - // "./" and "/." are caught by CHECK_PATH. - if(!strcmp(P_real_path, ".")) - WARN_RETURN(ERR::PATH_NON_CANONICAL); - - // (count this as "init" to obviate a separate timer) - stats_vfs_init_start(); - const Mount& m = add_mount(V_mount_point, P_real_path, 0, flags, pri); - LibError ret = remount(m); - stats_vfs_init_finish(); - return ret; -} - - -// rebuild the VFS, i.e. re-mount everything. open files are not affected. -// necessary after loose files or directories change, so that the VFS -// "notices" the changes and updates file locations. res calls this after -// dir_watch reports changes; can also be called from the console after a -// rebuild command. there is no provision for updating single VFS dirs - -// it's not worth the trouble. -LibError mount_rebuild() -{ - tree_clear(); - remount_all(); - return INFO::OK; -} - - -struct IsArchiveMount -{ - bool operator()(const Mount& m) const - { - return (m.type == MT_ARCHIVE); - } -}; - -// "backs off of" all archives - closes their files and allows them to -// be rewritten or deleted (required by archive builder). -// must call mount_rebuild when done with the rewrite/deletes, -// because this call leaves the VFS in limbo!! -// -// note: this works because archives are not "first-class" mount objects - -// they are added to the list whenever a real mount point's root directory -// contains archives. hence, we can just remove them from the list. -void mount_release_all_archives() -{ - mounts.remove_if(IsArchiveMount()); -} - - -// unmount a previously mounted item, and rebuild the VFS afterwards. -LibError vfs_unmount(const char* P_name) -{ - // this removes all Mounts ensuing from the given mounting. their dtors - // free all resources and there's no need to remove the files from - // VFS (nor is this possible), since it is completely rebuilt afterwards. - - MountIt begin = mounts.begin(), end = mounts.end(); - MountIt last = std::remove_if(begin, end, - std::bind2nd(Mount::equal_to(), P_name)); - // none were removed - need to complain so that the caller notices. - if(last == end) - WARN_RETURN(ERR::TNODE_NOT_FOUND); - // trim list and actually remove 'invalidated' entries. - mounts.erase(last, end); - - return mount_rebuild(); -} - - - - - - -// if or its ancestors are mounted, -// return a VFS path that accesses it. -// used when receiving paths from external code. -LibError mount_make_vfs_path(const char* P_path, char* V_path) -{ - debug_printf("mount_make_vfs_path %s %s\n", P_path, V_path); - for(MountIt it = mounts.begin(); it != mounts.end(); ++it) - { - const Mount& m = *it; - if(m.type != MT_FILE) - continue; - - const char* remove = m.P_name.c_str(); - const char* replace = m.V_mount_point.c_str(); - - if(path_replace(V_path, P_path, remove, replace) == INFO::OK) - return INFO::OK; - } - - WARN_RETURN(ERR::TNODE_NOT_FOUND); -} - - - - -static const Mount* write_target; - -// 2006-05-09 JW note: we are wanting to move XMB files into a separate -// folder tree (no longer interspersed with XML), so that deleting them is -// easier and dirs are less cluttered. -// -// if several mods are active, VFS would have several RealDirs mounted -// and could no longer automatically determine the write target. -// -// one solution would be to use this set_write_target support to choose the -// correct dir; however, XMB files may be generated whilst editing -// (which also requires a write_target to write files that are actually -// currently in archives), so we'd need to save/restore write_target. -// this would't be thread-safe => disaster. -// -// a vfs_store_to(filename, flags, N_actual_path) API would work, but it'd -// impose significant burden on users (finding the actual native dir), -// and be prone to abuse. additionally, it would be difficult to -// propagate N_actual_path to VFile_reload where it is needed; -// this would end up messy. -// -// instead, we'll write XMB files into VFS path "mods/$MODNAME/..", -// into which the realdir of the same name (located in some writable folder) -// is mounted; VFS therefore can write without problems. -// -// however, other code (e.g. archive builder) doesn't know about this -// trick - it only sees the flat VFS namespace, which doesn't -// include mods/$MODNAME (that is hidden). to solve this, we also mount -// any active mod's XMB dir into VFS root for read access. - - -// set current "mod write directory" to P_target_dir, which must -// already have been mounted into the VFS. -// all files opened for writing with the FILE_WRITE_TO_TARGET flag set will -// be written into the appropriate subdirectory of this mount point. -// -// this allows e.g. the editor to write files that are already -// stored in archives, which are read-only. -LibError vfs_set_write_target(const char* P_target_dir) -{ - for(MountIt it = mounts.begin(); it != mounts.end(); ++it) - { - const Mount& m = *it; - // skip if not a directory mounting - if(m.type != MT_FILE) - continue; - - // found it in list of mounted dirs - if(!strcmp(m.P_name.c_str(), P_target_dir)) - { - write_target = &m; - return INFO::OK; - } - } - - WARN_RETURN(ERR::NOT_MOUNTED); -} - - -// 'relocate' tf to the mounting established by vfs_set_write_target. -// call if is being opened with FILE_WRITE_TO_TARGET flag set. -LibError set_mount_to_write_target(TFile* tf) -{ - if(!write_target) - WARN_RETURN(ERR::NOT_MOUNTED); - - tfile_set_mount(tf, write_target); - - // invalidate the previous values. we don't need to be clever and - // set size to that of the file in the new write_target mount point. - // this is because we're only called for files that are being - // opened for writing, which will change these values anyway. - tree_update_file(tf, 0, 0); - - return INFO::OK; -} - - - - - -void mount_init() -{ - tree_init(); -} - - -void mount_shutdown() -{ - tree_shutdown(); - mount_unmount_all(); -} - - -static const Mount* MULTIPLE_MOUNTINGS = (const Mount*)-1; - -// RDTODO: when should this be called? TDir ctor can already set this. -LibError mount_attach_real_dir(RealDir* rd, const char* P_path, const Mount* m, uint flags) -{ - // more than one real dir mounted into VFS dir - // (=> can't create files for writing here) - if(rd->m) - { - // HACK: until RealDir reorg is done, we're going to have to deal with - // "attaching" to real dirs twice. don't mess up rd->m if m is the same. - if(rd->m != m) - rd->m = MULTIPLE_MOUNTINGS; - } - else - rd->m = m; - -#ifndef NO_DIR_WATCH - if(flags & VFS_MOUNT_WATCH) - { - // 'watch' this directory for changes to support hotloading. - // note: do not cause this function to return an error if - // something goes wrong - this step is basically optional. - char N_path[PATH_MAX]; - if(file_make_full_native_path(P_path, N_path) == INFO::OK) - (void)dir_add_watch(N_path, &rd->watch); - } -#endif - - return INFO::OK; -} - - -void mount_detach_real_dir(RealDir* rd) -{ - rd->m = 0; - -#ifndef NO_DIR_WATCH - if(rd->watch) // avoid dir_cancel_watch complaining - WARN_ERR(dir_cancel_watch(rd->watch)); - rd->watch = 0; -#endif -} - - -LibError mount_create_real_dir(const char* V_path, const Mount* m) -{ - debug_assert(VFS_PATH_IS_DIR(V_path)); - - if(!m || m == MULTIPLE_MOUNTINGS || m->type != MT_FILE) - return INFO::OK; - - char P_path[PATH_MAX]; - RETURN_ERR(mount_realpath(V_path, m, P_path)); - - return dir_create(P_path); -} - - -LibError mount_populate(TDir* td, RealDir* rd) -{ - UNUSED2(td); - UNUSED2(rd); - return INFO::OK; -} diff --git a/source/lib/res/file/vfs_mount.h b/source/lib/res/file/vfs_mount.h deleted file mode 100644 index 08003e4a0e..0000000000 --- a/source/lib/res/file/vfs_mount.h +++ /dev/null @@ -1,116 +0,0 @@ -/** - * ========================================================================= - * File : vfs_mount.h - * Project : 0 A.D. - * Description : mounts files and archives into VFS; provides x_* API - * : that dispatches to file or archive implementation. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_VFS_MOUNT -#define INCLUDED_VFS_MOUNT - -struct Mount; // must come before vfs_tree.h - -#include "lib/res/handle.h" -#include "file.h" -#include "archive/zip.h" -#include "vfs_tree.h" - - -namespace ERR -{ - const LibError ALREADY_MOUNTED = -110700; - const LibError NOT_MOUNTED = -110701; - const LibError MOUNT_INVALID_TYPE = -110702; -} - - -extern void mount_init(); -extern void mount_shutdown(); - - -// If it were possible to forward-declare enums in GCC, this one wouldn't be in -// the header. Don't use. -enum MountType -{ - // the relative ordering of values expresses efficiency of the sources - // (e.g. archives are faster than loose files). mount_should_replace - // makes use of this. - - MT_NONE = 0, - MT_FILE = 1, - MT_ARCHIVE = 2 -}; - - -// -// accessors that obviate the need to access Mount fields directly: -// - -extern bool mount_is_archivable(const Mount* m); - -extern bool mount_should_replace(const Mount* m_old, const Mount* m_new, - size_t size_old, size_t size_new, time_t mtime_old, time_t mtime_new); - -extern char mount_get_type(const Mount* m); - -extern Handle mount_get_archive(const Mount* m); - -// given Mount and V_path, return its actual location (portable path). -// works for any type of path: file or directory. -extern LibError mount_realpath(const char* V_path, const Mount* m, char* P_real_path); - - - -// stored by vfs_tree in TDir -struct RealDir -{ - // if exactly one real directory is mounted into this virtual dir, - // this points to its location. used to add files to VFS when writing. - // - // the Mount is actually in the mount info and is invalid when - // that's unmounted, but the VFS would then be rebuilt anyway. - // - // = 0 if no real dir mounted here; = -1 if more than one. - const Mount* m; -#ifndef NO_DIR_WATCH - intptr_t watch; -#endif -}; - -extern LibError mount_attach_real_dir(RealDir* rd, const char* P_path, const Mount* m, uint flags); -extern void mount_detach_real_dir(RealDir* rd); - -extern LibError mount_create_real_dir(const char* V_path, const Mount* m); - -extern LibError mount_populate(TDir* td, RealDir* rd); - - -// "backs off of" all archives - closes their files and allows them to -// be rewritten or deleted (required by archive builder). -// must call mount_rebuild when done with the rewrite/deletes, -// because this call leaves the VFS in limbo!! -extern void mount_release_all_archives(); - -// 'relocate' tf to the mounting established by vfs_set_write_target. -// call if is being opened with FILE_WRITE_TO_TARGET flag set. -extern LibError set_mount_to_write_target(TFile* tf); - - -// rebuild the VFS, i.e. re-mount everything. open files are not affected. -// necessary after loose files or directories change, so that the VFS -// "notices" the changes and updates file locations. res calls this after -// dir_watch reports changes; can also be called from the console after a -// rebuild command. there is no provision for updating single VFS dirs - -// it's not worth the trouble. -extern LibError mount_rebuild(); - -// if or its ancestors are mounted, -// return a VFS path that accesses it. -// used when receiving paths from external code. -extern LibError mount_make_vfs_path(const char* P_path, char* V_path); - -#endif // #ifndef INCLUDED_VFS_MOUNT diff --git a/source/lib/res/file/vfs_redirector.cpp b/source/lib/res/file/vfs_redirector.cpp deleted file mode 100644 index 329352e02c..0000000000 --- a/source/lib/res/file/vfs_redirector.cpp +++ /dev/null @@ -1,239 +0,0 @@ -/** - * ========================================================================= - * File : vfs_redirector.cpp - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "vfs_redirector.h" - -#include "lib/byte_order.h" // FOURCC -#include "file_internal.h" - - -static const u32 vtbl_magic = FOURCC('F','P','V','T'); - - -// HACK: these thunks and the vtbls are implemented here, -// although they belong in their respective provider's source file. -// this is currently necessary because vfs_mount doesn't yet -// abstract away the file provider (it's hardcoded for files+archives). - -LibError afile_open_vfs(const char* fn, uint flags, TFile* tf, - File* f) // out -{ - const uintptr_t memento = tfile_get_memento(tf); - const Mount* m = tfile_get_mount(tf); - const Handle ha = mount_get_archive(m); - return afile_open(ha, fn, memento, flags, f); -} - -LibError file_open_vfs(const char* V_path, uint flags, TFile* tf, - File* f) // out -{ - char N_path[PATH_MAX]; - const Mount* m = tfile_get_mount(tf); - RETURN_ERR(mount_realpath(V_path, m, N_path)); - RETURN_ERR(file_open(N_path, flags|FILE_DONT_SET_FN, f)); - // file_open didn't set fc.atom_fn due to FILE_DONT_SET_FN. - f->atom_fn = file_make_unique_fn_copy(V_path); - return INFO::OK; -} - -static const FileProvider_VTbl archive_vtbl = -{ - vtbl_magic, - 0,0,0, // not supported for archives ATM - afile_open_vfs, afile_close, afile_validate, - afile_io_issue, afile_io_has_completed, afile_io_wait, afile_io_discard, afile_io_validate, - afile_read, - afile_map, afile_unmap -}; - -static const FileProvider_VTbl file_vtbl = -{ - vtbl_magic, - dir_open, dir_next_ent, dir_close, - file_open_vfs, file_close, file_validate, - file_io_issue, file_io_has_completed, file_io_wait, file_io_discard, file_io_validate, - file_io, - file_map, file_unmap -}; - -// see FileProvider_VTbl decl for details on why this is so empty. -static const FileProvider_VTbl tree_vtbl = -{ - vtbl_magic, - tree_dir_open, tree_dir_next_ent, tree_dir_close, - 0, 0, 0, - 0, 0, 0, 0, 0, - 0, - 0, 0 -}; - - - -// rationale for not using virtual functions for file_open vs afile_open: -// it would spread out the implementation of each function and makes -// keeping them in sync harder. we will very rarely add new sources and -// all these functions are in one spot anyway. - - -static LibError vtbl_validate(const FileProvider_VTbl* vtbl) -{ - if(!vtbl) - WARN_RETURN(ERR::INVALID_PARAM); - if(vtbl->magic != vtbl_magic) - WARN_RETURN(ERR::CORRUPTED); - return INFO::OK; -} - -#define CHECK_VTBL(type) RETURN_ERR(vtbl_validate(type)) - - -// -// directory entry enumeration -// - -LibError xdir_open(const char* dir, DirIterator* di) -{ -// HACK: it is unclear ATM how to set this properly. assume tree_dir_* is -// the only user ATM. -di->type = &tree_vtbl; - CHECK_VTBL(di->type); - return di->type->dir_open(dir, di); -} - -LibError xdir_next_ent(DirIterator* di, DirEnt* ent) -{ - CHECK_VTBL(di->type); - return di->type->dir_next_ent(di, ent); -} - -LibError xdir_close(DirIterator* di) -{ - CHECK_VTBL(di->type); - return di->type->dir_close(di); -} - - -// -// file object -// - -bool xfile_is_open(const File* f) -{ - // not currently in use - if(f->type == 0) - return false; - - WARN_ERR(vtbl_validate(f->type)); - return true; -} - -LibError xfile_open(const char* V_path, uint flags, TFile* tf, File* f) -{ - // find out who is providing this file - const Mount* m = tfile_get_mount(tf); - debug_assert(m != 0); - -// HACK: see decl of vtbls. ideally vtbl would already be stored in -// Mount, but that's not implemented yet. -char c = mount_get_type(m); -const FileProvider_VTbl* vtbl = (c == 'F')? &file_vtbl : &archive_vtbl; - - CHECK_VTBL(vtbl); - RETURN_ERR(vtbl->file_open(V_path, flags, tf, f)); - - // success - // note: don't assign these unless we succeed to avoid the - // false impression that all is well. - f->type = vtbl; - return INFO::OK; -} - -LibError xfile_close(File* f) -{ - // we must not complain if the file is not open. this happens if - // attempting to open a nonexistent file: h_mgr automatically calls - // the dtor after reload fails. - // note: this takes care of checking the vtbl. - if(!xfile_is_open(f)) - return INFO::OK; - LibError ret = f->type->file_close(f); - f->type = 0; - return ret; -} - -LibError xfile_validate(const File* f) -{ - CHECK_VTBL(f->type); - return f->type->file_validate(f); -} - - -// -// IO -// - -LibError xfile_io_issue(File* f, off_t ofs, size_t size, u8* buf, FileIo* io) -{ - io->type = f->type; - CHECK_VTBL(io->type); - return io->type->io_issue(f, ofs, size, buf, io); -} - -int xfile_io_has_completed(FileIo* io) -{ - CHECK_VTBL(io->type); - return io->type->io_has_completed(io); -} - -LibError xfile_io_wait(FileIo* io, u8*& p, size_t& size) -{ - CHECK_VTBL(io->type); - return io->type->io_wait(io, p, size); -} - -LibError xfile_io_discard(FileIo* io) -{ - CHECK_VTBL(io->type); - return io->type->io_discard(io); -} - -LibError xfile_io_validate(const FileIo* io) -{ - CHECK_VTBL(io->type); - return io->type->io_validate(io); -} - -ssize_t xfile_io(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb, uintptr_t cbData) -{ - CHECK_VTBL(f->type); - // notes: - // - for archive file: vfs_open makes sure it wasn't opened for writing - // - normal file: let file_io alloc the buffer if the caller didn't - // (i.e. p = 0), because it knows about alignment / padding requirements - return f->type->io(f, ofs, size, pbuf, cb, cbData); -} - - -// -// file mapping -// - -LibError xfile_map(File* f, u8*& p, size_t& size) -{ - CHECK_VTBL(f->type); - return f->type->map(f, p, size); -} - -LibError xfile_unmap(File* f) -{ - CHECK_VTBL(f->type); - return f->type->unmap(f); -} diff --git a/source/lib/res/file/vfs_redirector.h b/source/lib/res/file/vfs_redirector.h deleted file mode 100644 index 2e5ae86145..0000000000 --- a/source/lib/res/file/vfs_redirector.h +++ /dev/null @@ -1,72 +0,0 @@ -/** - * ========================================================================= - * File : vfs_redirector.h - * Project : 0 A.D. - * Description : - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_VFS_REDIRECTOR -#define INCLUDED_VFS_REDIRECTOR - -#include "file.h" -struct FileIo; - -struct FileProvider_VTbl -{ - // FOURCC that is checked on each access to ensure this is a valid vtbl. - u32 magic; - // note: no need to store name of this provider for debugging purposes; - // that can be deduced from the function pointers below. - - // directory entry enumeration - // note: these don't really fit in with the other methods. - // they make sense for both the VFS tree as well as the concrete - // file providers underlying it. due to this overlap and to allow - // file.cpp's next_ent function to access dir_filtered_next_ent, - // it is included anyway. - LibError (*dir_open)(const char* dir, DirIterator* di); - LibError (*dir_next_ent)(DirIterator* di, DirEnt* ent); - LibError (*dir_close)(DirIterator* di); - - // file objects - LibError (*file_open)(const char* V_path, uint flags, TFile* tf, File* f); - LibError (*file_close)(File* f); - LibError (*file_validate)(const File* f); - - // IO - LibError (*io_issue)(File* f, off_t ofs, size_t size, u8* buf, FileIo* io); - int (*io_has_completed)(FileIo* io); - LibError (*io_wait)(FileIo* io, u8*& p, size_t& size); - LibError (*io_discard)(FileIo* io); - LibError (*io_validate)(const FileIo* io); - ssize_t (*io)(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb, uintptr_t cbData); - - // file mapping - LibError (*map)(File* f, u8*& p, size_t& size); - LibError (*unmap)(File* f); -}; - - -extern LibError xdir_open(const char* dir, DirIterator* di); -extern LibError xdir_next_ent(DirIterator* di, DirEnt* ent); -extern LibError xdir_close(DirIterator* di); - -extern bool xfile_is_open(const File* f); -extern LibError xfile_open(const char* V_path, uint flags, TFile* tf, File* f); -extern LibError xfile_close(File* f); -extern LibError xfile_validate(const File* f); - -extern LibError xfile_io_issue(File* f, off_t ofs, size_t size, u8* buf, FileIo* io); -extern int xfile_io_has_completed(FileIo* io); -extern LibError xfile_io_wait(FileIo* io, u8*& p, size_t& size); -extern LibError xfile_io_discard(FileIo* io); -extern LibError xfile_io_validate(const FileIo* io); -extern ssize_t xfile_io(File* f, off_t ofs, size_t size, FileIOBuf* pbuf, FileIOCB cb, uintptr_t cbData); - -extern LibError xfile_map(File* f, u8*& p, size_t& size); -extern LibError xfile_unmap(File* f); - -#endif // #ifndef INCLUDED_VFS_REDIRECTOR diff --git a/source/lib/res/file/vfs_tree.cpp b/source/lib/res/file/vfs_tree.cpp deleted file mode 100644 index d03a8bba86..0000000000 --- a/source/lib/res/file/vfs_tree.cpp +++ /dev/null @@ -1,771 +0,0 @@ -/** - * ========================================================================= - * File : vfs_tree.cpp - * Project : 0 A.D. - * Description : the actual 'filesystem' and its tree of directories. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "vfs_tree.h" - -#include -#include -#include -#include -#include - -#include "lib/posix/posix_pthread.h" -#include "lib/allocators.h" -#include "lib/adts.h" -#include "file_internal.h" - - -ERROR_ASSOCIATE(ERR::TNODE_NOT_FOUND, "File/directory not found", ENOENT); -ERROR_ASSOCIATE(ERR::TNODE_WRONG_TYPE, "Using a directory as file or vice versa", -1); - - -// Mount = location of a file in the tree. -// TFile = all information about a file stored in the tree. -// TDir = container holding TFile-s representing a dir. in the tree. - - -static void* node_alloc(); - - -// remembers which VFS file is the most recently modified. -static time_t most_recent_mtime; -static void set_most_recent_if_newer(time_t mtime) -{ - most_recent_mtime = std::max(most_recent_mtime, mtime); -} -time_t tree_most_recent_mtime() -{ - return most_recent_mtime; -} - -//----------------------------------------------------------------------------- -// locking -// these are exported to protect the vfs_mount list; apart from that, it is -// sufficient for VFS thread-safety to lock all of this module's APIs. - -static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; - -void tree_lock() -{ - pthread_mutex_lock(&mutex); -} - -void tree_unlock() -{ - pthread_mutex_unlock(&mutex); -} - - -//----------------------------------------------------------------------------- - -enum TNodeType -{ - NT_DIR, - NT_FILE -}; - -class TNode -{ -public: - TNodeType type; - // allocated and owned by vfs_mount - const Mount* m; - - // rationale: we store both entire path and name component. - // this increases size of VFS (2 pointers needed here) and - // filename storage, but allows getting path without having to - // iterate over all dir name components. - // we could retrieve name via strrchr(path, '/'), but that is slow. - const char* V_path; - // this is compared as a normal string (not pointer comparison), but - // the pointer passed must obviously remain valid, so it is - // usually an atom_fn. - const char* name; - - TNode(TNodeType type_, const char* V_path_, const char* name_, const Mount* m_) - : type(type_), V_path(V_path_), name(name_), m(m_) - { - } -}; - - -class TFile : public TNode -{ -public: - off_t size; - time_t mtime; - - uintptr_t memento; - - TFile(const char* V_path, const char* name, const Mount* m) - : TNode(NT_FILE, V_path, name, m) - { - size = 0; - mtime = 0; - memento = 0; - } -}; - - -template<> class DHT_Traits -{ -public: - static const size_t initial_entries = 32; - size_t hash(const char* key) const - { - return (size_t)fnv_lc_hash(key); - } - bool equal(const char* k1, const char* k2) const - { - // note: in theory, we could take advantage of the atom_fn - // mechanism to only compare string pointers. however, we're - // dealing with path *components* here. adding these as atoms would - // about double the memory used (to ~1 MB) and require a bit of - // care in the implementation of file_make_unique_path_copy - // (must not early-out before checking the hash table). - // - // given that path components are rather short, string comparisons - // are not expensive and we'll just go with that for simplicity. - if(!strcmp(k1, k2)) - return true; -#ifndef NDEBUG - // matched except for case: this can have 2 causes: - // - intentional. that would be legitimate but doesn't make much - // sense and isn't expected. - // - bug, e.g. discarding filename case in a filelist. - // this risks not being able to find the file (since VFS and - // possibly OS are case-sensitive) and wastes memory here. - // what we'll do is warn and treat as separate filename - // (least surprise). -// if(!strcasecmp(k1, k2)) -// debug_warn("filenames differ only in case: bug?"); -#endif - return false; - } - const char* get_key(TNode* t) const - { - return t->name; - } -}; -typedef DynHashTbl > TChildren; -typedef TChildren::iterator TChildrenIt; - -enum TDirFlags -{ - TD_POPULATED = 1 -}; - -class TDir : public TNode -{ - uint flags; // enum TDirFlags - - TChildren children; - -public: -RealDir rd; // HACK; removeme - - TDir(const char* V_path, const char* name, const Mount* m_) - : TNode(NT_DIR, V_path, name, 0), children() - { - flags = 0; - - rd.m = m_; - rd.watch = 0; - mount_create_real_dir(V_path, rd.m); - } - - TChildrenIt begin() const { return children.begin(); } - TChildrenIt end() const { return children.end(); } - - // non-const - caller may change e.g. rd.watch - RealDir& get_rd() { return rd; } - - void populate() - { - // the caller may potentially access this directory. - // make sure it has been populated with loose files/directories. - if(!(flags & TD_POPULATED)) - { - WARN_ERR(mount_populate(this, &rd)); - flags |= TD_POPULATED; - } - } - - TNode* find(const char* name) const - { - return children.find(name); - } - - // must not be called if already exists! use find() first or - // find_and_add instead. - LibError add(const char* name_tmp, TNodeType type, TNode** pnode, const Mount* m_override = 0) - { - // note: must be done before path_append for security - // (otherwise, '/' in wouldn't be caught) - RETURN_ERR(path_component_validate(name_tmp)); - - char V_new_path_tmp[PATH_MAX]; - const uint flags = (type == NT_DIR)? PATH_APPEND_SLASH : 0; - RETURN_ERR(path_append(V_new_path_tmp, V_path, name_tmp, flags)); - const char* V_new_path = file_make_unique_fn_copy(V_new_path_tmp); - const char* name = path_name_only(V_new_path); - // for directory nodes, V_path ends in slash, so name cannot be - // derived via path_last_component. instead, we have to make an - // atom_fn out of name_tmp. - // this effectively doubles the amount of directory path text, - // but it's not that bad. - if(type == NT_DIR) - name = file_make_unique_fn_copy(name_tmp); - - const Mount* m = rd.m; - if(m_override) - m = m_override; - - // note: if anything below fails, this mem remains allocated in the - // pool, but that "can't happen" and is OK because pool is big enough. - void* mem = node_alloc(); - if(!mem) - WARN_RETURN(ERR::NO_MEM); - TNode* node; -#include "lib/nommgr.h" - if(type == NT_FILE) - node = new(mem) TFile(V_new_path, name, m); - else - node = new(mem) TDir (V_new_path, name, m); -#include "lib/mmgr.h" - - children.insert(name, node); - - *pnode = node; - return INFO::OK; - } - - LibError find_and_add(const char* name, TNodeType type, TNode** pnode, const Mount* m = 0) - { - TNode* node = children.find(name); - if(node) - { - // wrong type (dir vs. file) - if(node->type != type) - WARN_RETURN(ERR::TNODE_WRONG_TYPE); - - *pnode = node; - return INFO::ALREADY_EXISTS; - } - - return add(name, type, pnode, m); - } - - - // empty this directory and all subdirectories; used when rebuilding VFS. - void clearR() - { - // recurse for all subdirs - // (preorder traversal - need to do this before clearing the list) - for(TChildrenIt it = children.begin(); it != children.end(); ++it) - { - TNode* node = *it; - if(node->type == NT_DIR) - { - ((TDir*)node)->clearR(); - ((TDir*)node)->~TDir(); - } - } - - // wipe out this directory - children.clear(); - - // the watch is restored when this directory is repopulated; we must - // remove it in case the real directory backing this one was deleted. - mount_detach_real_dir(&rd); - } -}; - - - - - - - -static Pool node_pool; - -static inline void node_init() -{ - const size_t el_size = std::max(sizeof(TDir), sizeof(TFile)); - (void)pool_create(&node_pool, VFS_MAX_FILES*el_size, el_size); -} - -static inline void node_shutdown() -{ - (void)pool_destroy(&node_pool); -} - -static void* node_alloc() -{ - return pool_alloc(&node_pool, 0); -} - -static inline void node_free_all() -{ - pool_free_all(&node_pool); -} - - -////////////////////////////////////////////////////////////////////////////// -// -// -// -////////////////////////////////////////////////////////////////////////////// - -static void displayR(TDir* td, int indent_level) -{ - const char indent[] = " "; - - TChildrenIt it; - - // list all files in this dir - for(it = td->begin(); it != td->end(); ++it) - { - TNode* node = (*it); - if(node->type != NT_FILE) - continue; - const char* name = node->name; - - TFile& file = *((TFile*)node); - char file_location = mount_get_type(file.m); - char* timestamp = ctime(&file.mtime); - timestamp[24] = '\0'; // remove '\n' - const off_t size = file.size; - - // build format string: tell it how long the filename may be, - // so that it takes up all space before file info column. - char fmt[25]; - int chars = 80 - indent_level*(sizeof(indent)-1); - sprintf(fmt, "%%-%d.%ds (%%c; %%6d; %%s)\n", chars, chars); - - for(int i = 0; i < indent_level; i++) - printf(indent); - printf(fmt, name, file_location, size, timestamp); - } - - // recurse over all subdirs - for(it = td->begin(); it != td->end(); ++it) - { - TNode* node = (*it); - if(node->type != NT_DIR) - continue; - const char* subdir_name = node->name; - - // write subdir's name - // note: do it now, instead of in recursive call so that: - // - we don't have to pass dir_name parameter; - // - the VFS root node isn't displayed. - for(int i = 0; i < indent_level; i++) - printf(indent); - printf("[%s/]\n", subdir_name); - - TDir* subdir = ((TDir*)node); - displayR(subdir, indent_level+1); - } -} - - -struct LookupCbParams : boost::noncopyable -{ - const bool create_missing; - TDir* td; // current dir; assigned from node - TNode* node; // latest node returned (dir or file) - LookupCbParams(uint flags, TDir* td_) - : create_missing((flags & LF_CREATE_MISSING) != 0), td(td_) - { - // init in case lookup's is "". - // this works because TDir is derived from TNode. - node = (TNode*)td; - } -}; - -static LibError lookup_cb(const char* component, bool is_dir, uintptr_t cbData) -{ - LookupCbParams* p = (LookupCbParams*)cbData; - const TNodeType type = is_dir? NT_DIR : NT_FILE; - - p->td->populate(); - - p->node = p->td->find(component); - if(!p->node) - { - if(p->create_missing) - RETURN_ERR(p->td->add(component, type, &p->node)); - else - // complaining is left to callers; vfs_exists must be - // able to fail quietly. - return ERR::TNODE_NOT_FOUND; // NOWARN - } - if(p->node->type != type) - WARN_RETURN(ERR::TNODE_WRONG_TYPE); - - if(is_dir) - p->td = (TDir*)p->node; - - return INFO::CB_CONTINUE; -} - -static LibError lookup(TDir* td, const char* path, uint flags, TNode** pnode) -{ - // no undefined bits set - debug_assert( (flags & ~(LF_CREATE_MISSING|LF_START_DIR)) == 0 ); - - LookupCbParams p(flags, td); - RETURN_ERR(path_foreach_component(path, lookup_cb, (uintptr_t)&p)); - - // success. - *pnode = p.node; - return INFO::OK; -} - - -////////////////////////////////////////////////////////////////////////////// -// -// -// -////////////////////////////////////////////////////////////////////////////// - -// this is a pointer to node_alloc-ed memory instead of a static TDir for -// 2 reasons: -// - no NLSO shutdown order issues; validity is well defined -// (namely between tree_init and tree_shutdown) -// - bonus: tree_init can use it when checking if called twice. -// -// this means we'll have to be extremely careful during tree_clear -// whether its memory remains valid. -static TDir* tree_root; - -// make tree_root valid. -static void tree_root_init() -{ - // must not be called more than once without intervening tree_shutdown. - debug_assert(!tree_root); - -#include "lib/nommgr.h" // placement new - void* mem = node_alloc(); - if(mem) - tree_root = new(mem) TDir("", "", 0); -#include "lib/mmgr.h" -} - -// destroy the tree root node and free any extra memory held by it. -// note that its node memory still remains allocated. -static void tree_root_shutdown() -{ - // must not be called without previous tree_root_init. - debug_assert(tree_root); - - // this frees the root node's hash table, which would otherwise leak. - tree_root->~TDir(); - tree_root = 0; -} - - -// establish a root node and prepare node_allocator for use. -// -// rationale: calling this from every tree_add* is ugly, so require -// manual init. -void tree_init() -{ - node_init(); - tree_root_init(); -} - - -// empty all directories and free their memory. -// however, node_allocator's DynArray still remains initialized and -// the root directory is usable (albeit empty). -// use when remounting. -void tree_clear() -{ - tree_root->clearR(); - tree_root_shutdown(); // must come before tree_root_init - - node_free_all(); - - // note: this is necessary because node_free_all - // pulls the rug out from under tree_root. - tree_root_init(); -} - - -// shut down entirely; destroys node_allocator. any further use after this -// requires another tree_init. -void tree_shutdown() -{ - // note: can't use tree_clear because that restores a root node - // ready for use, which allocates memory. - - // wipe out all dirs (including root node), thus - // freeing memory they hold. - tree_root->clearR(); - - tree_root_shutdown(); - - // free memory underlying the nodes themselves. - node_shutdown(); -} - - -// write a representation of the VFS tree to stdout. -void tree_display() -{ - displayR(tree_root, 0); -} - - -LibError tree_add_file(TDir* td, const char* name, - const Mount* m, off_t size, time_t mtime, uintptr_t memento) -{ - TNode* node; - LibError ret = td->find_and_add(name, NT_FILE, &node); - RETURN_ERR(ret); - if(ret == INFO::ALREADY_EXISTS) - { - TFile* tf = (TFile*)node; - if(!mount_should_replace(tf->m, m, tf->size, size, tf->mtime, mtime)) - return INFO::ALREADY_EXISTS; - - stats_vfs_file_remove(tf->size); - } - - TFile* tf = (TFile*)node; - tf->m = m; - tf->mtime = mtime; - tf->size = size; - tf->memento = memento; - stats_vfs_file_add(size); - - set_most_recent_if_newer(mtime); - return INFO::OK; -} - - -LibError tree_add_dir(TDir* td, const char* name, TDir** ptd) -{ - TNode* node; - RETURN_ERR(td->find_and_add(name, NT_DIR, &node)); - *ptd = (TDir*)node; - return INFO::OK; -} - - - -LibError tree_lookup_dir(const char* V_path, TDir** ptd, uint flags) -{ - // path is not a directory; TDir::lookup might return a file node - if(!VFS_PATH_IS_DIR(V_path)) - WARN_RETURN(ERR::TNODE_WRONG_TYPE); - - TDir* td = (flags & LF_START_DIR)? *ptd : tree_root; - TNode* node = NULL; - CHECK_ERR(lookup(td, V_path, flags, &node)); - // directories should exist, so warn if this fails - *ptd = (TDir*)node; - return INFO::OK; -} - - -LibError tree_lookup(const char* V_path, TFile** pfile, uint flags) -{ - // path is not a file; TDir::lookup might return a directory node - if(VFS_PATH_IS_DIR(V_path)) - WARN_RETURN(ERR::TNODE_WRONG_TYPE); - - TNode* node = NULL; - LibError ret = lookup(tree_root, V_path, flags, &node); - RETURN_ERR(ret); - *pfile = (TFile*)node; - return INFO::OK; -} - - -struct AddPathCbParams : boost::noncopyable -{ - const Mount* const m; - TDir* td; - AddPathCbParams(const Mount* m_) - : m(m_), td(tree_root) {} -}; - -static LibError add_path_cb(const char* component, bool is_dir, uintptr_t cbData) -{ - AddPathCbParams* p = (AddPathCbParams*)cbData; - - // should only be called for directory paths, so complain if not dir. - if(!is_dir) - WARN_RETURN(ERR::TNODE_WRONG_TYPE); - - TNode* node; - RETURN_ERR(p->td->find_and_add(component, NT_DIR, &node, p->m)); - - p->td = (TDir*)node; - return INFO::CB_CONTINUE; -} - -// iterate over all components in V_dir_path (must reference a directory, -// i.e. end in slash). for any that are missing, add them with the -// specified mount point. this is useful for mounting directories. -// -// passes back the last directory encountered. -LibError tree_add_path(const char* V_dir_path, const Mount* m, TDir** ptd) -{ - debug_assert(VFS_PATH_IS_DIR(V_dir_path)); - - AddPathCbParams p(m); - RETURN_ERR(path_foreach_component(V_dir_path, add_path_cb, (uintptr_t)&p)); - *ptd = p.td; - return INFO::OK; -} - - -////////////////////////////////////////////////////////////////////////////// - - -// rationale: see DirIterator definition in file.h. -struct TreeDirIterator -{ - TChildren::iterator it; - - // cache end() to avoid needless copies - TChildren::iterator end; - - // the directory we're iterating over; this is used to lock/unlock it, - // i.e. prevent modifications that would invalidate the iterator. - TDir* td; -}; - -cassert(sizeof(TreeDirIterator) <= DIR_ITERATOR_OPAQUE_SIZE); - - -LibError tree_dir_open(const char* V_dir_path, DirIterator* di) -{ - debug_assert(VFS_PATH_IS_DIR(V_dir_path)); - - TreeDirIterator* tdi = (TreeDirIterator*)di->opaque; - - TDir* td; - CHECK_ERR(tree_lookup_dir(V_dir_path, &td)); - - // we need to prevent modifications to this directory while an iterator is - // active, otherwise entries may be skipped or no longer valid addresses - // accessed. blocking other threads is much more convenient for callers - // than having to check for ERR::AGAIN on every call, so we use a mutex - // instead of a simple refcount. we don't bother with fine-grained locking - // (e.g. per directory or read/write locks) because it would result in - // more overhead (we have hundreds of directories) and is unnecessary. - tree_lock(); - - tdi->it = td->begin(); - tdi->end = td->end(); - tdi->td = td; - return INFO::OK; -} - - -LibError tree_dir_next_ent(DirIterator* di, DirEnt* ent) -{ - TreeDirIterator* tdi = (TreeDirIterator*)di->opaque; - - if(tdi->it == tdi->end) - return ERR::DIR_END; // NOWARN - - const TNode* node = *(tdi->it++); - ent->name = node->name; - - // set size and mtime fields depending on node type: - switch(node->type) - { - case NT_DIR: - ent->size = -1; - ent->mtime = 0; // not currently supported for dirs - ent->tf = 0; - break; - case NT_FILE: - { - TFile* tf = (TFile*)node; - ent->size = tf->size; - ent->mtime = tf->mtime; - ent->tf = tf; - break; - } - default: - debug_warn("invalid TNode type"); - } - - return INFO::OK; -} - - -LibError tree_dir_close(DirIterator* UNUSED(d)) -{ - tree_unlock(); - - // no further cleanup needed. we could zero out d but that might - // hide bugs; the iterator is safe (will not go beyond end) anyway. - return INFO::OK; -} - - -//----------------------------------------------------------------------------- -// get/set - -const Mount* tfile_get_mount(const TFile* tf) -{ - return tf->m; -} - -uintptr_t tfile_get_memento(const TFile* tf) -{ - return tf->memento; -} - -const char* tfile_get_atom_fn(const TFile* tf) -{ - return ((TNode*)tf)->V_path; -} - - - -void tfile_set_mount(TFile* tf, const Mount* m) -{ - tf->m = m; -} - -void tree_update_file(TFile* tf, off_t size, time_t mtime) -{ - tf->size = size; - tf->mtime = mtime; -} - - -// get file status (mode, size, mtime). output param is undefined on error. -LibError tree_stat(const TFile* tf, struct stat* s) -{ - // all stat members currently supported are stored in TFile, so we - // can return them directly without having to call file|zip_stat. - s->st_mode = S_IFREG; - s->st_size = tf->size; - s->st_mtime = tf->mtime; - - return INFO::OK; -} - - -RealDir* tree_get_real_dir(TDir* td) -{ - return &td->get_rd(); -} diff --git a/source/lib/res/file/vfs_tree.h b/source/lib/res/file/vfs_tree.h deleted file mode 100644 index a346429828..0000000000 --- a/source/lib/res/file/vfs_tree.h +++ /dev/null @@ -1,124 +0,0 @@ -/** - * ========================================================================= - * File : vfs_tree.h - * Project : 0 A.D. - * Description : the actual 'filesystem' and its tree of directories. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_VFS_TREE -#define INCLUDED_VFS_TREE - -class TFile; // must come before vfs_mount.h -class TDir; - -#include "file.h" // DirEnt -#include "vfs_mount.h" // Mount - - -namespace ERR -{ - const LibError TNODE_NOT_FOUND = -110600; - // attemped to treat a file as directory or vice versa. - const LibError TNODE_WRONG_TYPE = -110601; -} - -// establish a root node and prepare node_allocator for use. -extern void tree_init(); - -// shut down entirely; destroys node_allocator. any further use after this -// requires another tree_init. -extern void tree_shutdown(); - -extern void tree_display(); - -// empty all directories and free their memory. -// however, node_allocator's DynArray still remains initialized and -// the root directory is usable (albeit empty). -// use when remounting. -extern void tree_clear(); - -extern time_t tree_most_recent_mtime(); - -// attempt to add to , storing its attributes. -// overrides previously existing files of the same name if the new one -// is more important, determined via priority and file location. -// called by zip_cb and add_ent. -// -// note: if "priority" is the same, replace! -// this makes sure mods/patches etc. actually replace files. -extern LibError tree_add_file(TDir* td, const char* name, const Mount* m, - off_t size, time_t mtime, uintptr_t memento); - -extern LibError tree_add_dir(TDir* dir, const char* name, TDir** ptd); - - - -enum TreeLookupFlags -{ - LF_CREATE_MISSING = 1, - LF_START_DIR = 2 -}; - -// pass back file information for (relative to VFS root). -// -// if & LF_CREATE_MISSING, the file is added to VFS unless -// a higher-priority file of the same name already exists -// (used by VFile_reload when opening for writing). -// -// output params are only valid if INFO::OK is returned. -extern LibError tree_lookup(const char* path, TFile** ptf, uint flags = 0); - -// starting at VFS root, traverse and pass back information -// for its last directory component. -// -// if & LF_CREATE_MISSING, all missing subdirectory components are -// added to the VFS. -// if & LF_START_DIR, traversal starts at *ptd -// (used when looking up paths relative to a mount point). -// -// can be to a file or dir (in which case it must end in '/', -// to make sure the last component is treated as a directory). -// -// output params are only valid if INFO::OK is returned. -extern LibError tree_lookup_dir(const char* V_path, TDir** ptd, uint flags = 0); - - -// iterate over all components in V_dir_path (must reference a directory, -// i.e. end in slash). for any that are missing, add them with the -// specified mount point. this is useful for mounting directories. -// -// passes back the last directory encountered. -extern LibError tree_add_path(const char* V_dir_path, const Mount* m, TDir** ptd); - - -extern LibError tree_dir_open(const char* V_dir_path, DirIterator* di); -extern LibError tree_dir_next_ent(DirIterator* di, DirEnt* ent); -extern LibError tree_dir_close(DirIterator* di); - - -// given a file that is stored on disk and its VFS path, -// return its OS path (for use with file.cpp). -// used by vfs_realpath and VFile_reopen. -extern LibError tree_realpath(TFile* tf, const char* V_path, char* P_real_path); - -extern LibError tree_stat(const TFile* tf, struct stat* s); - -extern const Mount* tfile_get_mount(const TFile* tf); -extern uintptr_t tfile_get_memento(const TFile* tf); -extern const char* tfile_get_atom_fn(const TFile* tf); - -extern void tfile_set_mount(TFile* tf, const Mount* m); -extern void tree_update_file(TFile* tf, off_t size, time_t mtime); - -struct RealDir; -extern RealDir* tree_get_real_dir(TDir* td); - - -// for use in vfs_mount -extern void tree_lock(); -extern void tree_unlock(); - -#endif // #ifndef INCLUDED_VFS_TREE diff --git a/source/lib/res/graphics/cursor.cpp b/source/lib/res/graphics/cursor.cpp index a9c52ef923..8f90b93bd8 100644 --- a/source/lib/res/graphics/cursor.cpp +++ b/source/lib/res/graphics/cursor.cpp @@ -11,6 +11,10 @@ #include "precompiled.h" #include "cursor.h" +#include "../h_mgr.h" +#include "lib/file/vfs/vfs.h" +extern PIVFS g_VFS; + #include #include @@ -25,7 +29,6 @@ #include "lib/ogl.h" #include "lib/sysdep/cursor.h" -#include "lib/res/res.h" #include "ogl_tex.h" /* @@ -36,16 +39,14 @@ Shouldn't be called when both hardware/software cursor fails (i.e. invalid cursor file given) - in that case we'd rather use the default cursor. */ -static void *load_empty_sys_cursor() +static void* load_empty_sys_cursor() { - void *sys_cursor = 0; - + void* sys_cursor = 0; if(sys_cursor_create_empty(&sys_cursor) < 0) { - debug_warn("sys_cursor_create_empty failed"); - return NULL; + debug_assert(0); + return 0; } - return sys_cursor; } @@ -76,13 +77,13 @@ static void* load_sys_cursor(const char* filename, int hx, int hy) if(sys_cursor_create(t.w, t.h, bgra_img, hx, hy, &sys_cursor) < 0) goto fail; - (void)tex_free(&t); + tex_free(&t); return sys_cursor; } fail: - debug_warn("failed"); - (void)tex_free(&t); + debug_assert(0); + tex_free(&t); return 0; #endif } @@ -186,11 +187,10 @@ static LibError Cursor_reload(Cursor* c, const char* name, Handle) uint hotspotx = 0, hotspoty = 0; { snprintf(filename, ARRAY_SIZE(filename), "art/textures/cursors/%s.txt", name); - FileIOBuf buf; size_t size; - RETURN_ERR(vfs_load(filename, buf, size)); - std::stringstream s(std::string((const char*)buf, size)); + shared_ptr buf; size_t size; + RETURN_ERR(g_VFS->LoadFile(filename, buf, size)); + std::stringstream s(std::string((const char*)buf.get(), size)); s >> hotspotx >> hotspoty; - (void)file_buf_free(buf); } // load actual cursor diff --git a/source/lib/res/graphics/ogl_shader.cpp b/source/lib/res/graphics/ogl_shader.cpp index d2d04ec9da..68a003009e 100644 --- a/source/lib/res/graphics/ogl_shader.cpp +++ b/source/lib/res/graphics/ogl_shader.cpp @@ -9,12 +9,16 @@ #include "precompiled.h" #include "ogl_shader.h" -#include "lib/res/res.h" #include "lib/ogl.h" +#include "ps/CStr.h" #include "ps/CLogger.h" #include "ps/XML/Xeromyces.h" +#include "lib/res/h_mgr.h" +#include "lib/file/vfs/vfs.h" +extern PIVFS g_VFS; + #define LOG_CATEGORY "shaders" @@ -93,12 +97,8 @@ static LibError Ogl_Shader_reload(Ogl_Shader* shdr, const char* filename, Handle if (shdr->id) return INFO::OK; - FileIOBuf file; - size_t file_size; - GLint log_length; - GLint compile_success; - - RETURN_ERR(vfs_load(filename, file, file_size)); + shared_ptr file; size_t file_size; + RETURN_ERR(g_VFS->LoadFile(filename, file, file_size)); ogl_WarnIfError(); @@ -110,16 +110,17 @@ static LibError Ogl_Shader_reload(Ogl_Shader* shdr, const char* filename, Handle // bad code. ogl_WarnIfError(); - err = ERR::SHDR_CREATE; - goto fail_fileloaded; + WARN_RETURN(ERR::SHDR_CREATE); } { - const GLchar* strings[] = { (const GLchar*)file }; + const GLchar* strings[] = { (const GLchar*)file.get() }; pglShaderSourceARB(shdr->id, 1, strings, (const GLint*)&file_size); pglCompileShaderARB(shdr->id); } + GLint log_length; + GLint compile_success; pglGetObjectParameterivARB(shdr->id, GL_OBJECT_COMPILE_STATUS_ARB, &compile_success); pglGetObjectParameterivARB(shdr->id, GL_OBJECT_INFO_LOG_LENGTH_ARB, &log_length); if (log_length > 1) @@ -154,14 +155,11 @@ static LibError Ogl_Shader_reload(Ogl_Shader* shdr, const char* filename, Handle goto fail_shadercreated; } - (void)file_buf_free(file); return INFO::OK; fail_shadercreated: pglDeleteObjectARB(shdr->id); shdr->id = 0; -fail_fileloaded: - (void)file_buf_free(file); return err; } diff --git a/source/lib/res/graphics/ogl_tex.cpp b/source/lib/res/graphics/ogl_tex.cpp index ed6fc5aae8..218ddc5cdd 100644 --- a/source/lib/res/graphics/ogl_tex.cpp +++ b/source/lib/res/graphics/ogl_tex.cpp @@ -16,9 +16,11 @@ #include "lib/ogl.h" #include "lib/bits.h" #include "lib/sysdep/gfx.h" -#include "lib/res/res.h" -#include "tex.h" +#include "lib/tex/tex.h" +#include "../h_mgr.h" +#include "lib/file/vfs/vfs.h" +extern PIVFS g_VFS; //---------------------------------------------------------------------------- @@ -108,7 +110,7 @@ static GLint choose_fmt(uint bpp, uint flags) case 5: return GL_COMPRESSED_RGBA_S3TC_DXT5_EXT; default: - debug_warn("invalid DXT value"); + debug_assert(0); // invalid DXT value return 0; } } @@ -128,7 +130,7 @@ static GLint choose_fmt(uint bpp, uint flags) debug_assert(alpha); return bgr? GL_BGRA : GL_RGBA; default: - debug_warn("invalid bpp"); + debug_assert(0); // invalid bpp return 0; } @@ -221,7 +223,7 @@ static GLint choose_int_fmt(GLenum fmt, uint q_flags) wchar_t buf[100]; swprintf(buf, ARRAY_SIZE(buf), L"choose_int_fmt: fmt 0x%x isn't covered! please add it", fmt); DISPLAY_ERROR(buf); - debug_warn("given fmt isn't covered! please add it."); + debug_assert(0); // given fmt isn't covered! please add it. // fall back to a reasonable default return half_bpp? GL_RGB4 : GL_RGB8; } @@ -383,7 +385,7 @@ static void OglTex_dtor(OglTex* ot) { if(ot->flags & OT_TEX_VALID) { - (void)tex_free(&ot->t); + tex_free(&ot->t); ot->flags &= ~OT_TEX_VALID; } @@ -403,7 +405,7 @@ static LibError OglTex_reload(OglTex* ot, const char* fn, Handle h) // if we don't already have the texture in memory (*), load from file. // * this happens if the texture is "wrapped". if(!(ot->flags & OT_TEX_VALID)) - RETURN_ERR(tex_load(fn, &ot->t, FILE_CACHED_AT_HIGHER_LEVEL)); + RETURN_ERR(tex_load(fn, &ot->t)); ot->flags |= OT_TEX_VALID; glGenTextures(1, &ot->id); @@ -548,7 +550,7 @@ static void warn_if_uploaded(Handle ht, const OglTex* ot) return; // don't complain if(ot->flags & OT_IS_UPLOADED) - debug_warn("ogl_tex_set_*: texture already uploaded and shouldn't be changed"); + debug_assert(0); // ogl_tex_set_*: texture already uploaded and shouldn't be changed #else // (prevent warnings; the alternative of wrapping all call sites in // #ifndef is worse) @@ -627,7 +629,7 @@ void ogl_tex_override(OglTexOverrides what, OglTexAllow allow) have_auto_mipmap_gen = enable; break; default: - debug_warn("invalid "); + debug_assert(0); // invalid break; } } @@ -639,8 +641,7 @@ void ogl_tex_override(OglTexOverrides what, OglTexAllow allow) static void detect_gl_upload_caps() { // make sure us and the app hook have graphics card info available - if(gfx_card[0] == '\0') - debug_warn("gfx_detect must be called before ogl_tex_upload"); + debug_assert(gfx_card[0] != '\0'); // gfx_detect must be called before ogl_tex_upload // detect features, but only change the variables if they were at // "undecided" (if overrides were set before this, they must remain). @@ -856,7 +857,7 @@ LibError ogl_tex_upload(const Handle ht, GLenum fmt_ovr, uint q_flags_ovr, GLint if(refs == 1) { // note: we verify above that OT_TEX_VALID is set - (void)tex_free(t); + tex_free(t); ot->flags &= ~OT_TEX_VALID; } @@ -895,8 +896,7 @@ LibError ogl_tex_get_format(Handle ht, uint* flags, GLenum* fmt) *flags = ot->t.flags; if(fmt) { - if(!(ot->flags & OT_IS_UPLOADED)) - debug_warn("hasn't been defined yet!"); + debug_assert(ot->flags & OT_IS_UPLOADED); *fmt = ot->fmt; } return INFO::OK; diff --git a/source/lib/res/graphics/ogl_tex.h b/source/lib/res/graphics/ogl_tex.h index 808c9d7d7a..2208841b2b 100644 --- a/source/lib/res/graphics/ogl_tex.h +++ b/source/lib/res/graphics/ogl_tex.h @@ -133,7 +133,7 @@ the next function to fail, but real apps should check and report errors. #include "lib/res/handle.h" #include "lib/ogl.h" -#include "tex.h" +#include "lib/tex/tex.h" // diff --git a/source/lib/res/graphics/tests/test_tex.h b/source/lib/res/graphics/tests/test_tex.h index 3488532c14..e0d6a670ee 100644 --- a/source/lib/res/graphics/tests/test_tex.h +++ b/source/lib/res/graphics/tests/test_tex.h @@ -1,6 +1,6 @@ #include "lib/self_test.h" -#include "lib/res/graphics/tex.h" +#include "lib/tex/tex.h" #include "lib/res/graphics/tex_codec.h" #include "lib/res/graphics/tex_internal.h" // tex_encode @@ -35,7 +35,7 @@ class TestTex : public CxxTest::TestSuite TS_ASSERT_SAME_DATA(tex_get_data(&t), img, size); // cleanup - TS_ASSERT_OK(tex_free(&t)); + tex_free(&t); TS_ASSERT_OK(da_free(&da)); delete[] img; } diff --git a/source/lib/res/graphics/unifont.cpp b/source/lib/res/graphics/unifont.cpp index c2396b3f7a..95bf118962 100644 --- a/source/lib/res/graphics/unifont.cpp +++ b/source/lib/res/graphics/unifont.cpp @@ -14,9 +14,10 @@ #include #include -#include "lib/ogl.h" -#include "lib/res/res.h" #include "ogl_tex.h" +#include "../h_mgr.h" +#include "lib/file/vfs/vfs.h" +extern PIVFS g_VFS; // This isn't particularly efficient - it can be improved if we // (a) care enough, and (b) know about fixed ranges of characters @@ -74,15 +75,14 @@ static LibError UniFont_reload(UniFont* f, const char* fn, Handle UNUSED(h)) // Read font definition file into a stringstream const std::string FilenameFnt = FilenameBase+".fnt"; const char* fnt_fn = FilenameFnt.c_str(); - FileIOBuf buf; size_t size; - RETURN_ERR(vfs_load(fnt_fn, buf, size)); // [cumulative for 12: 36ms] - std::istringstream FNTStream (std::string((const char*)buf, (int)size)); - (void)file_buf_free(buf); + shared_ptr buf; size_t size; + RETURN_ERR(g_VFS->LoadFile(fnt_fn, buf, size)); // [cumulative for 12: 36ms] + std::istringstream FNTStream (std::string((const char*)buf.get(), (int)size)); int Version; FNTStream >> Version; if (Version != 100) // Make sure this is from a recent version of the font builder - WARN_RETURN(ERR::RES_UNKNOWN_FORMAT); + WARN_RETURN(ERR::FAIL); int TextureWidth, TextureHeight; FNTStream >> TextureWidth >> TextureHeight; @@ -109,7 +109,7 @@ static LibError UniFont_reload(UniFont* f, const char* fn, Handle UNUSED(h)) if (Codepoint > 0xFFFF) { - debug_warn("Invalid codepoint"); + debug_assert(0); // Invalid codepoint continue; } @@ -269,7 +269,7 @@ void glvwprintf(const wchar_t* fmt, va_list args) if (it == BoundGlyphs->end()) // Missing the missing glyph symbol - give up { - debug_warn("Missing the missing glyph in a unifont!\n"); + debug_assert(0); // Missing the missing glyph in a unifont! return; } @@ -312,7 +312,7 @@ LibError unifont_stringsize(const Handle h, const wchar_t* text, int& width, int if (it == f->glyphs_size->end()) // Missing the missing glyph symbol - give up { - debug_warn("Missing the missing glyph in a unifont!\n"); + debug_assert(0); // Missing the missing glyph in a unifont! return INFO::OK; } diff --git a/source/lib/res/h_mgr.cpp b/source/lib/res/h_mgr.cpp index 6d62851bf4..4d2033a4ad 100644 --- a/source/lib/res/h_mgr.cpp +++ b/source/lib/res/h_mgr.cpp @@ -483,8 +483,7 @@ static void warn_if_invalid(HDATA* hd) const u8* start = hd->user + vtbl->user_size; const u8* end = hd->user + HDATA_USER_SIZE; for(const u8* p = start; p < end; p++) - if(*p != 0) - debug_warn("handle user data was overrun!"); + debug_assert(*p == 0); // else: handle user data was overrun! } #else UNUSED2(hd); @@ -763,7 +762,7 @@ void* h_user_data(const Handle h, const H_Type type) if(!hd->refs) { // note: resetting the tag is not enough (user might pass in its value) - debug_warn("no references to resource (it's cached, but someone is accessing it directly)"); + debug_assert(0); // no references to resource (it's cached, but someone is accessing it directly) return 0; } @@ -779,7 +778,7 @@ const char* h_filename(const Handle h) HDATA* hd = h_data_tag(h); if(!hd) { - debug_warn("failed"); + debug_assert(0); return 0; } return hd->fn; @@ -872,14 +871,11 @@ void h_add_ref(Handle h) HDATA* hd = h_data_tag(h); if(!hd) { - debug_warn("invalid handle"); + debug_assert(0); // invalid handle return; } - // if there are no refs, how did the caller manage to keep a Handle?! - if(!hd->refs) - debug_warn("no refs open - resource is cached"); - + debug_assert(hd->refs); // if there are no refs, how did the caller manage to keep a Handle?! hd->refs++; } @@ -896,10 +892,7 @@ int h_get_refcnt(Handle h) if(!hd) WARN_RETURN(ERR::INVALID_PARAM); - // if there are no refs, how did the caller manage to keep a Handle?! - if(!hd->refs) - debug_warn("no refs open - resource is cached"); - + debug_assert(hd->refs); // if there are no refs, how did the caller manage to keep a Handle?! return hd->refs; } @@ -930,7 +923,7 @@ void h_mgr_shutdown() // each HDATA entry has already been allocated. if(!hd) { - debug_warn("h_data_from_idx failed - why?!"); + debug_assert(0); // h_data_from_idx failed - why?! continue; } diff --git a/source/lib/res/mem.cpp b/source/lib/res/mem.cpp deleted file mode 100644 index 1caa002354..0000000000 --- a/source/lib/res/mem.cpp +++ /dev/null @@ -1,432 +0,0 @@ -/** - * ========================================================================= - * File : mem.cpp - * Project : 0 A.D. - * Description : wrapper that treats memory as a "resource", i.e. - * : guarantees its lifetime and automatic release. - * ========================================================================= - */ - -// license: GPL; see lib/license.txt - -#include "precompiled.h" -#include "mem.h" - -#include -#include - -#include "lib/posix/posix_pthread.h" -#include "lib/bits.h" -#include "lib/allocators.h" // OverrunProtector -#include "h_mgr.h" - - -static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -struct ScopedLock -{ - ScopedLock() { pthread_mutex_lock(&mutex); } - ~ScopedLock() { pthread_mutex_unlock(&mutex); } -}; -#define SCOPED_LOCK ScopedLock UID__; - -struct Mem -{ - // initially what mem_alloc returns; can be changed via mem_assign_user - void* p; - size_t size; - - // unaligned mem from allocator - void* raw_p; - size_t raw_size; - - uintptr_t ctx; - MEM_DTOR dtor; // this allows user-specified dtors. - - void* owner; -}; - -H_TYPE_DEFINE(Mem); - - -////////////////////////////////////////////////////////////////////////////// - - -// raw pointer -> Handle -typedef std::map Ptr2H; -typedef Ptr2H::iterator It; -static OverrunProtector ptr2h_wrapper; - -// not needed by other modules - mem_get_size and mem_wrap is enough. -static Handle find_alloc(void* target_raw_p, It* out_it = 0) -{ - Ptr2H* ptr2h = ptr2h_wrapper.get(); - if(!ptr2h) - WARN_RETURN(ERR::NO_MEM); - - // early out optimization (don't pay for full subset check) - It it = ptr2h->find(target_raw_p); - if(it != ptr2h->end()) - return it->second; - - // initial return value: "not found" - Handle ret = -1; - - // not found; now check if target_raw_p is within one of the mem ranges - for(it = ptr2h->begin(); it != ptr2h->end(); ++it) - { - void* raw_p = it->first; - Handle hm = it->second; - - // not before this alloc's p; could be it. now do range check. - if(target_raw_p >= raw_p) - { - Mem* m = (Mem*)h_user_data(hm, H_Mem); - if(m) - { - // found it within this mem range. - if(target_raw_p <= (char*)m->raw_p + m->raw_size) - { - if(out_it) - *out_it = it; - ret = hm; - break; - } - } - } - } - - ptr2h_wrapper.lock(); - return ret; -} - - -// raw_p must be in map! -static void remove_alloc(void* raw_p) -{ - Ptr2H* ptr2h = ptr2h_wrapper.get(); - if(!ptr2h) - return; - -//debug_printf("REMOVE raw_p=%p\n", raw_p); - - size_t num_removed = ptr2h->erase(raw_p); - if(num_removed != 1) - debug_warn("not in map"); - - ptr2h_wrapper.lock(); -} - - -// raw_p must not already be in map! -static void set_alloc(void* raw_p, Handle hm) -{ - Ptr2H* ptr2h = ptr2h_wrapper.get(); - if(!ptr2h) - return; - -//debug_printf("SETALLOC raw_p=%p hm=%I64x\n", raw_p, hm); - - std::pair ret = ptr2h->insert(std::make_pair(raw_p, hm)); - if(!ret.second) - debug_warn("already in map"); - - ptr2h_wrapper.lock(); -} - - -////////////////////////////////////////////////////////////////////////////// - - - -static void Mem_init(Mem* m, va_list args) -{ - // these are passed to h_alloc instead of assigning in mem_wrap after a - // H_DEREF so that Mem_validate won't complain about invalid (0) values. - // - // additional bonus: by setting raw_p before reload, that and the - // dtor will be the only call site of set/remove_alloc. - m->p = va_arg(args, void*); - m->size = va_arg(args, size_t); - m->raw_p = va_arg(args, void*); - m->raw_size = va_arg(args, size_t); - m->dtor = va_arg(args, MEM_DTOR); - m->ctx = va_arg(args, uintptr_t); - m->owner = va_arg(args, void*); -} - -static void Mem_dtor(Mem* m) -{ - // (reload can't fail) - - remove_alloc(m->raw_p); - - if(m->dtor) - m->dtor(m->raw_p, m->raw_size, m->ctx); -} - -// can't alloc here, because h_alloc needs the key when called -// (key == pointer we allocate) -static LibError Mem_reload(Mem* m, const char* UNUSED(fn), Handle hm) -{ - set_alloc(m->raw_p, hm); - return INFO::OK; -} - -static LibError Mem_validate(const Mem* m) -{ - if(debug_is_pointer_bogus(m->p)) - WARN_RETURN(ERR::_1); - if(!m->size) - WARN_RETURN(ERR::_2); - if(m->raw_p && m->raw_p > m->p) - WARN_RETURN(ERR::_3); - if(m->raw_size && m->raw_size < m->size) - WARN_RETURN(ERR::_4); - return INFO::OK; -} - -static LibError Mem_to_string(const Mem* m, char* buf) -{ - char owner_sym[DBG_SYMBOL_LEN]; - if(debug_resolve_symbol(m->owner, owner_sym, 0, 0) < 0) - { - if(m->owner) - snprintf(owner_sym, ARRAY_SIZE(owner_sym), "(%p)", m->owner); - else - strcpy_s(owner_sym, ARRAY_SIZE(owner_sym), "(?)"); - } - - snprintf(buf, H_STRING_LEN, "p=%p size=%d owner=%s", m->p, m->size, owner_sym); - return INFO::OK; -} - - -////////////////////////////////////////////////////////////////////////////// - -// "*": aligned memory returned by allocator. -// "user_*": same as above, until someones changes it via mem_assign_user - -// allocator interface: -// alloc: return at least size bytes of memory (alignment done by caller) - -////////////////////////////////////////////////////////////////////////////// - -// implementation must be thread-safe! (since mem_alloc doesn't take a lock) - -static void heap_free(void* raw_p, size_t UNUSED(raw_size), uintptr_t UNUSED(ctx)) -{ - free(raw_p); -} - - -static void* heap_alloc(size_t raw_size, uintptr_t& ctx) -{ - ctx = 0; - void* raw_p = malloc(raw_size); - return raw_p; -} - - -////////////////////////////////////////////////////////////////////////////// - - -LibError mem_free_h(Handle& hm) -{ - SCOPED_LOCK; - return h_free(hm, H_Mem); -} - - -LibError mem_free_p(void*& p) -{ - if(!p) - return INFO::OK; - - Handle hm; - { - SCOPED_LOCK; - hm = find_alloc(p); - } - - p = 0; - if(hm <= 0) - WARN_RETURN(ERR::MEM_ALLOC_NOT_FOUND); - return mem_free_h(hm); -} - - -// create a H_MEM handle of type MEM_USER, -// and assign it the specified memory range. -// if dtor is non-NULL, it is called (passing ctx) when the handle is freed. -Handle mem_wrap(void* p, size_t size, uint flags, void* raw_p, size_t raw_size, MEM_DTOR dtor, uintptr_t ctx, void* owner) -{ - if(!p || !size) - WARN_RETURN(ERR::INVALID_PARAM); - - SCOPED_LOCK; - - // we've already allocated that pointer; return its handle and - // increment refcnt. - Handle hm = find_alloc(p); - if(hm > 0) - { - h_add_ref(hm); - return hm; - } - - //

wasn't allocated via mem_alloc, or we would've found its Handle. - // it is therefore some user-allocated mem and might therefore not have - // a valid set. since that's our search key, we set it to

. - if(!raw_p) - raw_p = p; - if(!raw_size) - raw_size = size; - - hm = h_alloc(H_Mem, (const char*)p, flags|RES_KEY|RES_NO_CACHE, - p, size, raw_p, raw_size, dtor, ctx, owner); - return hm; -} - - -/* -LibError mem_assign_user(Handle hm, void* user_p, size_t user_size) -{ - H_DEREF(hm, Mem, m); - - // security check: must be a subset of the existing buffer - // (otherwise, could reference other buffers / cause mischief) - char* raw_end = (char*)m->raw_p + m->raw_size; - char* user_end = (char*)user_p + user_size; - if(user_p < m->raw_p || user_end > raw_end) - { - debug_warn("mem_assign_user: user buffer not contained in real buffer"); - return -1; - } - - m->p = user_p; - m->size = user_size; - return INFO::OK; -} -*/ - - -// implementation note: does not currently take a lock; all the -// heavy-lifting happens inside mem_wrap. -void* mem_alloc(size_t size, const size_t align, uint flags, Handle* phm) -{ - if(phm) - *phm = ERR::NO_MEM; - -#ifdef NDEBUG - void* owner = 0; -#else - void* owner = debug_get_nth_caller(1, 0); -#endif - - // note: this is legitimate. vfs_load on 0-length files must return - // a valid and unique pointer to an (at least) 0-length buffer. - if(size == 0) - { - debug_printf("MEM 0 byte alloc\n"); - size = 1; - } - - void* raw_p; - const size_t raw_size = size + align-1; - - uintptr_t ctx; - MEM_DTOR dtor; - -// if(scope == RES_TEMP) -// { -// raw_p = pool_alloc(raw_size, ctx); -// dtor = pool_free; -// } -// else - { - raw_p = heap_alloc(raw_size, ctx); - dtor = heap_free; - } - - if(!raw_p) - return 0; - void* p = (void*)round_up((uintptr_t)raw_p, align); - -//debug_printf("MEMWRAP p=%p size=%x raw_p=%p raw_size=%x owner=%p\n", p, size, raw_p, raw_size, owner); - Handle hm = mem_wrap(p, size, flags, raw_p, raw_size, dtor, ctx, owner); - if(!hm) // failed to allocate a handle - { - debug_warn("mem_wrap failed"); - dtor(p, size, ctx); - return 0; - } - - // check if pointer was already allocated? - - - // caller is asking for the handle - // (freeing the memory via handle is faster than mem_free, because - // we wouldn't have to scan all handles looking for the pointer) - if(phm) - *phm = hm; - - if(flags & MEM_ZERO) - memset(p, 0, size); - - return p; -} - - -void* mem_get_ptr(Handle hm, size_t* user_size /* = 0 */) -{ - SCOPED_LOCK; - - Mem* m = H_USER_DATA(hm, Mem); - if(!m) - { - if(user_size) - *user_size = 0; - return 0; - } - - debug_assert((!m->p || m->size) && "mem_get_ptr: mem corrupted (p valid =/=> size > 0)"); - - if(user_size) - *user_size = m->size; - return m->p; -} - - -LibError mem_get(Handle hm, u8** pp, size_t* psize) -{ - SCOPED_LOCK; - - H_DEREF(hm, Mem, m); - if(pp) - *pp = (u8*)m->p; - if(psize) - *psize = m->size; - // leave hm locked - return INFO::OK; -} - - -/* -ssize_t mem_size(void* p) -{ - SCOPED_LOCK; - Handle hm = find_alloc(p); - H_DEREF(hm, Mem, m); - return (ssize_t)m->size; -} -*/ - - -// exception to normal resource shutdown: must not be called before -// h_mgr_shutdown! (this is because h_mgr calls us to free memory, which -// requires the pointer -> Handle index still be in place) -void mem_shutdown() -{ - // ptr2h_wrapper is currently freed at NLSO dtor time. - // if that's a problem, use OverrunProtector::shutdown. -} diff --git a/source/lib/res/mem.h b/source/lib/res/mem.h deleted file mode 100644 index d81d11f1a6..0000000000 --- a/source/lib/res/mem.h +++ /dev/null @@ -1,50 +0,0 @@ -/** -* ========================================================================= -* File : mem.h -* Project : 0 A.D. -* Description : wrapper that treats memory as a "resource", i.e. -* : guarantees its lifetime and automatic release. -* ========================================================================= -*/ - -// license: GPL; see lib/license.txt - -#ifndef INCLUDED_MEM -#define INCLUDED_MEM - -#include "handle.h" - - -typedef void (*MEM_DTOR)(void* p, size_t size, uintptr_t ctx); - -// mem_alloc flags -enum -{ - // RES_* - - MEM_ZERO = 0x1000 -}; - -extern void* mem_alloc(size_t size, size_t align = 1, uint flags = 0, Handle* ph = 0); - -#define mem_free(p) mem_free_p((void*&)p) -extern LibError mem_free_p(void*& p); - -extern LibError mem_free_h(Handle& hm); - -// returns 0 if the handle is invalid -extern void* mem_get_ptr(Handle h, size_t* size = 0); - -extern LibError mem_get(Handle hm, u8** pp, size_t* psize); - - -extern Handle mem_wrap(void* p, size_t size, uint flags, void* raw_p, size_t raw_size, MEM_DTOR dtor, uintptr_t ctx, void* owner); - -// exception to normal resource shutdown: must not be called before -// h_mgr_shutdown! (this is because h_mgr calls us to free memory, which -// requires the pointer -> Handle index still be in place) -extern void mem_shutdown(void); - - - -#endif // #ifndef INCLUDED_MEM diff --git a/source/lib/res/res.h b/source/lib/res/res.h deleted file mode 100644 index 951373dfd6..0000000000 --- a/source/lib/res/res.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef INCLUDED_RES -#define INCLUDED_RES - -// common headers needed by lib/res code - -#include "h_mgr.h" -#include "lib/res/file/vfs.h" -#include "lib/path_util.h" -#include "mem.h" - - -namespace ERR -{ - const LibError RES_UNKNOWN_FORMAT = -120000; - const LibError RES_INCOMPLETE_HEADER = -120001; -} - -#endif // #ifndef INCLUDED_RES diff --git a/source/lib/res/sound/ogghack.cpp b/source/lib/res/sound/ogghack.cpp index bfe3f79341..ef06498dc5 100644 --- a/source/lib/res/sound/ogghack.cpp +++ b/source/lib/res/sound/ogghack.cpp @@ -132,7 +132,7 @@ void ogg_open(void* _o, ALenum& fmt, ALsizei& freq) void* datasource = &o->incoming_bufs; if(ov_open_callbacks(datasource, &o->oggStream, NULL, 0, cbs) < 0) { - debug_warn("ov_open failed"); + debug_assert(0); // ov_open failed } o->vorbisInfo = ov_info(&o->oggStream, -1); @@ -170,7 +170,7 @@ size_t ogg_read(void* _o, void* buf, size_t max_size) if(result > 0) bytes_written += result; else if(result < 0) - debug_warn("ogg read error"); + debug_assert(0); // ogg read error // clean break - end of data else break; diff --git a/source/lib/res/sound/snd_mgr.cpp b/source/lib/res/sound/snd_mgr.cpp index 176073d7e1..38a7b1b98e 100644 --- a/source/lib/res/sound/snd_mgr.cpp +++ b/source/lib/res/sound/snd_mgr.cpp @@ -19,14 +19,17 @@ #include #include -#include "maths/MathUtil.h" // PI +#include "lib/path_util.h" +#include "../h_mgr.h" +#include "lib/file/vfs/vfs.h" +extern PIVFS g_VFS; + // for DLL-load hack in alc_init #if OS_WIN # include "lib/sysdep/win/win.h" #endif -#include "lib/res/res.h" #include "lib/timer.h" #include "lib/app_hooks.h" #include "lib/external_libraries/openal.h" @@ -120,7 +123,7 @@ static void al_check(const char* caller = "(unknown)") const char* str = (const char*)alGetString(err); debug_printf("OpenAL error: %s; called from %s (#%d)\n", str, caller, num_times_within_same_function); - debug_warn("OpenAL error"); + debug_assert(0); } // convenience version that automatically passes in function name. @@ -438,8 +441,7 @@ static void al_buf_free(ALuint al_buf) */ static void al_buf_shutdown() { - if(al_bufs_outstanding != 0) - debug_warn("not all buffers freed!"); + debug_assert(al_bufs_outstanding == 0); } @@ -696,294 +698,6 @@ const char* snd_dev_next() } -//----------------------------------------------------------------------------- -// stream: passes chunks of data (read via async I/O) to snd_data on request. -//----------------------------------------------------------------------------- - -// one stream apiece for music and voiceover (narration during tutorial). -// allowing more is possible, but would be inefficent due to seek overhead. -// set this limit to catch questionable usage (e.g. streaming normal sounds). -static const uint MAX_STREAMS = 2; - -// maximum IOs queued per stream. -static const uint MAX_IOS = 4; - -static const size_t STREAM_BUF_SIZE = 32*KiB; - - -//----------------------------------------------------------------------------- -// I/O buffer suballocator: avoids frequent large alloc/frees -// when streaming, and therefore heap fragmentation. - -static const int TOTAL_IOS = MAX_STREAMS * MAX_IOS; -static const size_t TOTAL_BUF_SIZE = TOTAL_IOS * STREAM_BUF_SIZE; - -// one large allocation for all buffers -static u8* io_bufs; - -// list of free buffers. start of buffer holds pointer to next in list. -static u8* io_buf_freelist; - - -/** - * Free an IO buffer. - * - * @param p IO buffer - */ -static void io_buf_free(u8* p) -{ - debug_assert(io_bufs <= p && p <= io_bufs+TOTAL_BUF_SIZE); - *(u8**)p = io_buf_freelist; - io_buf_freelist = p; -} - - -/** - * Allocate a memory pool for all IO buffers. - * Called from first io_buf_alloc. - */ -static void io_buf_init() -{ - // allocate 1 big aligned block for all buffers. - io_bufs = (u8*)mem_alloc(TOTAL_BUF_SIZE, 4*KiB); - // .. failed; io_buf_alloc calls will return 0 - if(!io_bufs) - return; - - // build freelist. - u8* p = io_bufs; - for(int i = 0; i < TOTAL_IOS; i++) - { - io_buf_free(p); - p += STREAM_BUF_SIZE; - } -} - - -/** - * Allocate a fixed-size IO buffer. - * - * @return buffer, or 0 (and warning) if not enough memory. - */ -static u8* io_buf_alloc() -{ - ONCE(io_buf_init()); - - u8* buf = io_buf_freelist; - // note: we have to bail now; can't update io_buf_freelist. - if(!buf) - { - // no buffer allocated - if(!io_bufs) - WARN_ERR(ERR::NO_MEM); - // too many streams - can't happen (tm) because - // stream_open enforces MAX_STREAMS. - else - WARN_ERR(ERR::LIMIT); - - return 0; - } - - io_buf_freelist = *(u8**)io_buf_freelist; - return buf; -} - - -/** - * Free memory pool holding all IO buffers. - * no-op if io_buf_alloc was never called. - * called by snd_shutdown. - */ -static void io_buf_shutdown() -{ - mem_free(io_bufs); -} - - -//----------------------------------------------------------------------------- -// stream: owns queue and buffers; uses VFS async I/O - -// rationale: no need for a centralized queue - we have a suballocator, -// so reallocs aren't a problem; central scheduling isn't necessary, -// because we'll only have <= 2 streams active at a time. - -/** - * Information for streaming sounds from file - * (i.e. loading pieces of it in the background) - */ -struct Stream -{ - Handle hf; - Handle ios[MAX_IOS]; - uint active_ios; - /// set by stream_buf_get, used by stream_buf_discard to free buf. - u8* last_buf; -}; - -/** - * Begin reading the next segment (asynchronously). - * Called from SndData_reload and snd_data_buf_get. - * - * @param Stream* - * @return LibError - */ -static LibError stream_issue(Stream * s) -{ - if(s->active_ios >= MAX_IOS) - return INFO::OK; - - u8* buf = io_buf_alloc(); - if(!buf) - WARN_RETURN(ERR::NO_MEM); - - Handle h = vfs_io_issue(s->hf, STREAM_BUF_SIZE, buf); - RETURN_ERR(h); - s->ios[s->active_ios++] = h; - return INFO::OK; -} - - -/** - * Access the data most recently streamed in. - * - * @param Stream* - * @param data pointer to buffer - * @param size [bytes] - * @return LibError; if the first pending IO hasn't completed, - * ERR::AGAIN (not an error). - */ -static LibError stream_buf_get(Stream * s, u8*& data, size_t& size) -{ - if(s->active_ios == 0) - WARN_RETURN(ERR::IO_EOF); - Handle hio = s->ios[0]; - - // has it finished? if not, bail. - int is_complete = vfs_io_has_completed(hio); - RETURN_ERR(is_complete); - if(is_complete == 0) - return ERR::AGAIN; // NOWARN - - // get its buffer. - // no delay, since vfs_io_has_completed == 1 - RETURN_ERR(vfs_io_wait(hio, data, size)); - - // (next stream_buf_discard will free this buffer) - s->last_buf = data; - return INFO::OK; -} - - -/** - * Free the buffer that was last returned by stream_buf_get, - * and remove its IO slot from our queue. - * - * Must be called exactly once after every successful stream_buf_get; - * call before calling any other stream_ * functions! - * - * @param Stream* - * @return LibError - */ -static LibError stream_buf_discard(Stream * s) -{ - Handle hio = s->ios[0]; - - LibError ret = vfs_io_discard(hio); - - // we implement the required 'circular queue' as a stack; - // have to shift all items after this one down. - s->active_ios--; - for(uint i = 0; i < s->active_ios; i++) - s->ios[i] = s->ios[i+1]; - - io_buf_free(s->last_buf); // can't fail - s->last_buf = 0; - - return ret; -} - - -static uint active_streams; - - -/** - * open a stream and begin reading from disk. - * - * @param Stream* - * @param fn VFS filename. - * @return LibError - */ -static LibError stream_open(Stream * s, const char* fn) -{ - // bail because we wouldn't have enough IO buffers for all - if(active_streams >= MAX_STREAMS) - WARN_RETURN(ERR::LIMIT); - active_streams++; - - s->hf = vfs_open(fn); - RETURN_ERR(s->hf); - - for(int i = 0; i < MAX_IOS; i++) - RETURN_ERR(stream_issue(s)); - - return INFO::OK; -} - - -/** - * close a stream, which may currently be active. - * - * @param Stream* - * @return LibError - the first error that occurred while waiting for - * IOs / closing file. - */ -static LibError stream_close(Stream * s) -{ - LibError ret = INFO::OK; - LibError err; - - // for each pending IO: - for(uint i = 0; i < s->active_ios; i++) - { - // .. wait until complete, - u8* data; size_t size; // unused - do - err = stream_buf_get(s, data, size); - while(err == ERR::AGAIN); - if(err < 0 && ret == 0) - ret = err; - - // .. and discard. - err = stream_buf_discard(s); - if(err < 0 && ret == 0) - ret = err; - } - - err = vfs_close(s->hf); - if(err < 0 && ret == 0) - ret = err; - - active_streams--; - - return ret; -} - - -/** - * Make sure the given Stream is valid/self-consistent. - * - * @param const Stream* - * @return LibError - */ -static LibError stream_validate(const Stream * s) -{ - if(s->active_ios > MAX_IOS) - WARN_RETURN(ERR::_1); - // has no invariant we could check. - return INFO::OK; -} - - //----------------------------------------------------------------------------- // sound data provider: holds audio data (clip or stream) and returns // OpenAL buffers on request. @@ -1010,15 +724,11 @@ static LibError stream_validate(const Stream * s) */ struct SndData { - // stream - Stream s; ALenum al_fmt; ALsizei al_freq; - // clip ALuint al_buf; - uint is_stream : 1; uint is_valid : 1; #ifdef OGG_HACK @@ -1029,10 +739,8 @@ void* o; H_TYPE_DEFINE(SndData); -static void SndData_init(SndData * sd, va_list args) +static void SndData_init(SndData * UNUSED(sd), va_list UNUSED(args)) { - // olsner: pass as int instead of bool for GCC compat. - sd->is_stream = va_arg(args, int) != 0; } static void SndData_dtor(SndData * sd) @@ -1042,10 +750,7 @@ static void SndData_dtor(SndData * sd) if(!sd->is_valid) return; - if(sd->is_stream) - stream_close(&sd->s); - else - al_buf_free(sd->al_buf); + al_buf_free(sd->al_buf); #ifdef OGG_HACK if(sd->o) ogg_release(sd->o); @@ -1094,34 +799,17 @@ static LibError SndData_reload(SndData * sd, const char* fn, Handle hsd) } // .. unknown extension else - WARN_RETURN(ERR::RES_UNKNOWN_FORMAT); + WARN_RETURN(ERR::FAIL); // note: WAV is no longer supported. writing our own loader is infeasible // due to a seriously watered down spec with many incompatible variants. // pulling in an external library (e.g. freealut) is deemed not worth the // effort - OGG should be better in all cases. - if(sd->is_stream) - { - // refuse to stream anything that cannot be passed directly to OpenAL - - // we'd have to extract the audio data ourselves (not worth it). - if(file_type != FT_OGG) - WARN_RETURN(ERR::NOT_SUPPORTED); + shared_ptr file; size_t file_size; + RETURN_ERR(g_VFS->LoadFile(fn, file, file_size)); - RETURN_ERR(stream_open(&sd->s, fn)); - - sd->is_valid = 1; - hsd_list_add(hsd); - return INFO::OK; - } - - // else: clip - - FileIOBuf file; - size_t file_size; - RETURN_ERR(vfs_load(fn, file, file_size)); - - ALvoid* al_data = (ALvoid*)file; + ALvoid* al_data = (ALvoid*)file.get(); ALsizei al_size = (ALsizei)file_size; #ifdef OGG_HACK @@ -1130,7 +818,7 @@ static LibError SndData_reload(SndData * sd, const char* fn, Handle hsd) if(file_type == FT_OGG) { sd->o = ogg_create(); - ogg_give_raw(sd->o, (void*)file, file_size); + ogg_give_raw(sd->o, file.get(), file_size); ogg_open(sd->o, sd->al_fmt, sd->al_freq); size_t datasize=0; size_t bytes_read; @@ -1152,8 +840,6 @@ static LibError SndData_reload(SndData * sd, const char* fn, Handle hsd) } #endif - (void)file_buf_free(file); - sd->al_buf = al_buf_alloc(al_data, al_size, sd->al_fmt, sd->al_freq); sd->is_valid = 1; @@ -1170,15 +856,12 @@ static LibError SndData_validate(const SndData * sd) if(sd->al_buf == 0) WARN_RETURN(ERR::_13); - if(sd->is_stream) - RETURN_ERR(stream_validate(&sd->s)); - return INFO::OK; } static LibError SndData_to_string(const SndData * sd, char * buf) { - const char* type = sd->is_stream? "stream" : "clip"; + const char* type = "clip"; snprintf(buf, H_STRING_LEN, "%s; al_buf=%d", type, sd->al_buf); return INFO::OK; } @@ -1196,12 +879,9 @@ static LibError SndData_to_string(const SndData * sd, char * buf) */ static Handle snd_data_load(const char* fn, bool is_stream) { - // don't allow reloading stream objects - // (both references would read from the same file handle). - const uint flags = is_stream? RES_UNIQUE : 0; + debug_assert(!is_stream); // no longer supported - return h_alloc(H_SndData, fn, flags, (int)is_stream); - // (int) rationale: see SndData_init + return h_alloc(H_SndData, fn); } @@ -1296,49 +976,16 @@ static void hsd_list_free_all() * @param hsd Handle to SndData. * @param al_buf buffer name. * @return LibError, most commonly: - * INFO::OK = buffer has been returned; more are expected to be available. - * ERR::IO_EOF = buffer has been returned but is the last one - * (end of file reached) - * ERR::AGAIN = no buffer returned yet; still streaming in ATM. - * call back later. + * INFO::CB_CONTINUE = buffer has been returned; more are expected to be available. + * INFO::OK = buffer has been returned but is the last one (EOF). + * ERR::AGAIN = IO pending; call again later. */ static LibError snd_data_buf_get(Handle hsd, ALuint& al_buf) { - LibError err = INFO::OK; - - // in case H_DEREF fails - al_buf = 0; - H_DEREF(hsd, SndData, sd); // clip: just return buffer (which was created in snd_data_load) - if(!sd->is_stream) - { - al_buf = sd->al_buf; - return ERR::IO_EOF; // NOWARN - } - - // stream: - - // .. check if IO finished. - u8* data; size_t size; - err = stream_buf_get(&sd->s, data, size); - if(err == ERR::AGAIN) - return ERR::AGAIN; // NOWARN - RETURN_ERR(err); - - // .. yes: pass to OpenAL and discard IO buffer. - al_buf = al_buf_alloc(data, (ALsizei)size, sd->al_fmt, sd->al_freq); - stream_buf_discard(&sd->s); - - // .. try to issue the next IO. - // if EOF reached, indicate al_buf is the last that will be returned. - err = stream_issue(&sd->s); - if(err == ERR::IO_EOF) - return ERR::IO_EOF; // NOWARN - RETURN_ERR(err); - - // al_buf valid and next IO issued successfully. + al_buf = sd->al_buf; return INFO::OK; } @@ -1350,17 +997,12 @@ static LibError snd_data_buf_get(Handle hsd, ALuint& al_buf) * @param al_buf buffer name * @return LibError */ -static LibError snd_data_buf_free(Handle hsd, ALuint al_buf) +static LibError snd_data_buf_free(Handle hsd, ALuint UNUSED(al_buf)) { H_DEREF(hsd, SndData, sd); // clip: no-op (caller will later release hsd reference; // when hsd actually unloads, sd->al_buf will be freed). - if(!sd->is_stream) - return INFO::OK; - - // stream: we had allocated an additional buffer, so free it now. - al_buf_free(al_buf); return INFO::OK; } @@ -1395,7 +1037,8 @@ static float fade_factor_exponential(float t) static float fade_factor_s_curve(float t) { // cosine curve - float y = cos(t*PI + PI); + const double pi = 3.14159265358979323846; + float y = cos(t*pi + pi); // map [-1,1] to [0,1] return (y + 1.0f) / 2.0f; } @@ -1609,10 +1252,9 @@ static LibError VSrc_reload(VSrc* vs, const char* fn, Handle hvs) const char* ext = path_extension(fn); if(!strcasecmp(ext, "txt")) { - FileIOBuf buf; size_t size; - RETURN_ERR(vfs_load(fn, buf, size)); - std::istringstream def(std::string((char*)buf, (int)size)); - (void)file_buf_free(buf); + shared_ptr buf; size_t size; + RETURN_ERR(g_VFS->LoadFile(fn, buf, size)); + std::istringstream def(std::string((char*)buf.get(), (int)size)); float gain_percent; def >> snd_fn_s; @@ -1788,7 +1430,7 @@ static void list_remove(VSrc* vs) } } - debug_warn("VSrc not found"); + debug_assert(0); // VSrc not found } @@ -1941,7 +1583,8 @@ static LibError vsrc_update(VSrc* vs) alGetSourcei(vs->al_src, AL_BUFFERS_QUEUED, &num_queued); al_check("vsrc_update alGetSourcei"); - int num_processed = vsrc_deque_finished_bufs(vs); + const int num_processed = vsrc_deque_finished_bufs(vs); + UNUSED2(num_processed); if(vs->flags & VS_EOF) { @@ -1955,29 +1598,17 @@ static LibError vsrc_update(VSrc* vs) // can still read from SndData else { - // decide how many buffers we are going to request - // (start off with MAX_IOS; after that, replace finished buffers). - int to_fill = MAX_IOS; - if(num_queued > 0) - to_fill = num_processed; + // request and queue a buffer. + ALuint al_buf; + LibError ret = snd_data_buf_get(vs->hsd, al_buf); + if(ret < 0 && ret != ERR::AGAIN) + return ret; - // request and queue each buffer. - int ret; - do - { - ALuint al_buf; - ret = snd_data_buf_get(vs->hsd, al_buf); - // these 2 are legit (see above); otherwise, bail. - if(ret != ERR::AGAIN && ret != ERR::IO_EOF) - RETURN_ERR(ret); - - alSourceQueueBuffers(vs->al_src, 1, &al_buf); - al_check("vsrc_update alSourceQueueBuffers"); - } - while(to_fill-- && ret == INFO::OK); + alSourceQueueBuffers(vs->al_src, 1, &al_buf); + al_check("vsrc_update alSourceQueueBuffers"); // SndData has reported that no further buffers are available. - if(ret == ERR::IO_EOF) + if(ret == INFO::OK) vs->flags |= VS_EOF; } @@ -2448,35 +2079,17 @@ static LibError snd_init() } -/** - * (temporarily) disable all sound output. because it causes future snd_open - * calls to immediately abort before they demand-initialize OpenAL, - * startup is sped up considerably (500..1000ms). therefore, this must be - * called before the first snd_open to have any effect; otherwise, the - * cat will already be out of the bag and we debug_warn of it. - * - * rationale: this is a quick'n dirty way of speeding up startup during - * development without having to change the game's sound code. - * - * can later be called to reactivate sound; all settings ever changed - * will be applied and subsequent sound load / play requests will work. - * - * @param bool disabled - * @return LibError - */ LibError snd_disable(bool disabled) { snd_disabled = disabled; if(snd_disabled) { - if(al_initialized) - debug_warn("already initialized => disable is pointless"); + debug_assert(!al_initialized); // already initialized => disable is pointless return INFO::OK; } else - return snd_init(); - // note: won't return ERR::AGAIN, since snd_disabled == false + return snd_init(); // note: won't return ERR::AGAIN, since snd_disabled == false } @@ -2486,8 +2099,5 @@ LibError snd_disable(bool disabled) */ void snd_shutdown() { - io_buf_shutdown(); - - al_shutdown(); - // calls list_free_all + al_shutdown(); // calls list_free_all } diff --git a/source/lib/sysdep/arch.h b/source/lib/sysdep/arch.h new file mode 100644 index 0000000000..d866bea1b2 --- /dev/null +++ b/source/lib/sysdep/arch.h @@ -0,0 +1,52 @@ +/** + * ========================================================================= + * File : arch.h + * Project : 0 A.D. + * Description : CPU architecture detection. + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#ifndef INCLUDED_ARCH +#define INCLUDED_ARCH + +// detect target CPU architecture via predefined macros +// .. IA-32 +#if defined(_M_IX86) || defined(i386) || defined(_X86_) +# define ARCH_IA32 1 +#else +# define ARCH_IA32 0 +#endif +// .. IA-64 +#if defined(_M_IA64) || defined(__ia64__) +# define ARCH_IA64 1 +#else +# define ARCH_IA64 0 +#endif +// .. AMD64 +#if defined(_M_AMD64) || defined(__amd64__) || defined(__amd64) +# define ARCH_AMD64 1 +#else +# define ARCH_AMD64 0 +#endif +// .. Alpha +#if defined(_M_ALPHA) || defined(__alpha__) || defined(__alpha) +# define ARCH_ALPHA 1 +#else +# define ARCH_ALPHA 0 +#endif +// .. ARM +#if defined(__arm__) +# define ARCH_ARM 1 +#else +# define ARCH_ARM 0 +#endif +// .. MIPS +#if defined(__MIPS__) || defined(__mips__) || defined(__mips) +# define ARCH_MIPS 1 +#else +# define ARCH_MIPS 0 +#endif + +#endif // #ifndef INCLUDED_ARCH diff --git a/source/lib/sysdep/os.h b/source/lib/sysdep/os.h new file mode 100644 index 0000000000..77106aafdd --- /dev/null +++ b/source/lib/sysdep/os.h @@ -0,0 +1,97 @@ +/** + * ========================================================================= + * File : os.h + * Project : 0 A.D. + * Description : OS-specific macros + * ========================================================================= + */ + +// license: GPL; see lib/license.txt + +#ifndef INCLUDED_OS +#define INCLUDED_OS + +// detect OS via predefined macros. rationale: +// - these macros have consistent names and numerical values; using +// them saves other code from having to know the obscure predefined macros. +// - we'd like to use #if/#elif/#endif chains for e.g. OS_* to allow warning +// if none is selected, but there's no good way to #define all inapplicable +// settings to 0. doing so up front is hard to maintain and would require +// #undef before setting any one to 1. #ifndef afterwards for each setting +// is ugly and brittle as well. we therefore use #if/#else/#endif. + +// Windows +#if defined(_WIN64) +# define OS_WIN64 1 +#else +# define OS_WIN64 0 +#endif +#if defined(_WIN32) +# define OS_WIN 1 +#else +# define OS_WIN 0 +#endif +// Linux +#if defined(linux) || defined(__linux) || defined(__linux__) +# define OS_LINUX 1 +#else +# define OS_LINUX 0 +#endif +// Mac OS X +#if (defined(__APPLE__) && defined(__MACH__)) +# define OS_MACOSX 1 +#else +# define OS_MACOSX 0 +#endif +// BSD +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) +# define OS_BSD 1 +#else +# define OS_BSD 0 +#endif +// Solaris +#if defined(sun) || defined(__sun) +# define OS_SOLARIS 1 +#else +# define OS_SOLARIS 0 +#endif +// BeOS +#if defined(__BEOS__) +# define OS_BEOS 1 +#else +# define OS_BEOS 0 +#endif +// Mac OS 9 or below +#if defined(macintosh) +# define OS_MAC 1 +#else +# define OS_MAC 0 +#endif +// Amiga +#if defined(AMIGA) +# define OS_AMIGA 1 +#else +# define OS_AMIGA 0 +#endif +// Unix-based +#if defined(unix) || defined(__unix) || defined(__unix__) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) +# define OS_UNIX 1 +#else +# define OS_UNIX 0 +#endif + +// convenience: additionally set OS_UNIX for Unix-based OSes +// note: doing this in an separate section instead of adding the extra define +// to all affected OSes saves a few undefs or macro redefinition warnings. +#if OS_LINUX || OS_MACOSX || OS_BSD || OS_SOLARIS +# undef OS_UNIX +# define OS_UNIX 1 +#endif + +// convenience: additionally set OS_BSD for BSD-based OSes. see note above. +#if OS_MACOSX +# undef OS_BSD +# define OS_BSD 1 +#endif + +#endif // #ifndef INCLUDED_OS diff --git a/source/lib/sysdep/win/wposix/waio.cpp b/source/lib/sysdep/win/wposix/waio.cpp index 5111f982f4..a0d15c96d9 100644 --- a/source/lib/sysdep/win/wposix/waio.cpp +++ b/source/lib/sysdep/win/wposix/waio.cpp @@ -19,241 +19,72 @@ #include "lib/sysdep/cpu.h" #include "lib/bits.h" + WINIT_REGISTER_MAIN_INIT(waio_Init); WINIT_REGISTER_MAIN_SHUTDOWN(waio_Shutdown); - -static void lock() -{ - win_lock(WAIO_CS); -} - -static void unlock() -{ - win_unlock(WAIO_CS); -} - - -// return the largest sector size [bytes] of any storage medium -// (HD, optical, etc.) in the system. -// -// this may be a bit slow to determine (iterates over all drives), -// but caches the result so subsequent calls are free. -// (caveat: device changes won't be noticed during this program run) -// +// note: we assume sector sizes no larger than a page. +// (GetDiskFreeSpace allows querying the actual size, but we'd +// have to do so for all drives, and that may change depending on whether +// there is a DVD in the drive or not) // sector size is relevant because Windows aio requires all IO // buffers, offsets and lengths to be a multiple of it. this requirement // is also carried over into the vfs / file.cpp interfaces for efficiency // (avoids the need for copying to/from align buffers). -// -// waio uses the sector size to (in some cases) align IOs if -// they aren't already, but it's also needed by user code when -// aligning their buffers to meet the requirements. -// -// the largest size is used so that we can read from any drive. while this -// is a bit wasteful (more padding) and requires iterating over all drives, -// it is the only safe way: this may be called before we know which -// drives will be needed, and hardlinks may confuse things. -size_t sys_max_sector_size() +const uintptr_t sectorSize = 0x1000; + +//----------------------------------------------------------------------------- + +// note: the Windows lowio file descriptor limit is currrently 2048. + +/** + * thread-safe association between POSIX file descriptor and Win32 HANDLE + **/ +class HandleManager { - // users may call us more than once, so cache the results. - static DWORD cached_sector_size; - if(cached_sector_size) - return static_cast(cached_sector_size); - - // currently disabled: DVDs have 2..4KB, but this causes - // waio to unnecessarily align some file transfers (when at EOF) - // this means that we might not be able to read from CD/DVD drives - // (ReadFile will return error) - // reactivated for correctness. - - // temporarily disable the "insert disk into drive" error box; we are - // only interested in fixed drives anyway. - // - // note: use SetErrorMode (crappy interface, grr) twice so as not to - // stomp on other flags (e.g. alignment exception). - const UINT old_err_mode = SetErrorMode(0); - SetErrorMode(old_err_mode|SEM_FAILCRITICALERRORS); - - // find maximum of all drive's sector sizes. - const DWORD drives = GetLogicalDrives(); - char drive_str[4] = "?:\\"; - for(int drive = 2; drive <= 25; drive++) // C: .. Z: +public: + /** + * associate an aio handle with a file descriptor. + **/ + void Associate(int fd, HANDLE hFile) { - // avoid BoundsChecker warning by skipping invalid drives - if(!(drives & BIT(drive))) - continue; + debug_assert(fd > 2); + debug_assert(GetFileSize(hFile, 0) != INVALID_FILE_SIZE); - drive_str[0] = (char)('A'+drive); - - DWORD spc, nfc, tnc; // don't need these - DWORD cur_sector_size; - if(GetDiskFreeSpace(drive_str, &spc, &cur_sector_size, &nfc, &tnc)) - cached_sector_size = std::max(cached_sector_size, cur_sector_size); - // otherwise, it's probably an empty CD drive. ignore the - // BoundsChecker error; GetDiskFreeSpace seems to be the - // only way of getting at the sector size. + WinScopedLock lock; + std::pair ret = m_map.insert(std::make_pair(fd, hFile)); + debug_assert(ret.second); // fd better not already have been associated } - SetErrorMode(old_err_mode); - - // sanity check; believed to be the case for all drives. - debug_assert(cached_sector_size % 512 == 0); - - return cached_sector_size; -} - - -////////////////////////////////////////////////////////////////////////////// -// -// associate async-capable handle with POSIX file descriptor (int) -// -////////////////////////////////////////////////////////////////////////////// - -// current implementation: open file again for async access on each open(); -// wastes 1 HANDLE per file, but that's less overhead than storing the -// filename/mode for every file and re-opening that on demand. -// -// note: current Windows lowio file descriptor limit is 2k - -static HANDLE* aio_hs; - // array; expanded when needed in aio_h_set - -static int aio_hs_size; - // often compared against fd => int - - -// aio_h: no init needed. - -static void aio_h_cleanup() -{ - lock(); - - for(int i = 0; i < aio_hs_size; i++) + void Dissociate(int fd) { - if(aio_hs[i] != INVALID_HANDLE_VALUE) - { - WARN_IF_FALSE(CloseHandle(aio_hs[i])); - aio_hs[i] = INVALID_HANDLE_VALUE; - } + WinScopedLock lock; + const size_t numRemoved = m_map.erase(fd); + debug_assert(numRemoved == 1); } - SAFE_FREE(aio_hs); - aio_hs_size = 0; - - unlock(); -} - - -static bool is_valid_file_handle(const HANDLE h) -{ - const bool valid = (GetFileSize(h, 0) != INVALID_FILE_SIZE); - if(!valid) - debug_warn("waio: invalid file handle"); - return valid; -} - - -// return true iff an aio-capable HANDLE has been attached to . -// used by aio_close. -static bool aio_h_is_set(const int fd) -{ - lock(); - bool is_set = (0 <= fd && fd < aio_hs_size && aio_hs[fd] != INVALID_HANDLE_VALUE); - unlock(); - return is_set; -} - - -// return async-capable handle associated with file -static HANDLE aio_h_get(const int fd) -{ - HANDLE h = INVALID_HANDLE_VALUE; - - lock(); - - if(0 <= fd && fd < aio_hs_size) + /** + * @return aio handle associated with file descriptor or + * INVALID_HANDLE_VALUE if there is none. + **/ + HANDLE Get(int fd) const { - h = aio_hs[fd]; - if(!is_valid_file_handle(h)) - h = INVALID_HANDLE_VALUE; - } - else - debug_warn("aio_h_get: fd's aio handle not set"); - // h is already INVALID_HANDLE_VALUE - - unlock(); - - return h; -} - - -// associate h (an async-capable file handle) with fd; -// returned by subsequent aio_h_get(fd) calls. -// setting h = INVALID_HANDLE_VALUE removes the association. -static LibError aio_h_set(const int fd, const HANDLE h) -{ - if(fd < 0) - WARN_RETURN(ERR::INVALID_PARAM); - - lock(); - - LibError err; - - // grow hs array to at least fd+1 entries - if(fd >= aio_hs_size) - { - const uint size2 = (uint)round_up(fd+8, 8); - HANDLE* hs2 = (HANDLE*)realloc(aio_hs, size2*sizeof(HANDLE)); - if(!hs2) - { - err = ERR::NO_MEM; - goto fail; - } - // don't assign directly from realloc - - // we'd leak the previous array if realloc fails. - - for(uint i = aio_hs_size; i < size2; i++) - hs2[i] = INVALID_HANDLE_VALUE; - aio_hs = hs2; - aio_hs_size = size2; + WinScopedLock lock; + Map::const_iterator it = m_map.find(fd); + if(it == m_map.end()) + return INVALID_HANDLE_VALUE; + return it->second; } +private: + typedef std::map Map; + Map m_map; +}; - if(h == INVALID_HANDLE_VALUE) - { - // nothing to do; will set aio_hs[fd] to INVALID_HANDLE_VALUE below. - } - else - { - // already set - if(aio_hs[fd] != INVALID_HANDLE_VALUE) - { - err = ERR::LOGIC; - goto fail; - } - // setting invalid handle - if(!is_valid_file_handle(h)) - { - err = ERR::INVALID_HANDLE; - goto fail; - } - } - - aio_hs[fd] = h; - - unlock(); - return INFO::OK; - -fail: - unlock(); - WARN_RETURN(err); -} +static HandleManager* handleManager; - - -// open fn in async mode; associate with fd (retrieve via aio_h(fd)) +// open fn in async mode and associate handle with fd. int aio_reopen(int fd, const char* fn, int oflag, ...) { // interpret oflag @@ -276,52 +107,34 @@ int aio_reopen(int fd, const char* fn, int oflag, ...) // open file DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_SEQUENTIAL_SCAN; WIN_SAVE_LAST_ERROR; // CreateFile - HANDLE h = CreateFile(fn, access, share, 0, create, flags, 0); + HANDLE hFile = CreateFile(fn, access, share, 0, create, flags, 0); WIN_RESTORE_LAST_ERROR; - if(h == INVALID_HANDLE_VALUE) - goto fail; - - if(aio_h_set(fd, h) < 0) - { - CloseHandle(h); - goto fail; - } + if(hFile == INVALID_HANDLE_VALUE) + WARN_RETURN(-1); + handleManager->Associate(fd, hFile); return 0; - -fail: - debug_warn("failed"); - return -1; } int aio_close(int fd) { + HANDLE hFile = handleManager->Get(fd); // early out for files that were never re-opened for AIO. // since there is no way for wposix close to know this, we mustn't // return an error (which would cause it to WARN_ERR). - if(!aio_h_is_set(fd)) + if(hFile == INVALID_HANDLE_VALUE) return 0; - HANDLE h = aio_h_get(fd); - // out of bounds or already closed - if(h == INVALID_HANDLE_VALUE) - goto fail; + if(!CloseHandle(hFile)) + WARN_RETURN(-1); - if(!CloseHandle(h)) - goto fail; - RETURN_ERR(aio_h_set(fd, INVALID_HANDLE_VALUE)); + handleManager->Dissociate(fd); return 0; - -fail: - debug_warn("failed"); - return -1; } - - // do we want to open a second AIO-capable handle? static bool isAioPossible(int fd, bool is_com_port, int oflag) { @@ -420,398 +233,204 @@ off_t lseek(int fd, off_t ofs, int whence) return _lseek(fd, ofs, whence); } - -////////////////////////////////////////////////////////////////////////////// -// -// Req -// -////////////////////////////////////////////////////////////////////////////// - - -// information about active transfers (reused) -struct Req +int truncate(const char* path, off_t length) { - // used to identify this request; != 0 <==> request valid. - // set by req_alloc. - aiocb* cb; - - OVERLAPPED ovl; - // hEvent signals when transfer complete - - // align buffer - unaligned reads are padded to sector boundaries and - // go here; the desired data is then copied into the user's buffer. - // reused, since the Req has global lifetime; resized if too small. - void* buf; - size_t buf_size; - - HANDLE hFile; - // needed to GetOverlappedResult in aio_return - - size_t pad; // offset from starting sector - bool read_into_align_buffer; -}; - - -// an aiocb is used to pass the request from caller to aio, -// and serves as a "token" identifying the IO - its address is unique. -// Req holds some state needed for the Windows AIO calls (OVERLAPPED). -// -// cb -> req (e.g. in aio_return) is accomplished by searching reqs -// for the given cb (no problem since MAX_REQS is small). -// req stores a pointer to its associated cb. - - -const int MAX_REQS = 8; -static Req reqs[MAX_REQS]; - - -static void req_cleanup(void) -{ - Req* r = reqs; - - for(int i = 0; i < MAX_REQS; i++, r++) + HANDLE hFile = CreateFile(path, GENERIC_WRITE, 0, 0, OPEN_EXISTING, 0, 0); + debug_assert(hFile != INVALID_HANDLE_VALUE); + LARGE_INTEGER ofs; ofs.QuadPart = length; + BOOL ok; + ok = SetFilePointerEx(hFile, ofs, 0, FILE_BEGIN); + debug_assert(ok); + ok = SetEndOfFile(hFile); + debug_assert(ok); + ok = CloseHandle(hFile); + debug_assert(ok); { - HANDLE& h = r->ovl.hEvent; - debug_assert(h != INVALID_HANDLE_VALUE); - CloseHandle(h); - h = INVALID_HANDLE_VALUE; - - _aligned_free(r->buf); - r->buf = 0; + // LibError_set_errno(LibError_from_GLE()); + // WARN_RETURN(-1); } -} - - -static void req_init() -{ - for(int i = 0; i < MAX_REQS; i++) - reqs[i].ovl.hEvent = CreateEvent(0,1,0,0); // manual reset - - // buffers are allocated on-demand. -} - - -// return first Req with given cb field -// (0 if searching for a free Req) -static Req* req_find(const aiocb* cb) -{ - Req* r = reqs; - for(int i = 0; i < MAX_REQS; i++, r++) - if(r->cb == cb) - return r; - - // not found return 0; } -static Req* req_alloc(aiocb* cb) +//----------------------------------------------------------------------------- + +class aiocb::Impl { - debug_assert(cb); +public: + Impl() + { + m_hFile = INVALID_HANDLE_VALUE; + const BOOL manualReset = TRUE; + const BOOL initialState = FALSE; + m_overlapped.hEvent = CreateEvent(0, manualReset, initialState, 0); + } - // first free Req, or 0 - Req* r = req_find(0); - // .. found one: mark it in-use - if(r) - r->cb = cb; + ~Impl() + { + CloseHandle(m_overlapped.hEvent); + } - return r; -} + LibError Issue(HANDLE hFile, off_t ofs, void* buf, size_t size, bool isWrite) + { + m_hFile = hFile; + // note: Read-/WriteFile reset m_overlapped.hEvent, so we don't have to. + m_overlapped.Internal = m_overlapped.InternalHigh = 0; + m_overlapped.Offset = u64_lo(ofs); + m_overlapped.OffsetHigh = u64_hi(ofs); -static LibError req_free(Req* r) -{ - debug_assert(r->cb != 0 && "req_free: not currently in use"); - r->cb = 0; - return INFO::OK; -} + DWORD bytesTransferred; + BOOL ok; + WIN_SAVE_LAST_ERROR; + if(isWrite) + ok = WriteFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped); + else + ok = ReadFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped); + if(!ok && GetLastError() == ERROR_IO_PENDING) // "pending" isn't an error + { + ok = TRUE; + SetLastError(0); + } + WIN_RESTORE_LAST_ERROR; + return LibError_from_win32(ok); + } + bool HasCompleted() const + { + debug_assert(m_overlapped.Internal == 0 || m_overlapped.Internal == STATUS_PENDING); + return HasOverlappedIoCompleted(&m_overlapped); + } + // required for WaitForMultipleObjects + HANDLE Event() const + { + return m_overlapped.hEvent; + } + + LibError GetResult(size_t* pBytesTransferred) + { + DWORD bytesTransferred; + const BOOL wait = FALSE; // callers should wait until HasCompleted + if(!GetOverlappedResult(m_hFile, &m_overlapped, &bytesTransferred, wait)) + { + *pBytesTransferred = 0; + return LibError_from_GLE(); + } + else + { + *pBytesTransferred = bytesTransferred; + return INFO::OK; + } + } + +private: + OVERLAPPED m_overlapped; + HANDLE m_hFile; +}; // called by aio_read, aio_write, and lio_listio // cb->aio_lio_opcode specifies desired operation -// -// if cb->aio_fildes doesn't support seeking (e.g. a socket), -// cb->aio_offset must be 0. -static int aio_rw(struct aiocb* cb) +static int aio_issue(struct aiocb* cb) { - int ret = -1; - Req* r = 0; - - WIN_SAVE_LAST_ERROR; - // no-op from lio_listio if(!cb || cb->aio_lio_opcode == LIO_NOP) return 0; - // fail if aiocb is already in use (forbidden by SUSv3) - if(req_find(cb)) - { - debug_warn("aiocb is already in use"); - goto fail; - } - // extract aiocb fields for convenience - const bool is_write = (cb->aio_lio_opcode == LIO_WRITE); - const int fd = cb->aio_fildes; - const size_t size = cb->aio_nbytes; - const off_t ofs = cb->aio_offset; - void* buf = (void*)cb->aio_buf; // from volatile void* - debug_assert(buf); + const bool isWrite = (cb->aio_lio_opcode == LIO_WRITE); + const int fd = cb->aio_fildes; + const size_t size = cb->aio_nbytes; + const off_t ofs = cb->aio_offset; + void* const buf = (void*)cb->aio_buf; // from volatile void* - // allocate IO request - r = req_alloc(cb); - if(!r) + // Win32 requires transfers to be sector-aligned. + if(!IsAligned(ofs, sectorSize) || !IsAligned(buf, sectorSize) || !IsAligned(size, sectorSize)) + WARN_RETURN(-EINVAL); + + const HANDLE hFile = handleManager->Get(fd); + if(hFile == INVALID_HANDLE_VALUE) { - debug_warn("cannot allocate a Req (too many concurrent IOs)"); - goto fail; + errno = -EINVAL; + WARN_RETURN(-1); } - HANDLE h = aio_h_get(fd); - if(h == INVALID_HANDLE_VALUE) + debug_assert(!cb->impl); // fail if aiocb is already in use (forbidden by SUSv3) + cb->impl.reset(new aiocb::Impl); + LibError ret = cb->impl->Issue(hFile, ofs, buf, size, isWrite); + if(ret < 0) { - debug_warn("associated handle is invalid"); - ret = -EINVAL; - goto fail; + LibError_set_errno(ret); + WARN_RETURN(-1); } - - - r->hFile = h; - r->pad = 0; - r->read_into_align_buffer = false; - - - // - // align - // - - // Win32 requires transfers to be sector aligned. - // we check if the transfer is aligned to sector size (the max of - // all drives in the system) and copy to/from align buffer if not. - - // actual transfer parameters (possibly rounded up/down) - size_t actual_ofs = 0; - // assume socket; if file, set below - size_t actual_size = size; - void* actual_buf = buf; - - const size_t sector_size = sys_max_sector_size(); - - // leave offset 0 if h is a socket (don't support seeking); - // otherwise, calculate aligned offset/size - const bool is_file = (GetFileType(h) == FILE_TYPE_DISK); - if(is_file) - { - // round offset down to start of previous sector, and total - // transfer size up to an integral multiple of sector_size. - r->pad = ofs % sector_size; - actual_ofs = ofs - r->pad; - actual_size = round_up(size + r->pad, sector_size); - - // and whether it was ofs or buf in particular - // (needed for unaligned write handling below). - const bool ofs_misaligned = r->pad != 0; - const bool buf_misaligned = (uintptr_t)buf % sector_size != 0; - const bool misaligned = ofs_misaligned || buf_misaligned || actual_size != size; - // note: actual_size != size if ofs OR size is unaligned - - // misaligned => will need to go through align buffer - // (we fail some types of misalignment for convenience; see below). - if(misaligned) - { - // expand current align buffer if too small - if(r->buf_size < actual_size) - { - void* buf2 = _aligned_realloc(r->buf, actual_size, sector_size); - if(!buf2) - { - ret = -ENOMEM; - goto fail; - } - r->buf = buf2; - r->buf_size = actual_size; - } - - if(!is_write) - { - actual_buf = r->buf; - r->read_into_align_buffer = true; - } - else - { - // unaligned write offset: not supported. - // (we'd have to read padding, then write our data. ugh.) - if(ofs_misaligned) - { - ret = -EINVAL; - goto fail; - } - - // unaligned buffer: copy to align buffer and write from there. - if(buf_misaligned) - { - cpu_memcpy(r->buf, buf, size); - actual_buf = r->buf; - // clear previous contents at end of align buf - memset((char*)r->buf + size, 0, actual_size - size); - } - - // unaligned size: already taken care of (we round up) - } - } // misaligned - } // is_file - - // set OVERLAPPED fields - // note: Read-/WriteFile reset ovl.hEvent - no need to do that. - r->ovl.Internal = r->ovl.InternalHigh = 0; - // note: OVERLAPPED.Pointer is more convenient but not defined on VC6. - r->ovl.Offset = u64_lo(actual_ofs); - r->ovl.OffsetHigh = u64_hi(actual_ofs); - - DWORD size32 = (DWORD)(actual_size & 0xFFFFFFFF); - BOOL ok; - - DWORD bytes_transferred; - if(is_write) - ok = WriteFile(h, actual_buf, size32, &bytes_transferred, &r->ovl); - else - ok = ReadFile(h, actual_buf, size32, &bytes_transferred, &r->ovl); - - // check result. - // .. "pending" isn't an error - if(!ok && GetLastError() == ERROR_IO_PENDING) - ok = TRUE; - // .. translate from Win32 result code to POSIX - LibError err = LibError_from_win32(ok); - if(err == INFO::OK) - ret = 0; - LibError_set_errno(err); - -done: - WIN_RESTORE_LAST_ERROR; - - return ret; - -fail: - debug_warn("waio failure"); - req_free(r); - goto done; + return 0; } // return status of transfer int aio_error(const struct aiocb* cb) { - // must not pass 0 to req_find - we'd look for a free cb! - if(!cb) - { - debug_warn("invalid cb"); - return -1; - } - - Req* r = req_find(cb); - if(!r) - return -1; - - switch(r->ovl.Internal) // I/O status - { - case 0: - return 0; - case STATUS_PENDING: - return EINPROGRESS; - default: - return -1; - } + return cb->impl->HasCompleted()? 0 : EINPROGRESS; } // get bytes transferred. call exactly once for each op. ssize_t aio_return(struct aiocb* cb) { - // must not pass 0 to req_find - we'd look for a free cb! - if(!cb) + debug_assert(cb->impl->HasCompleted()); // SUSv3 says we mustn't be callable before IO completes + size_t bytesTransferred; + LibError ret = cb->impl->GetResult(&bytesTransferred); + cb->impl.reset(); // disallow calling again, as required by SUSv3 + if(ret < 0) { - debug_warn("invalid cb"); - return -1; + LibError_set_errno(ret); + WARN_RETURN(-1); } - - Req* r = req_find(cb); - if(!r) - { - debug_warn("cb not found (already called aio_return?)"); - return -1; - } - - debug_assert(r->ovl.Internal == 0 && "aio_return with transfer in progress"); - - const BOOL wait = FALSE; // should already be done! - DWORD bytes_transferred; - if(!GetOverlappedResult(r->hFile, &r->ovl, &bytes_transferred, wait)) - { - debug_warn("GetOverlappedResult failed"); - return -1; - } - - // we read into align buffer - copy to user's buffer - if(r->read_into_align_buffer) - cpu_memcpy((void*)cb->aio_buf, (u8*)r->buf + r->pad, cb->aio_nbytes); - - // TODO: this copies data back into original buffer from align buffer - // when writing from unaligned buffer. unnecessarily slow. - - req_free(r); - - return (ssize_t)bytes_transferred; + return (ssize_t)bytesTransferred; } int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* ts) { - int i; - if(n <= 0 || n > MAXIMUM_WAIT_OBJECTS) - return -1; + WARN_RETURN(-1); - int cnt = 0; // actual number of valid cbs - HANDLE hs[MAXIMUM_WAIT_OBJECTS]; - - for(i = 0; i < n; i++) + // build array of event handles + HANDLE hEvents[MAXIMUM_WAIT_OBJECTS]; + unsigned numPendingIos = 0; + for(int i = 0; i < n; i++) { - // ignore NULL list entries - if(!cbs[i]) + if(!cbs[i]) // SUSv3 says NULL entries are to be ignored continue; - Req* r = req_find(cbs[i]); - if(r) - { - if(r->ovl.Internal == STATUS_PENDING) - hs[cnt++] = r->ovl.hEvent; - } + aiocb::Impl* impl = cbs[i]->impl.get(); + if(!impl->HasCompleted()) + hEvents[numPendingIos++] = impl->Event(); } - - // no valid, pending transfers - done - if(!cnt) + if(!numPendingIos) // done, don't need to suspend. return 0; - // timeout: convert timespec to ms (NULL ptr -> no timeout) - DWORD timeout = INFINITE; - if(ts) - timeout = (DWORD)(ts->tv_sec*1000 + ts->tv_nsec/1000000); + const BOOL waitAll = FALSE; + // convert timespec to milliseconds (ts == 0 => no timeout) + const DWORD timeout = ts? (DWORD)(ts->tv_sec*1000 + ts->tv_nsec/1000000) : INFINITE; + DWORD result = WaitForMultipleObjects(numPendingIos, hEvents, waitAll, timeout); - const BOOL wait_all = FALSE; - DWORD result = WaitForMultipleObjects(cnt, hs, wait_all, timeout); + for(unsigned i = 0; i < numPendingIos; i++) + ResetEvent(hEvents[i]); - for(i = 0; i < cnt; i++) - ResetEvent(hs[i]); - - if(result == WAIT_TIMEOUT) + switch(result) { - //errno = -EAGAIN; + case WAIT_FAILED: + WARN_RETURN(-1); + + case WAIT_TIMEOUT: + errno = -EAGAIN; return -1; + + default: + return 0; } - else - return (result == WAIT_FAILED)? -1 : 0; } @@ -821,49 +440,36 @@ int aio_cancel(int fd, struct aiocb* cb) // all pending reads on this file are cancelled. UNUSED2(cb); - const HANDLE h = aio_h_get(fd); - if(h == INVALID_HANDLE_VALUE) - return -1; + const HANDLE hFile = handleManager->Get(fd); + if(hFile == INVALID_HANDLE_VALUE) + WARN_RETURN(-1); - CancelIo(h); + CancelIo(hFile); return AIO_CANCELED; } - - int aio_read(struct aiocb* cb) { cb->aio_lio_opcode = LIO_READ; - return aio_rw(cb); // checks for cb == 0 + return aio_issue(cb); } int aio_write(struct aiocb* cb) { cb->aio_lio_opcode = LIO_WRITE; - return aio_rw(cb); // checks for cb == 0 + return aio_issue(cb); } -int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se) +int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* UNUSED(se)) { - UNUSED2(se); - - int err = 0; - for(int i = 0; i < n; i++) - { - int ret = aio_rw(cbs[i]); // checks for cbs[i] == 0 - // don't RETURN_ERR yet - we want to try to issue each one - if(ret < 0 && !err) - err = ret; - } - - RETURN_ERR(err); + RETURN_ERR(aio_issue(cbs[i])); if(mode == LIO_WAIT) - return aio_suspend(cbs, n, 0); + RETURN_ERR(aio_suspend(cbs, n, 0)); return 0; } @@ -871,27 +477,20 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se) int aio_fsync(int, struct aiocb*) { - return -ENOSYS; + WARN_RETURN(-ENOSYS); } -////////////////////////////////////////////////////////////////////////////// -// -// init / cleanup -// -////////////////////////////////////////////////////////////////////////////// - +//----------------------------------------------------------------------------- static LibError waio_Init() { - req_init(); + handleManager = new HandleManager; return INFO::OK; } - static LibError waio_Shutdown() { - req_cleanup(); - aio_h_cleanup(); + delete handleManager; return INFO::OK; } diff --git a/source/lib/sysdep/win/wposix/waio.h b/source/lib/sysdep/win/wposix/waio.h index 49805f2d9e..f2a846ac50 100644 --- a/source/lib/sysdep/win/wposix/waio.h +++ b/source/lib/sysdep/win/wposix/waio.h @@ -71,6 +71,7 @@ extern int close(int); extern int read (int fd, void* buf, size_t nbytes); // thunk extern int write(int fd, void* buf, size_t nbytes); // thunk extern off_t lseek(int fd, off_t ofs, int whence); // thunk +extern int truncate(const char* path, off_t length); // @@ -86,6 +87,9 @@ struct aiocb int aio_reqprio; // Request priority offset. struct sigevent aio_sigevent; // Signal number and value. int aio_lio_opcode; // Operation to be performed. + + class Impl; + shared_ptr impl; }; enum diff --git a/source/lib/res/graphics/tex.cpp b/source/lib/tex/tex.cpp similarity index 81% rename from source/lib/res/graphics/tex.cpp rename to source/lib/tex/tex.cpp index 1ab65c44d8..84923d9d14 100644 --- a/source/lib/res/graphics/tex.cpp +++ b/source/lib/tex/tex.cpp @@ -17,10 +17,13 @@ #include "lib/timer.h" #include "lib/bits.h" -#include "lib/res/res.h" #include "tex_codec.h" +#include "lib/file/vfs/vfs.h" +#include "lib/file/io/io_internal.h" +extern PIVFS g_VFS; + ERROR_ASSOCIATE(ERR::TEX_FMT_INVALID, "Invalid/unsupported texture format", -1); ERROR_ASSOCIATE(ERR::TEX_INVALID_COLOR_TYPE, "Invalid color type", -1); @@ -39,34 +42,34 @@ ERROR_ASSOCIATE(INFO::TEX_CODEC_CANNOT_HANDLE, "Texture codec cannot handle the // be careful not to use other tex_* APIs here because they call us. LibError tex_validate(const Tex* t) { - // pixel data - size_t tex_file_size; - void* tex_file = mem_get_ptr(t->hm, &tex_file_size); - // .. only check validity if the image is still in memory. - // (e.g. ogl_tex frees the data after uploading to GL) - if(tex_file) + if(t->flags & TEX_UNDEFINED_FLAGS) + WARN_RETURN(ERR::_1); + + // pixel data (only check validity if the image is still in memory; + // ogl_tex frees the data after uploading to GL) + if(t->data) { // file size smaller than header+pixels. // possible causes: texture file header is invalid, // or file wasn't loaded completely. - if(tex_file_size < t->ofs + t->w*t->h*t->bpp/8) - WARN_RETURN(ERR::_1); + if(t->dataSize < t->ofs + t->w*t->h*t->bpp/8) + WARN_RETURN(ERR::_2); } // bits per pixel // (we don't bother checking all values; a sanity check is enough) if(t->bpp % 4 || t->bpp > 32) - WARN_RETURN(ERR::_2); + WARN_RETURN(ERR::_3); // flags // .. DXT value const uint dxt = t->flags & TEX_DXT; if(dxt != 0 && dxt != 1 && dxt != DXT1A && dxt != 3 && dxt != 5) - WARN_RETURN(ERR::_3); + WARN_RETURN(ERR::_4); // .. orientation const uint orientation = t->flags & TEX_ORIENTATION; if(orientation == (TEX_BOTTOM_UP|TEX_TOP_DOWN)) - WARN_RETURN(ERR::_4); + WARN_RETURN(ERR::_5); return INFO::OK; } @@ -109,18 +112,17 @@ LibError tex_validate_plain_format(uint bpp, uint flags) // mipmaps //----------------------------------------------------------------------------- -void tex_util_foreach_mipmap(uint w, uint h, uint bpp, const u8* RESTRICT data, - int levels_to_skip, uint data_padding, MipmapCB cb, void* RESTRICT cbData) +void tex_util_foreach_mipmap(uint w, uint h, uint bpp, const u8* pixels, int levels_to_skip, uint data_padding, MipmapCB cb, void* RESTRICT cbData) { + debug_assert(levels_to_skip >= 0 || levels_to_skip == TEX_BASE_LEVEL_ONLY); + uint level_w = w, level_h = h; - const u8* level_data = data; + const u8* level_data = pixels; // we iterate through the loop (necessary to skip over image data), // but do not actually call back until the requisite number of // levels have been skipped (i.e. level == 0). - int level = -(int)levels_to_skip; - if(levels_to_skip == -1) - level = 0; + int level = (levels_to_skip == TEX_BASE_LEVEL_ONLY)? 0 : -levels_to_skip; // until at level 1x1: for(;;) @@ -164,8 +166,7 @@ struct CreateLevelData }; // uses 2x2 box filter -static void create_level(uint level, uint level_w, uint level_h, - const u8* RESTRICT level_data, size_t level_data_size, void* RESTRICT cbData) +static void create_level(uint level, uint level_w, uint level_h, const u8* RESTRICT level_data, size_t level_data_size, void* RESTRICT cbData) { CreateLevelData* cld = (CreateLevelData*)cbData; const size_t src_w = cld->prev_level_w; @@ -232,8 +233,7 @@ static void create_level(uint level, uint level_w, uint level_h, } -static LibError add_mipmaps(Tex* t, uint w, uint h, uint bpp, - void* new_data, size_t data_size) +static LibError add_mipmaps(Tex* t, uint w, uint h, uint bpp, void* newData, size_t data_size) { // this code assumes the image is of POT dimension; we don't // go to the trouble of implementing image scaling because @@ -242,14 +242,11 @@ static LibError add_mipmaps(Tex* t, uint w, uint h, uint bpp, WARN_RETURN(ERR::TEX_INVALID_SIZE); t->flags |= TEX_MIPMAPS; // must come before tex_img_size! const size_t mipmap_size = tex_img_size(t); - Handle hm; - const u8* mipmap_data = (const u8*)mem_alloc(mipmap_size, 4*KiB, 0, &hm); - if(!mipmap_data) - WARN_RETURN(ERR::NO_MEM); - CreateLevelData cld = { bpp/8, w, h, (const u8*)new_data, data_size }; - tex_util_foreach_mipmap(w, h, bpp, mipmap_data, 0, 1, create_level, &cld); - mem_free_h(t->hm); - t->hm = hm; + shared_ptr mipmapData = io_Allocate(mipmap_size, 0); + CreateLevelData cld = { bpp/8, w, h, (const u8*)newData, data_size }; + tex_util_foreach_mipmap(w, h, bpp, mipmapData.get(), 0, 1, create_level, &cld); + t->data = mipmapData; + t->dataSize = mipmap_size; t->ofs = 0; return INFO::OK; @@ -297,17 +294,14 @@ TIMER_ACCRUE(tc_plain_transform); // line buffer leads to thrashing. we'll assume the whole texture*2 // fits in cache, allocate a copy, and transfer directly from there. // - // this is necessary even when not flipping because the initial Tex.hm - // (which is a FileIOBuf) is read-only. - Handle hm; - void* new_data = mem_alloc(data_size, 4*KiB, 0, &hm); - if(!new_data) - WARN_RETURN(ERR::NO_MEM); - cpu_memcpy(new_data, data, data_size); + // this is necessary even when not flipping because the initial data + // is read-only. + shared_ptr newData = io_Allocate(data_size); + cpu_memcpy(newData.get(), data, data_size); // setup row source/destination pointers (simplifies outer loop) - u8* dst = (u8*)new_data; - const u8* src = (const u8*)new_data; + u8* dst = (u8*)newData.get(); + const u8* src = (const u8*)newData.get(); const size_t pitch = w * bpp/8; // .. avoid y*pitch multiply in row loop; instead, add row_ofs. ssize_t row_ofs = (ssize_t)pitch; @@ -362,12 +356,12 @@ TIMER_ACCRUE(tc_plain_transform); } } - mem_free_h(t->hm); - t->hm = hm; + t->data = newData; + t->dataSize = data_size; t->ofs = 0; if(!(t->flags & TEX_MIPMAPS) && transforms & TEX_MIPMAPS) - RETURN_ERR(add_mipmaps(t, w, h, bpp, new_data, data_size)); + RETURN_ERR(add_mipmaps(t, w, h, bpp, newData.get(), data_size)); CHECK_TEX(t); return INFO::OK; @@ -500,29 +494,18 @@ bool tex_is_known_extension(const char* filename) // however, we don't want to provide an alternate interface for each API; // these would have to be changed whenever fields are added to Tex. // instead, provide one entry point for specifying images. -// note: since we do not know how was allocated, the caller must do -// so (after calling tex_free, which is required regardless of alloc type). // // we need only add bookkeeping information and "wrap" it in // our Tex struct, hence the name. -LibError tex_wrap(uint w, uint h, uint bpp, uint flags, void* img, Tex* t) +LibError tex_wrap(uint w, uint h, uint bpp, uint flags, shared_ptr data, size_t ofs, Tex* t) { t->w = w; t->h = h; t->bpp = bpp; t->flags = flags; - - // note: we can't use tex_img_size because that requires all - // Tex fields to be valid, but this calculation must be done first. - const size_t img_size = w*h*bpp/8; - t->hm = mem_wrap(img, img_size, 0, 0, 0, 0, 0, (void*)&tex_wrap); - RETURN_ERR(t->hm); - - // the exact value of img is lost, since the handle references the - // allocation and disregards the offset within it given by . - // fix that up by setting t->ofs. - void* reported_ptr = mem_get_ptr(t->hm); - t->ofs = (u8*)img - (u8*)reported_ptr; + t->data = data; + t->dataSize = w * h * bpp / 8; + t->ofs = ofs; CHECK_TEX(t); return INFO::OK; @@ -531,17 +514,16 @@ LibError tex_wrap(uint w, uint h, uint bpp, uint flags, void* img, Tex* t) // free all resources associated with the image and make further // use of it impossible. -LibError tex_free(Tex* t) +void tex_free(Tex* t) { // do not validate - this is called from tex_load if loading // failed, so not all fields may be valid. - LibError ret = mem_free_h(t->hm); + t->data.reset(); // do not zero out the fields! that could lead to trouble since // ogl_tex_upload followed by ogl_tex_free is legit, but would // cause OglTex_validate to fail (since its Tex.w is == 0). - return ret; } @@ -550,21 +532,20 @@ LibError tex_free(Tex* t) //----------------------------------------------------------------------------- // returns a pointer to the image data (pixels), taking into account any -// header(s) that may come before it. see Tex.hm comment above. +// header(s) that may come before it. u8* tex_get_data(const Tex* t) { // (can't use normal CHECK_TEX due to u8* return value) WARN_ERR(tex_validate(t)); - u8* p = (u8*)mem_get_ptr(t->hm); + u8* p = t->data.get(); if(!p) return 0; return p + t->ofs; } -static void add_level_size(uint UNUSED(level), uint UNUSED(level_w), uint UNUSED(level_h), - const u8* RESTRICT UNUSED(level_data), size_t level_data_size, void* RESTRICT cbData) +static void add_level_size(uint UNUSED(level), uint UNUSED(level_w), uint UNUSED(level_h), const u8* RESTRICT UNUSED(level_data), size_t level_data_size, void* RESTRICT cbData) { size_t* ptotal_size = (size_t*)cbData; *ptotal_size += level_data_size; @@ -581,8 +562,7 @@ size_t tex_img_size(const Tex* t) const int levels_to_skip = (t->flags & TEX_MIPMAPS)? 0 : TEX_BASE_LEVEL_ONLY; const uint data_padding = (t->flags & TEX_DXT)? 4 : 1; size_t out_size = 0; - tex_util_foreach_mipmap(t->w, t->h, t->bpp, 0, levels_to_skip, - data_padding, add_level_size, &out_size); + tex_util_foreach_mipmap(t->w, t->h, t->bpp, 0, levels_to_skip, data_padding, add_level_size, &out_size); return out_size; } @@ -606,31 +586,27 @@ size_t tex_hdr_size(const char* fn) // read/write from memory and disk //----------------------------------------------------------------------------- -LibError tex_decode(const u8* data, size_t data_size, MEM_DTOR dtor, Tex* t) +LibError tex_decode(shared_ptr data, size_t data_size, Tex* t) { const TexCodecVTbl* c; - RETURN_ERR(tex_codec_for_header(data, data_size, &c)); + RETURN_ERR(tex_codec_for_header(data.get(), data_size, &c)); // make sure the entire header is available const size_t min_hdr_size = c->hdr_size(0); if(data_size < min_hdr_size) - WARN_RETURN(ERR::RES_INCOMPLETE_HEADER); - const size_t hdr_size = c->hdr_size(data); + WARN_RETURN(ERR::TEX_INCOMPLETE_HEADER); + const size_t hdr_size = c->hdr_size(data.get()); if(data_size < hdr_size) - WARN_RETURN(ERR::RES_INCOMPLETE_HEADER); + WARN_RETURN(ERR::TEX_INCOMPLETE_HEADER); - // wrap pointer into a Handle; required for Tex.hm. - // rationale: a Handle protects the texture memory from being - // accidentally free-d. - Handle hm = mem_wrap((void*)data, data_size, 0, 0, 0, dtor, 0, (void*)tex_decode); - - t->hm = hm; + t->data = data; + t->dataSize = data_size; t->ofs = hdr_size; // for orthogonality, encode and decode both receive the memory as a // DynArray. package data into one and free it again after decoding: DynArray da; - RETURN_ERR(da_wrap_fixed(&da, (u8*)data, data_size)); + RETURN_ERR(da_wrap_fixed(&da, data.get(), data_size)); RETURN_ERR(c->decode(&da, t)); @@ -641,10 +617,7 @@ LibError tex_decode(const u8* data, size_t data_size, MEM_DTOR dtor, Tex* t) // sanity checks if(!t->w || !t->h || t->bpp > 32) WARN_RETURN(ERR::TEX_FMT_INVALID); - // .. note: can't use data_size - decode may have decompressed the image. - size_t hm_size; - (void)mem_get_ptr(t->hm, &hm_size); - if(hm_size < t->ofs + tex_img_size(t)) + if(t->dataSize < t->ofs + tex_img_size(t)) WARN_RETURN(ERR::TEX_INVALID_SIZE); flip_to_global_orientation(t); @@ -660,7 +633,7 @@ LibError tex_encode(Tex* t, const char* fn, DynArray* da) // we could be clever here and avoid the extra alloc if our current // memory block ensued from the same kind of texture file. this is - // most likely the case if in_img == + c->hdr_size(0). + // most likely the case if in_img == tex_get_data() + c->hdr_size(0). // this would make for zero-copy IO. const size_t max_out_size = tex_img_size(t)*4 + 256*KiB; @@ -681,29 +654,22 @@ LibError tex_encode(Tex* t, const char* fn, DynArray* da) } - -// MEM_DTOR -> file_buf_free adapter (used for mem_wrap-ping FileIOBuf) -static void file_buf_dtor(void* p, size_t UNUSED(size), uintptr_t UNUSED(ctx)) -{ - (void)file_buf_free((FileIOBuf)p); -} - // load the specified image from file into the given Tex object. // currently supports BMP, TGA, JPG, JP2, PNG, DDS. -LibError tex_load(const char* fn, Tex* t, uint file_flags) +LibError tex_load(const char* fn, Tex* t) { // load file - FileIOBuf file; size_t file_size; - RETURN_ERR(vfs_load(fn, file, file_size, file_flags)); + shared_ptr file; size_t file_size; + RETURN_ERR(g_VFS->LoadFile(fn, file, file_size)); - LibError ret = tex_decode(file, file_size, file_buf_dtor, t); + LibError ret = tex_decode(file, file_size, t); if(ret < 0) { - (void)tex_free(t); - WARN_RETURN(ret); + tex_free(t); + RETURN_ERR(ret); } - // do not free hm! it either still holds the image data (i.e. texture + // do not free data! it either still holds the image data (i.e. texture // wasn't compressed) or was replaced by a new buffer for the image data. CHECK_TEX(t); @@ -722,9 +688,9 @@ LibError tex_write(Tex* t, const char* fn) // write to disk LibError ret = INFO::OK; { - const size_t sector_aligned_size = round_up(da.cur_size, file_sector_size); - (void)da_set_size(&da, sector_aligned_size); - const ssize_t bytes_written = vfs_store(fn, da.base, da.pos); + (void)da_set_size(&da, round_up(da.cur_size, BLOCK_SIZE)); + shared_ptr file(da.base, DummyDeleter()); + const ssize_t bytes_written = g_VFS->CreateFile(fn, file, da.pos); if(bytes_written > 0) debug_assert(bytes_written == (ssize_t)da.pos); else diff --git a/source/lib/res/graphics/tex.h b/source/lib/tex/tex.h similarity index 91% rename from source/lib/res/graphics/tex.h rename to source/lib/tex/tex.h index ae0d019b95..4fc7501cb6 100644 --- a/source/lib/res/graphics/tex.h +++ b/source/lib/tex/tex.h @@ -89,26 +89,29 @@ library and IO layer. Read and write are zero-copy. #define INCLUDED_TEX #include "lib/res/handle.h" +#include "lib/allocators/dynarray.h" namespace ERR { - const LibError TEX_FMT_INVALID = -120100; - const LibError TEX_INVALID_COLOR_TYPE = -120101; - const LibError TEX_NOT_8BIT_PRECISION = -120102; - const LibError TEX_INVALID_LAYOUT = -120103; - const LibError TEX_COMPRESSED = -120104; - const LibError TEX_INVALID_SIZE = -120105; + const LibError TEX_UNKNOWN_FORMAT = -120100; + const LibError TEX_INCOMPLETE_HEADER = -120101; + const LibError TEX_FMT_INVALID = -120102; + const LibError TEX_INVALID_COLOR_TYPE = -120103; + const LibError TEX_NOT_8BIT_PRECISION = -120104; + const LibError TEX_INVALID_LAYOUT = -120105; + const LibError TEX_COMPRESSED = -120106; + const LibError TEX_INVALID_SIZE = -120107; } namespace WARN { - const LibError TEX_INVALID_DATA = +120106; + const LibError TEX_INVALID_DATA = +120108; } namespace INFO { - const LibError TEX_CODEC_CANNOT_HANDLE = +120107; + const LibError TEX_CODEC_CANNOT_HANDLE = +120109; } @@ -175,7 +178,9 @@ enum TexFlags * to highest (1x1), one after the other. * (conversion is not applicable here) **/ - TEX_MIPMAPS = 0x100 + TEX_MIPMAPS = 0x100, + + TEX_UNDEFINED_FLAGS = ~0x1FF }; @@ -187,15 +192,17 @@ enum TexFlags struct Tex { /** - * H_Mem handle to image data. note: during the course of transforms + * file buffer or image data. note: during the course of transforms * (which may occur when being loaded), this may be replaced with - * a Handle to a new buffer (e.g. if decompressing file contents). + * a new buffer (e.g. if decompressing file contents). **/ - Handle hm; + shared_ptr data; + + size_t dataSize; /** * offset to image data in file. this is required since - * tex_get_data needs to return the pixels, but mem_get_ptr(hm) + * tex_get_data needs to return the pixels, but data * returns the actual file buffer. zero-copy load and * write-back to file is also made possible. **/ @@ -249,10 +256,9 @@ extern void tex_codec_register_all(); * * @param fn filename * @param t output texture object - * @param file_flags additional flags for vfs_load * @return LibError **/ -extern LibError tex_load(const char* fn, Tex* t, uint file_flags = 0); +extern LibError tex_load(const char* fn, Tex* t); /** * store the given image data into a Tex object; this will be as if @@ -277,7 +283,7 @@ extern LibError tex_load(const char* fn, Tex* t, uint file_flags = 0); * @param t output texture object. * @return LibError **/ -extern LibError tex_wrap(uint w, uint h, uint bpp, uint flags, void* img, Tex* t); +extern LibError tex_wrap(uint w, uint h, uint bpp, uint flags, shared_ptr data, size_t ofs, Tex* t); /** * free all resources associated with the image and make further @@ -286,7 +292,7 @@ extern LibError tex_wrap(uint w, uint h, uint bpp, uint flags, void* img, Tex* t * @param t texture object (note: not zeroed afterwards; see impl) * @return LibError **/ -extern LibError tex_free(Tex* t); +extern void tex_free(Tex* t); // @@ -324,7 +330,7 @@ extern LibError tex_transform_to(Tex* t, uint new_flags); /** * return a pointer to the image data (pixels), taking into account any - * header(s) that may come before it. see Tex.hm comment above. + * header(s) that may come before it. * * @param t input texture object * @return pointer to data returned by mem_get_ptr (holds reference)! @@ -358,8 +364,7 @@ const int TEX_BASE_LEVEL_ONLY = -1; * @param level_data_size [bytes] * @param cbData passed through from tex_util_foreach_mipmap. **/ -typedef void (*MipmapCB)(uint level, uint level_w, uint level_h, - const u8* RESTRICT level_data, size_t level_data_size, void* RESTRICT cbData); +typedef void (*MipmapCB)(uint level, uint level_w, uint level_h, const u8* RESTRICT level_data, size_t level_data_size, void* RESTRICT cbData); /** * for a series of mipmaps stored from base to highest, call back for @@ -378,8 +383,7 @@ typedef void (*MipmapCB)(uint level, uint level_w, uint level_h, * @param cb MipmapCB to call * @param cbData extra data to pass to cb **/ -extern void tex_util_foreach_mipmap(uint w, uint h, uint bpp, const u8* RESTRICT data, - int levels_to_skip, uint data_padding, MipmapCB cb, void* RESTRICT cbData); +extern void tex_util_foreach_mipmap(uint w, uint h, uint bpp, const u8* data, int levels_to_skip, uint data_padding, MipmapCB cb, void* RESTRICT cbData); // diff --git a/source/lib/res/graphics/tex_bmp.cpp b/source/lib/tex/tex_bmp.cpp similarity index 100% rename from source/lib/res/graphics/tex_bmp.cpp rename to source/lib/tex/tex_bmp.cpp diff --git a/source/lib/res/graphics/tex_codec.cpp b/source/lib/tex/tex_codec.cpp similarity index 95% rename from source/lib/res/graphics/tex_codec.cpp rename to source/lib/tex/tex_codec.cpp index d69ce2b718..21a795163e 100644 --- a/source/lib/res/graphics/tex_codec.cpp +++ b/source/lib/tex/tex_codec.cpp @@ -38,7 +38,7 @@ int tex_codec_register(TexCodecVTbl* c) // find codec that recognizes the desired output file extension, -// or return ERR::RES_UNKNOWN_FORMAT if unknown. +// or return ERR::TEX_UNKNOWN_FORMAT if unknown. // note: does not raise a warning because it is used by // tex_is_known_extension. LibError tex_codec_for_filename(const char* fn, const TexCodecVTbl** c) @@ -51,7 +51,7 @@ LibError tex_codec_for_filename(const char* fn, const TexCodecVTbl** c) return INFO::OK; } - return ERR::RES_UNKNOWN_FORMAT; // NOWARN + return ERR::TEX_UNKNOWN_FORMAT; // NOWARN } @@ -60,7 +60,7 @@ LibError tex_codec_for_header(const u8* file, size_t file_size, const TexCodecVT { // we guarantee at least 4 bytes for is_hdr to look at if(file_size < 4) - WARN_RETURN(ERR::RES_INCOMPLETE_HEADER); + WARN_RETURN(ERR::TEX_INCOMPLETE_HEADER); for(*c = codecs; *c; *c = (*c)->next) { @@ -69,7 +69,7 @@ LibError tex_codec_for_header(const u8* file, size_t file_size, const TexCodecVT return INFO::OK; } - WARN_RETURN(ERR::RES_UNKNOWN_FORMAT); + WARN_RETURN(ERR::TEX_UNKNOWN_FORMAT); } @@ -99,7 +99,7 @@ LibError tex_codec_transform(Tex* t, uint transforms) else if(err != INFO::TEX_CODEC_CANNOT_HANDLE) { ret = err; - debug_warn("codec indicates error"); + debug_assert(0); // codec indicates error } } diff --git a/source/lib/res/graphics/tex_codec.h b/source/lib/tex/tex_codec.h similarity index 97% rename from source/lib/res/graphics/tex_codec.h rename to source/lib/tex/tex_codec.h index c73cacbb97..e6dd6576e5 100644 --- a/source/lib/res/graphics/tex_codec.h +++ b/source/lib/tex/tex_codec.h @@ -13,7 +13,6 @@ #include "tex.h" #include "tex_internal.h" // for codec's convenience -#include "lib/allocators.h" /** * virtual method table for TexCodecs. @@ -34,7 +33,7 @@ struct TexCodecVTbl * @param t output texture object * @return LibError **/ - LibError (*decode)(DynArray * RESTRICT da, Tex * RESTRICT t); + LibError (*decode)(DynArray* RESTRICT da, Tex * RESTRICT t); /** @@ -48,7 +47,7 @@ struct TexCodecVTbl * by the caller. * @return LibError **/ - LibError (*encode)(Tex * RESTRICT t, DynArray * RESTRICT da); + LibError (*encode)(Tex* RESTRICT t, DynArray * RESTRICT da); /** * transform the texture's pixel format. @@ -69,7 +68,7 @@ struct TexCodecVTbl * (this should be enough to examine the header's 'magic' field) * @return bool **/ - bool (*is_hdr)(const u8 * file); + bool (*is_hdr)(const u8* file); /** * is the extension that of a file format supported by this codec? diff --git a/source/lib/res/graphics/tex_dds.cpp b/source/lib/tex/tex_dds.cpp similarity index 98% rename from source/lib/res/graphics/tex_dds.cpp rename to source/lib/tex/tex_dds.cpp index 237d90273a..1ec0e3c633 100644 --- a/source/lib/res/graphics/tex_dds.cpp +++ b/source/lib/tex/tex_dds.cpp @@ -273,19 +273,16 @@ static LibError s3tc_decompress(Tex* t) // (tex.cpp's plain transform could cover it, if ever needed). const uint dxt = t->flags & TEX_DXT; const uint out_bpp = (dxt != 1)? 32 : 24; - Handle hm; const size_t out_size = tex_img_size(t) * out_bpp / t->bpp; - void* out_data = mem_alloc(out_size, 64*KiB, 0, &hm); - if(!out_data) - WARN_RETURN(ERR::NO_MEM); + shared_ptr decompressedData = io_Allocate(out_size); const uint s3tc_block_size = (dxt == 3 || dxt == 5)? 16 : 8; - S3tcDecompressInfo di = { dxt, s3tc_block_size, out_bpp/8, (u8*)out_data }; + S3tcDecompressInfo di = { dxt, s3tc_block_size, out_bpp/8, decompressedData.get() }; const u8* s3tc_data = tex_get_data(t); const int levels_to_skip = (t->flags & TEX_MIPMAPS)? 0 : TEX_BASE_LEVEL_ONLY; tex_util_foreach_mipmap(t->w, t->h, t->bpp, s3tc_data, levels_to_skip, 4, s3tc_decompress_level, &di); - (void)mem_free_h(t->hm); - t->hm = hm; + t->data = decompressedData; + t->dataSize = out_size; t->ofs = 0; t->bpp = out_bpp; t->flags &= ~TEX_DXT; @@ -501,7 +498,7 @@ static LibError decode_sd(const DDSURFACEDESC2* sd, uint* w_, uint* h_, // note: we can't guess dimensions - the image may not be square. const u32 sd_req_flags = DDSD_CAPS|DDSD_HEIGHT|DDSD_WIDTH|DDSD_PIXELFORMAT; if((sd_flags & sd_req_flags) != sd_req_flags) - WARN_RETURN(ERR::RES_INCOMPLETE_HEADER); + WARN_RETURN(ERR::TEX_INCOMPLETE_HEADER); // image dimensions const u32 h = read_le32(&sd->dwHeight); diff --git a/source/lib/res/graphics/tex_internal.h b/source/lib/tex/tex_internal.h similarity index 83% rename from source/lib/res/graphics/tex_internal.h rename to source/lib/tex/tex_internal.h index dd76c40c57..660d5adb42 100644 --- a/source/lib/res/graphics/tex_internal.h +++ b/source/lib/tex/tex_internal.h @@ -11,8 +11,8 @@ #ifndef INCLUDED_TEX_INTERNAL #define INCLUDED_TEX_INTERNAL -#include "lib/res/res.h" // error codes and mem.h #include "lib/allocators.h" // DynArray +#include "lib/file/io/io.h" // io_Allocate /** * check if the given texture format is acceptable: 8bpp grey, @@ -50,13 +50,10 @@ extern bool tex_orientations_match(uint src_flags, uint dst_orientation); * * @param data input data * @param data_size its size [bytes] - * @param dtor the function used to release it when the texture object is - * freed (can be NULL). note: this is necessary because the Tex object - * assumes ownership (necessary due to Tex.hm). * @param t output texture object. * @return LibError. **/ -extern LibError tex_decode(const u8* data, size_t data_size, MEM_DTOR dtor, Tex* t); +extern LibError tex_decode(const u8* data, size_t data_size, Tex* t); /** * encode a texture into a memory buffer in the desired file format. diff --git a/source/lib/res/graphics/tex_jpg.cpp b/source/lib/tex/tex_jpg.cpp similarity index 91% rename from source/lib/res/graphics/tex_jpg.cpp rename to source/lib/tex/tex_jpg.cpp index ec20075939..936943d8be 100644 --- a/source/lib/res/graphics/tex_jpg.cpp +++ b/source/lib/tex/tex_jpg.cpp @@ -12,7 +12,6 @@ #include "lib/external_libraries/libjpeg.h" -#include "lib/res/res.h" #include "tex_codec.h" #include @@ -410,9 +409,7 @@ static LibError jpg_transform(Tex* UNUSED(t), uint UNUSED(transforms)) // due to less copying. -static LibError jpg_decode_impl(DynArray* da, - jpeg_decompress_struct* cinfo, - Handle& img_hm, RowArray& rows, Tex* t) +static LibError jpg_decode_impl(DynArray* da, jpeg_decompress_struct* cinfo, RowArray& rows, Tex* t) { src_prepare(cinfo, da); @@ -449,12 +446,10 @@ static LibError jpg_decode_impl(DynArray* da, // alloc destination buffer const size_t pitch = w * bpp / 8; const size_t img_size = pitch * h; // for allow_rows - u8* img = (u8*)mem_alloc(img_size, 64*KiB, 0, &img_hm); - if(!img) - WARN_RETURN(ERR::NO_MEM); + shared_ptr data = io_Allocate(img_size); // read rows - RETURN_ERR(tex_codec_alloc_rows(img, h, pitch, TEX_TOP_DOWN, 0, rows)); + RETURN_ERR(tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0, rows)); // could use cinfo->output_scanline to keep track of progress, // but we need to count lines_left anyway (paranoia). JSAMPARRAY row = (JSAMPARRAY)rows; @@ -479,10 +474,9 @@ static LibError jpg_decode_impl(DynArray* da, ret = WARN::TEX_INVALID_DATA; // store image info - // .. transparently switch handles - free the old (compressed) - // buffer and replace it with the decoded-image memory handle. - mem_free_h(t->hm); // must come after jpeg_finish_decompress - t->hm = img_hm; + t->data = data; + t->dataSize = img_size; + t->ofs = 0; t->w = w; t->h = h; t->bpp = bpp; @@ -492,9 +486,7 @@ static LibError jpg_decode_impl(DynArray* da, } -static LibError jpg_encode_impl(Tex* t, - jpeg_compress_struct* cinfo, - RowArray& rows, DynArray* da) +static LibError jpg_encode_impl(Tex* t, jpeg_compress_struct* cinfo, RowArray& rows, DynArray* da) { dst_prepare(cinfo, da); @@ -567,75 +559,48 @@ static size_t jpg_hdr_size(const u8* UNUSED(file)) static LibError jpg_decode(DynArray* RESTRICT da, Tex* RESTRICT t) { - LibError err; - - // freed when ret is reached: - // .. contains the JPEG decompression parameters and pointers to - // working space (allocated as needed by the JPEG library). + // contains the JPEG decompression parameters and pointers to + // working space (allocated as needed by the JPEG library). struct jpeg_decompress_struct cinfo; - // .. array of pointers to scanlines (see rationale above) - RowArray rows = 0; - - // freed when fail is reached: - Handle img_hm; // decompressed image memory JpgErrorMgr jerr((j_common_ptr)&cinfo); if(setjmp(jerr.call_site)) - { - err = ERR::FAIL; - goto fail; - } + return ERR::FAIL; jpeg_create_decompress(&cinfo); - err = jpg_decode_impl(da, &cinfo, img_hm, rows, t); - if(err < 0) - goto fail; - -ret: - jpeg_destroy_decompress(&cinfo); // releases a "good deal" of memory + // array of pointers to scanlines (see rationale above) + RowArray rows = 0; + LibError ret = jpg_decode_impl(da, &cinfo, rows, t); free(rows); - return err; -fail: - mem_free_h(img_hm); - goto ret; + jpeg_destroy_decompress(&cinfo); // releases a "good deal" of memory + + return ret; } // limitation: palette images aren't supported static LibError jpg_encode(Tex* RESTRICT t, DynArray* RESTRICT da) { - LibError err; - - // freed when ret is reached: - // .. contains the JPEG compression parameters and pointers to - // working space (allocated as needed by the JPEG library). + // contains the JPEG compression parameters and pointers to + // working space (allocated as needed by the JPEG library). struct jpeg_compress_struct cinfo; - // .. array of pointers to scanlines (see rationale above) - RowArray rows = 0; - + JpgErrorMgr jerr((j_common_ptr)&cinfo); if(setjmp(jerr.call_site)) - { - err = ERR::FAIL; - goto fail; - } + WARN_RETURN(ERR::FAIL); jpeg_create_compress(&cinfo); - err = jpg_encode_impl(t, &cinfo, rows, da); - if(err < 0) - goto fail; - -ret: - jpeg_destroy_compress(&cinfo); // releases a "good deal" of memory + // array of pointers to scanlines (see rationale above) + RowArray rows = 0; + LibError ret = jpg_encode_impl(t, &cinfo, rows, da); free(rows); - return err; -fail: - // currently no extra cleanup needed - goto ret; + jpeg_destroy_compress(&cinfo); // releases a "good deal" of memory + + return ret; } TEX_CODEC_REGISTER(jpg); diff --git a/source/lib/res/graphics/tex_png.cpp b/source/lib/tex/tex_png.cpp similarity index 79% rename from source/lib/res/graphics/tex_png.cpp rename to source/lib/tex/tex_png.cpp index 8e1d945ba2..16f92701a2 100644 --- a/source/lib/res/graphics/tex_png.cpp +++ b/source/lib/tex/tex_png.cpp @@ -13,7 +13,6 @@ #include "lib/external_libraries/png.h" #include "lib/byte_order.h" -#include "lib/res/res.h" #include "tex_codec.h" #include "lib/timer.h" @@ -71,9 +70,7 @@ static LibError png_transform(Tex* UNUSED(t), uint UNUSED(transforms)) // split out of png_decode to simplify resource cleanup and avoid // "dtor / setjmp interaction" warning. -static LibError png_decode_impl(DynArray* da, - png_structp png_ptr, png_infop info_ptr, - Handle& img_hm, RowArray& rows, Tex* t) +static LibError png_decode_impl(DynArray* da, png_structp png_ptr, png_infop info_ptr, RowArray& rows, Tex* t) { png_set_read_fn(png_ptr, da, io_read); @@ -98,11 +95,9 @@ static LibError png_decode_impl(DynArray* da, WARN_RETURN(ERR::TEX_INVALID_COLOR_TYPE); const size_t img_size = pitch * h; - u8* img = (u8*)mem_alloc(img_size, 64*KiB, 0, &img_hm); - if(!img) - WARN_RETURN(ERR::NO_MEM); + shared_ptr data = io_Allocate(img_size); - RETURN_ERR(tex_codec_alloc_rows(img, h, pitch, TEX_TOP_DOWN, 0, rows)); + RETURN_ERR(tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0, rows)); png_read_image(png_ptr, (png_bytepp)rows); png_read_end(png_ptr, info_ptr); @@ -112,10 +107,9 @@ static LibError png_decode_impl(DynArray* da, debug_assert(da->pos == da->cur_size); // store image info - // .. transparently switch handles - free the old (compressed) - // buffer and replace it with the decoded-image memory handle. - mem_free_h(t->hm); // must come after png_read_end - t->hm = img_hm; + t->data = data; + t->dataSize = img_size; + t->ofs = 0; t->w = w; t->h = h; t->bpp = bpp; @@ -127,9 +121,7 @@ static LibError png_decode_impl(DynArray* da, // split out of png_encode to simplify resource cleanup and avoid // "dtor / setjmp interaction" warning. -static LibError png_encode_impl(Tex* t, - png_structp png_ptr, png_infop info_ptr, - RowArray& rows, DynArray* da) +static LibError png_encode_impl(Tex* t, png_structp png_ptr, png_infop info_ptr, RowArray& rows, DynArray* da) { const png_uint_32 w = t->w, h = t->h; const size_t pitch = w * t->bpp / 8; @@ -197,57 +189,44 @@ static LibError png_decode(DynArray* RESTRICT da, Tex* RESTRICT t) { TIMER_ACCRUE(tc_png_decode); - LibError err = ERR::FAIL; - // freed when ret is reached: - png_structp png_ptr = 0; + LibError ret = ERR::FAIL; png_infop info_ptr = 0; - RowArray rows = 0; - - // freed if fail is reached: - Handle img_hm = 0; // decompressed image memory // allocate PNG structures; use default stderr and longjmp error handlers - png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, 0, 0, 0); + png_structp png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, 0, 0, 0); if(!png_ptr) - goto fail; + WARN_RETURN(ERR::FAIL); info_ptr = png_create_info_struct(png_ptr); if(!info_ptr) goto fail; - // setup error handling if(setjmp(png_jmpbuf(png_ptr))) { // libpng longjmps here after an error -fail: - mem_free_h(img_hm); - goto ret; + goto fail; } - err = png_decode_impl(da, png_ptr, info_ptr, img_hm, rows, t); - if(err < 0) - goto fail; - -ret: - if(png_ptr) - png_destroy_read_struct(&png_ptr, &info_ptr, 0); + RowArray rows = 0; + ret = png_decode_impl(da, png_ptr, info_ptr, rows, t); free(rows); - return err; + +fail: + png_destroy_read_struct(&png_ptr, &info_ptr, 0); + + return ret; } // limitation: palette images aren't supported static LibError png_encode(Tex* RESTRICT t, DynArray* RESTRICT da) { - LibError err = ERR::FAIL; - // freed when ret is reached: - png_structp png_ptr = 0; + LibError ret = ERR::FAIL; png_infop info_ptr = 0; - RowArray rows = 0; // allocate PNG structures; use default stderr and longjmp error handlers - png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, 0, 0, 0); + png_structp png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, 0, 0, 0); if(!png_ptr) - goto fail; + WARN_RETURN(ERR::FAIL); info_ptr = png_create_info_struct(png_ptr); if(!info_ptr) goto fail; @@ -256,20 +235,17 @@ static LibError png_encode(Tex* RESTRICT t, DynArray* RESTRICT da) if(setjmp(png_jmpbuf(png_ptr))) { // libpng longjmps here after an error -fail: - // currently no cleanup to be done. - goto ret; + goto fail; } - err = png_encode_impl(t, png_ptr, info_ptr, rows, da); - if(err < 0) - goto fail; + RowArray rows = 0; + ret = png_encode_impl(t, png_ptr, info_ptr, rows, da); + free(rows); // shared cleanup -ret: +fail: png_destroy_write_struct(&png_ptr, &info_ptr); - free(rows); - return err; + return ret; } TEX_CODEC_REGISTER(png); diff --git a/source/lib/res/graphics/tex_tga.cpp b/source/lib/tex/tex_tga.cpp similarity index 100% rename from source/lib/res/graphics/tex_tga.cpp rename to source/lib/tex/tex_tga.cpp