WIP of new file code
archive: pretty much done except for trace and archive_builder IO: io_manager still needs to be fixed/revised (especially WRT buffer and alignment management) posix: done vfs: need to complete vfs_mount and tie it in to vfs.cpp dir_util: needs further revisions (VFS interface changed since then) This was SVN commit r5475.
This commit is contained in:
parent
d19a2cba65
commit
317f98a6c0
23
source/lib/file/archive/archive.cpp
Normal file
23
source/lib/file/archive/archive.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : archive.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : interface for reading from and creating archives.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "archive.h"
|
||||
|
||||
ERROR_ASSOCIATE(ERR::ARCHIVE_UNKNOWN_FORMAT, "Unknown archive format", -1);
|
||||
ERROR_ASSOCIATE(ERR::ARCHIVE_UNKNOWN_METHOD, "Unknown compression method", -1);
|
||||
|
||||
IArchiveReader::~IArchiveReader()
|
||||
{
|
||||
}
|
||||
|
||||
IArchiveWriter::~IArchiveWriter()
|
||||
{
|
||||
}
|
@ -11,91 +11,98 @@
|
||||
#ifndef INCLUDED_ARCHIVE
|
||||
#define INCLUDED_ARCHIVE
|
||||
|
||||
struct ICodec;
|
||||
// rationale: this module doesn't build a directory tree of the entries
|
||||
// within an archive. that task is left to the VFS; here, we are only
|
||||
// concerned with enumerating all archive entries.
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError UNKNOWN_FORMAT = -110400;
|
||||
const LibError IS_COMPRESSED = -110401;
|
||||
const LibError ARCHIVE_UNKNOWN_FORMAT = -110400;
|
||||
const LibError ARCHIVE_UNKNOWN_METHOD = -110401;
|
||||
}
|
||||
|
||||
enum ArchiveEntryFlags
|
||||
// opaque 'memento' of an archive entry. the instances are stored by
|
||||
// the IArchiveReader implementation.
|
||||
struct ArchiveEntry;
|
||||
|
||||
struct IArchiveReader
|
||||
{
|
||||
// (this avoids having to interpret each archive's method values)
|
||||
AEF_COMPRESSED = 1,
|
||||
|
||||
// indicates ArchiveEntry.ofs points to a "local file header" instead of
|
||||
// the file data. a fixup routine is called upon file open; it skips
|
||||
// past LFH and clears this flag.
|
||||
// this is somewhat of a hack, but vital to archive open performance.
|
||||
// without it, we'd have to scan through the entire archive file,
|
||||
// which can take *seconds*.
|
||||
// (we cannot use the information in CDFH, because its 'extra' field
|
||||
// has been observed to differ from that of the LFH)
|
||||
// by reading LFH when a file in archive is opened, the block cache
|
||||
// absorbs the IO cost because the file will likely be read anyway.
|
||||
AEF_NEEDS_FIXUP = 2
|
||||
};
|
||||
|
||||
// holds all per-file information extracted from the header.
|
||||
// this is intended to work for all archive types.
|
||||
struct ArchiveEntry
|
||||
{
|
||||
off_t ofs;
|
||||
size_t usize;
|
||||
size_t csize;
|
||||
time_t mtime;
|
||||
u32 checksum;
|
||||
uint method;
|
||||
uint flags; // ArchiveEntryFlags
|
||||
|
||||
// note that size == usize isn't foolproof, and adding a flag to
|
||||
// ofs or size is ugly and error-prone.
|
||||
bool IsCompressed() const
|
||||
{
|
||||
return (flags & AEF_COMPRESSED) != 0;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// successively called for each archive entry.
|
||||
typedef LibError (*ArchiveCB)(const char* pathname, const ArchiveEntry& ae, uintptr_t cbData);
|
||||
|
||||
|
||||
struct IArchive
|
||||
{
|
||||
IArchive(const char* pathname);
|
||||
virtual ~IArchive();
|
||||
|
||||
virtual boost::shared_ptr<ICodec> CreateDecompressor() const = 0;
|
||||
virtual ~IArchiveReader();
|
||||
|
||||
/**
|
||||
* call back for each file entry in the archive.
|
||||
* called for each archive entry.
|
||||
* @param pathname full pathname of entry (unique pointer)
|
||||
**/
|
||||
virtual LibError ForEachEntry(ArchiveCB cb, uintptr_t cbData) const = 0;
|
||||
typedef void (*ArchiveEntryCallback)(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData);
|
||||
virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) = 0;
|
||||
|
||||
virtual LibError LoadFile(ArchiveEntry& archiveEntry) const = 0;
|
||||
virtual LibError LoadFile(ArchiveEntry& archiveEntry, u8* fileContents) const = 0;
|
||||
};
|
||||
|
||||
|
||||
// note: when creating an archive, any existing file with the given pathname
|
||||
// will be overwritten.
|
||||
struct IArchiveBuilder
|
||||
|
||||
// rationale: don't support partial adding, i.e. updating archive with
|
||||
// only one file. this would require overwriting parts of the Zip archive,
|
||||
// which is annoying and slow. also, archives are usually built in
|
||||
// seek-optimal order, which would break if we start inserting files.
|
||||
// while testing, loose files can be used, so there's no loss.
|
||||
|
||||
struct IArchiveWriter
|
||||
{
|
||||
/**
|
||||
* write out the archive to disk; only hereafter is it valid.
|
||||
**/
|
||||
virtual ~IArchiveBuilder();
|
||||
|
||||
virtual boost::shared_ptr<ICodec> CreateCompressor() const = 0;
|
||||
virtual ~IArchiveWriter();
|
||||
|
||||
/**
|
||||
* add a file to the archive.
|
||||
*
|
||||
* @param fileContents the file data; its compression method is defined by
|
||||
* ae.method and can be CM_NONE.
|
||||
* rationale: passing in a filename instead of the compressed file
|
||||
* contents makes for better encapsulation because callers don't need
|
||||
* to know about the codec. one disadvantage is that loading the file
|
||||
* contents can no longer take advantage of the VFS cache nor previously
|
||||
* archived versions. however, the archive builder usually adds files
|
||||
* precisely because they aren't in archives, and the cache would
|
||||
* thrash anyway, so this is deemed acceptable.
|
||||
**/
|
||||
virtual LibError AddFile(const ArchiveEntry& ae, const u8* fileContents) = 0;
|
||||
virtual LibError AddFile(const char* pathname) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* describes an archive entry (either file or directory).
|
||||
**/
|
||||
struct ArchiveEntry
|
||||
{
|
||||
ArchiveEntry()
|
||||
{
|
||||
}
|
||||
|
||||
ArchiveEntry(off_t ofs, off_t usize, off_t csize, time_t mtime, u32 checksum, uint method, uint flags)
|
||||
{
|
||||
this->ofs = ofs;
|
||||
this->usize = usize;
|
||||
this->csize = csize;
|
||||
this->mtime = mtime;
|
||||
this->checksum = checksum;
|
||||
this->method = method;
|
||||
this->flags = flags;
|
||||
}
|
||||
|
||||
// this is needed by the VFS and stored here instead of in VfsFile to
|
||||
// yield a nice 32-byte size.
|
||||
IArchiveReader* archiveReader;
|
||||
|
||||
off_t ofs;
|
||||
off_t usize;
|
||||
off_t csize;
|
||||
time_t mtime;
|
||||
|
||||
// the archive implementation establishes the meaning of the following:
|
||||
|
||||
u32 checksum;
|
||||
uint method;
|
||||
uint flags;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_ARCHIVE
|
||||
|
1048
source/lib/file/archive/archive_builder.cpp
Normal file
1048
source/lib/file/archive/archive_builder.cpp
Normal file
File diff suppressed because it is too large
Load Diff
26
source/lib/file/archive/archive_builder.h
Normal file
26
source/lib/file/archive/archive_builder.h
Normal file
@ -0,0 +1,26 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_optimizer.h
|
||||
* Project : 0 A.D.
|
||||
* Description : automatically bundles files into archives in order of
|
||||
* : access to optimize I/O.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_VFS_OPTIMIZER
|
||||
#define INCLUDED_VFS_OPTIMIZER
|
||||
|
||||
|
||||
extern LibError vfs_opt_rebuild_main_archive(const char* trace_filename, const char* archive_fn_fmt);
|
||||
|
||||
extern void vfs_opt_auto_build_cancel();
|
||||
|
||||
extern int vfs_opt_auto_build(const char* trace_filename,
|
||||
const char* archive_fn_fmt, const char* mini_archive_fn_fmt, bool force_build = false);
|
||||
|
||||
extern void vfs_opt_notify_loose_file(const char* atom_fn);
|
||||
extern void vfs_opt_notify_archived_file(const char* atom_fn);
|
||||
|
||||
#endif // #ifndef INCLUDED_VFS_OPTIMIZER
|
588
source/lib/file/archive/archive_zip.cpp
Normal file
588
source/lib/file/archive/archive_zip.cpp
Normal file
@ -0,0 +1,588 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : archive_zip.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : archive backend for Zip files.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "archive_zip.h"
|
||||
|
||||
#include <time.h>
|
||||
#include <limits>
|
||||
|
||||
#include "lib/bits.h"
|
||||
#include "lib/byte_order.h"
|
||||
#include "lib/fat_time.h"
|
||||
#include "lib/allocators/pool.h"
|
||||
#include "lib/sysdep/cpu.h" // cpu_memcpy
|
||||
#include "lib/file/path.h"
|
||||
#include "archive.h"
|
||||
#include "codec_zlib.h"
|
||||
#include "stream.h"
|
||||
#include "lib/file/filesystem.h"
|
||||
#include "lib/file/posix/fs_posix.h"
|
||||
#include "lib/file/posix/io_posix.h"
|
||||
#include "lib/file/io/io_manager.h"
|
||||
|
||||
|
||||
enum ArchiveZipFlags
|
||||
{
|
||||
// indicates ArchiveEntry.ofs points to a "local file header" instead of
|
||||
// the file data. a fixup routine is called when reading the file;
|
||||
// it skips past the LFH and clears this flag.
|
||||
// this is somewhat of a hack, but vital to archive open performance.
|
||||
// without it, we'd have to scan through the entire archive file,
|
||||
// which can take *seconds*.
|
||||
// (we cannot use the information in CDFH, because its 'extra' field
|
||||
// has been observed to differ from that of the LFH)
|
||||
// since we read the LFH right before the rest of the file, the block
|
||||
// cache will absorb the IO cost.
|
||||
NEEDS_LFH_FIXUP = 1
|
||||
};
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Zip file data structures and signatures
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
enum ZipMethod
|
||||
{
|
||||
ZIP_METHOD_NONE = 0,
|
||||
ZIP_METHOD_DEFLATE = 8
|
||||
};
|
||||
|
||||
static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2');
|
||||
static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4');
|
||||
static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6');
|
||||
|
||||
#pragma pack(push, 1)
|
||||
|
||||
class LFH
|
||||
{
|
||||
public:
|
||||
void Init(const ArchiveEntry& archiveEntry, const char* pathname, size_t pathnameLength)
|
||||
{
|
||||
m_magic = lfh_magic;
|
||||
m_x1 = to_le16(0);
|
||||
m_flags = to_le16(0);
|
||||
m_method = to_le16(u16_from_larger(archiveEntry.method));
|
||||
m_fat_mtime = to_le32(FAT_from_time_t(archiveEntry.mtime));
|
||||
m_crc = to_le32(archiveEntry.checksum);
|
||||
m_csize = to_le32(u32_from_larger(archiveEntry.csize));
|
||||
m_usize = to_le32(u32_from_larger(archiveEntry.usize));
|
||||
m_fn_len = to_le16(u16_from_larger((uint)pathnameLength));
|
||||
m_e_len = to_le16(0);
|
||||
|
||||
cpu_memcpy((char*)this + sizeof(LFH), pathname, pathnameLength);
|
||||
}
|
||||
|
||||
size_t Size() const
|
||||
{
|
||||
debug_assert(m_magic == lfh_magic);
|
||||
const size_t fn_len = read_le16(&m_fn_len);
|
||||
const size_t e_len = read_le16(&m_e_len);
|
||||
// note: LFH doesn't have a comment field!
|
||||
|
||||
return sizeof(LFH) + fn_len + e_len;
|
||||
}
|
||||
|
||||
private:
|
||||
u32 m_magic;
|
||||
u16 m_x1; // version needed
|
||||
u16 m_flags;
|
||||
u16 m_method;
|
||||
u32 m_fat_mtime; // last modified time (DOS FAT format)
|
||||
u32 m_crc;
|
||||
u32 m_csize;
|
||||
u32 m_usize;
|
||||
u16 m_fn_len;
|
||||
u16 m_e_len;
|
||||
};
|
||||
|
||||
cassert(sizeof(LFH) == 30);
|
||||
|
||||
|
||||
class CDFH
|
||||
{
|
||||
public:
|
||||
void Init(const ArchiveEntry& archiveEntry, const char* pathname, size_t pathnameLength, size_t slack)
|
||||
{
|
||||
m_magic = cdfh_magic;
|
||||
m_x1 = to_le32(0);
|
||||
m_flags = to_le16(0);
|
||||
m_method = to_le16(u16_from_larger(archiveEntry.method));
|
||||
m_fat_mtime = to_le32(FAT_from_time_t(archiveEntry.mtime));
|
||||
m_crc = to_le32(archiveEntry.checksum);
|
||||
m_csize = to_le32(u32_from_larger(archiveEntry.csize));
|
||||
m_usize = to_le32(u32_from_larger(archiveEntry.usize));
|
||||
m_fn_len = to_le16(u16_from_larger((uint)pathnameLength));
|
||||
m_e_len = to_le16(0);
|
||||
m_c_len = to_le16(u16_from_larger((uint)slack));
|
||||
m_x2 = to_le32(0);
|
||||
m_x3 = to_le32(0);
|
||||
m_lfh_ofs = to_le32(archiveEntry.ofs);
|
||||
|
||||
cpu_memcpy((char*)this + sizeof(CDFH), pathname, pathnameLength);
|
||||
}
|
||||
|
||||
void Decompose(ArchiveEntry& archiveEntry, const char*& pathname, size_t& cdfhSize) const
|
||||
{
|
||||
const u16 zipMethod = read_le16(&m_method);
|
||||
const u32 fat_mtime = read_le32(&m_fat_mtime);
|
||||
const u32 crc = read_le32(&m_crc);
|
||||
const u32 csize = read_le32(&m_csize);
|
||||
const u32 usize = read_le32(&m_usize);
|
||||
const u16 fn_len = read_le16(&m_fn_len);
|
||||
const u16 e_len = read_le16(&m_e_len);
|
||||
const u16 c_len = read_le16(&m_c_len);
|
||||
const u32 lfh_ofs = read_le32(&m_lfh_ofs);
|
||||
|
||||
archiveEntry.ofs = (off_t)lfh_ofs;
|
||||
archiveEntry.usize = (off_t)usize;
|
||||
archiveEntry.csize = (off_t)csize;
|
||||
archiveEntry.mtime = time_t_from_FAT(fat_mtime);
|
||||
archiveEntry.checksum = crc;
|
||||
archiveEntry.method = (uint)zipMethod;
|
||||
archiveEntry.flags = NEEDS_LFH_FIXUP;
|
||||
|
||||
// return 0-terminated copy of filename
|
||||
const char* fn = (const char*)this + sizeof(CDFH); // not 0-terminated!
|
||||
char buf[PATH_MAX]; // path_Pool()->UniqueCopy requires a 0-terminated string
|
||||
cpu_memcpy(buf, fn, fn_len*sizeof(char));
|
||||
buf[fn_len] = '\0';
|
||||
pathname = path_Pool()->UniqueCopy(buf);
|
||||
|
||||
cdfhSize = sizeof(CDFH) + fn_len + e_len + c_len;
|
||||
}
|
||||
|
||||
private:
|
||||
u32 m_magic;
|
||||
u32 m_x1; // versions
|
||||
u16 m_flags;
|
||||
u16 m_method;
|
||||
u32 m_fat_mtime; // last modified time (DOS FAT format)
|
||||
u32 m_crc;
|
||||
u32 m_csize;
|
||||
u32 m_usize;
|
||||
u16 m_fn_len;
|
||||
u16 m_e_len;
|
||||
u16 m_c_len;
|
||||
u32 m_x2; // spanning
|
||||
u32 m_x3; // attributes
|
||||
u32 m_lfh_ofs;
|
||||
};
|
||||
|
||||
cassert(sizeof(CDFH) == 46);
|
||||
|
||||
|
||||
class ECDR
|
||||
{
|
||||
public:
|
||||
void Init(uint cd_numEntries, off_t cd_ofs, size_t cd_size)
|
||||
{
|
||||
m_magic = ecdr_magic;
|
||||
memset(m_x1, 0, sizeof(m_x1));
|
||||
m_cd_numEntries = to_le16(u16_from_larger(cd_numEntries));
|
||||
m_cd_size = to_le32(u32_from_larger(cd_size));
|
||||
m_cd_ofs = to_le32(u32_from_larger(cd_ofs));
|
||||
m_comment_len = to_le16(0);
|
||||
}
|
||||
|
||||
void Decompose(uint& cd_numEntries, off_t& cd_ofs, size_t& cd_size) const
|
||||
{
|
||||
cd_numEntries = (uint)read_le16(&m_cd_numEntries);
|
||||
cd_ofs = (off_t)read_le32(&m_cd_ofs);
|
||||
cd_size = (size_t)read_le32(&m_cd_size);
|
||||
}
|
||||
|
||||
private:
|
||||
u32 m_magic;
|
||||
u8 m_x1[6]; // multiple-disk support
|
||||
u16 m_cd_numEntries;
|
||||
u32 m_cd_size;
|
||||
u32 m_cd_ofs;
|
||||
u16 m_comment_len;
|
||||
};
|
||||
|
||||
cassert(sizeof(ECDR) == 22);
|
||||
|
||||
#pragma pack(pop)
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* scan buffer for a Zip file record.
|
||||
*
|
||||
* @param start position within buffer
|
||||
* @param magic signature of record
|
||||
* @param recordSize size of record (including signature)
|
||||
* @return pointer to record within buffer or 0 if not found.
|
||||
**/
|
||||
static const u8* zip_FindRecord(const u8* buf, size_t size, const u8* start, u32 magic, size_t recordSize)
|
||||
{
|
||||
// (don't use <start> as the counter - otherwise we can't tell if
|
||||
// scanning within the buffer was necessary.)
|
||||
for(const u8* p = start; p <= buf+size-recordSize; p++)
|
||||
{
|
||||
// found it
|
||||
if(*(u32*)p == magic)
|
||||
{
|
||||
debug_assert(p == start); // otherwise, the archive is a bit broken
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
// passed EOF, didn't find it.
|
||||
// note: do not warn - this happens in the initial ECDR search at
|
||||
// EOF if the archive contains a comment field.
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
// search for ECDR in the last <maxScanSize> bytes of the file.
|
||||
// if found, fill <dst_ecdr> with a copy of the (little-endian) ECDR and
|
||||
// return INFO::OK, otherwise IO error or ERR::CORRUPTED.
|
||||
static LibError zip_ReadECDR(const File_Posix& file, off_t fileSize, IoBuf& buf, size_t maxScanSize, uint& cd_numEntries, off_t& cd_ofs, size_t& cd_size)
|
||||
{
|
||||
// don't scan more than the entire file
|
||||
const size_t scanSize = std::min(maxScanSize, (size_t)fileSize);
|
||||
|
||||
// read desired chunk of file into memory
|
||||
const off_t ofs = fileSize - (off_t)scanSize;
|
||||
RETURN_ERR(io_Read(file, ofs, buf, scanSize));
|
||||
|
||||
// look for ECDR in buffer
|
||||
const u8* start = (const u8*)buf.get();
|
||||
const ECDR* ecdr = (const ECDR*)zip_FindRecord(start, scanSize, start, ecdr_magic, sizeof(ECDR));
|
||||
if(!ecdr)
|
||||
WARN_RETURN(INFO::CANNOT_HANDLE);
|
||||
|
||||
ecdr->Decompose(cd_numEntries, cd_ofs, cd_size);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
static LibError zip_LocateCentralDirectory(const File_Posix& file, off_t fileSize, off_t& cd_ofs, uint& cd_numEntries, size_t& cd_size)
|
||||
{
|
||||
const size_t maxScanSize = 66000u; // see below
|
||||
IoBuf buf = io_buf_Allocate(maxScanSize);
|
||||
|
||||
// expected case: ECDR at EOF; no file comment
|
||||
LibError ret = zip_ReadECDR(file, fileSize, buf, sizeof(ECDR), cd_numEntries, cd_ofs, cd_size);
|
||||
if(ret == INFO::OK)
|
||||
return INFO::OK;
|
||||
// worst case: ECDR precedes 64 KiB of file comment
|
||||
ret = zip_ReadECDR(file, fileSize, buf, maxScanSize, cd_numEntries, cd_ofs, cd_size);
|
||||
if(ret == INFO::OK)
|
||||
return INFO::OK;
|
||||
|
||||
// both ECDR scans failed - this is not a valid Zip file.
|
||||
RETURN_ERR(io_Read(file, 0, buf, sizeof(LFH)));
|
||||
// the Zip file has an LFH but lacks an ECDR. this can happen if
|
||||
// the user hard-exits while an archive is being written.
|
||||
// notes:
|
||||
// - return ERR::CORRUPTED so VFS will not include this file.
|
||||
// - we could work around this by scanning all LFHs, but won't bother
|
||||
// because it'd be slow.
|
||||
// - do not warn - the corrupt archive will be deleted on next
|
||||
// successful archive builder run anyway.
|
||||
if(zip_FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH)))
|
||||
return ERR::CORRUPTED; // NOWARN
|
||||
// totally bogus
|
||||
else
|
||||
WARN_RETURN(ERR::ARCHIVE_UNKNOWN_FORMAT);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// ArchiveReader_Zip
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
static Filesystem_Posix fsPosix;
|
||||
|
||||
class ArchiveReader_Zip : public IArchiveReader
|
||||
{
|
||||
public:
|
||||
ArchiveReader_Zip(const char* pathname)
|
||||
{
|
||||
m_file.Open(pathname, 'r');
|
||||
|
||||
FileInfo fileInfo;
|
||||
fsPosix.GetFileInfo(pathname, fileInfo);
|
||||
m_fileSize = fileInfo.Size();
|
||||
debug_assert(m_fileSize >= sizeof(LFH)+sizeof(CDFH)+sizeof(ECDR));
|
||||
}
|
||||
|
||||
virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData)
|
||||
{
|
||||
// locate and read Central Directory
|
||||
off_t cd_ofs; uint cd_numEntries; size_t cd_size;
|
||||
RETURN_ERR(zip_LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size));
|
||||
IoBuf buf = io_buf_Allocate(cd_size);
|
||||
RETURN_ERR(io_Read(m_file, cd_ofs, buf, cd_size));
|
||||
const u8* cd = (const u8*)buf.get();
|
||||
|
||||
m_entries.reserve(cd_numEntries);
|
||||
|
||||
// iterate over Central Directory
|
||||
const u8* pos = cd;
|
||||
for(uint i = 0; i < cd_numEntries; i++)
|
||||
{
|
||||
// scan for next CDFH
|
||||
CDFH* cdfh = (CDFH*)zip_FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH));
|
||||
if(!cdfh)
|
||||
WARN_RETURN(ERR::CORRUPTED);
|
||||
|
||||
ArchiveEntry archiveEntry;
|
||||
const char* pathname;
|
||||
size_t cdfhSize;
|
||||
cdfh->Decompose(archiveEntry, pathname, cdfhSize);
|
||||
|
||||
m_entries.push_back(archiveEntry);
|
||||
cb(pathname, m_entries.back(), cbData);
|
||||
|
||||
pos += cdfhSize;
|
||||
}
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual LibError LoadFile(ArchiveEntry& archiveEntry, u8* fileContents) const
|
||||
{
|
||||
if((archiveEntry.flags & NEEDS_LFH_FIXUP) != 0)
|
||||
{
|
||||
FixupEntry(archiveEntry);
|
||||
archiveEntry.flags &= ~NEEDS_LFH_FIXUP;
|
||||
}
|
||||
|
||||
boost::shared_ptr<ICodec> codec;
|
||||
switch(archiveEntry.method)
|
||||
{
|
||||
case ZIP_METHOD_NONE:
|
||||
codec = CreateCodec_ZLibNone();
|
||||
break;
|
||||
case ZIP_METHOD_DEFLATE:
|
||||
codec = CreateDecompressor_ZLibDeflate();
|
||||
break;
|
||||
default:
|
||||
WARN_RETURN(ERR::ARCHIVE_UNKNOWN_METHOD);
|
||||
}
|
||||
|
||||
Stream stream(codec);
|
||||
stream.SetOutputBuffer(fileContents, archiveEntry.usize);
|
||||
RETURN_ERR(io_Read(m_file, archiveEntry.ofs, IO_BUF_TEMP, archiveEntry.csize, FeedStream, (uintptr_t)&stream));
|
||||
RETURN_ERR(stream.Finish());
|
||||
debug_assert(archiveEntry.checksum == stream.Checksum());
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
struct LFH_Copier
|
||||
{
|
||||
u8* lfh_dst;
|
||||
size_t lfh_bytes_remaining;
|
||||
};
|
||||
|
||||
// this code grabs an LFH struct from file block(s) that are
|
||||
// passed to the callback. usually, one call copies the whole thing,
|
||||
// but the LFH may straddle a block boundary.
|
||||
//
|
||||
// rationale: this allows using temp buffers for zip_fixup_lfh,
|
||||
// which avoids involving the file buffer manager and thus
|
||||
// avoids cluttering the trace and cache contents.
|
||||
static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size, size_t& bytesProcessed)
|
||||
{
|
||||
LFH_Copier* p = (LFH_Copier*)cbData;
|
||||
|
||||
debug_assert(size <= p->lfh_bytes_remaining);
|
||||
cpu_memcpy(p->lfh_dst, block, size);
|
||||
p->lfh_dst += size;
|
||||
p->lfh_bytes_remaining -= size;
|
||||
|
||||
bytesProcessed = size;
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
|
||||
/**
|
||||
* fix up archiveEntry's offset within the archive.
|
||||
* called by archive_LoadFile iff NEEDS_LFH_FIXUP is set.
|
||||
* side effects: adjusts archiveEntry.ofs.
|
||||
*
|
||||
* ensures <ent.ofs> points to the actual file contents; it is initially
|
||||
* the offset of the LFH. we cannot use CDFH filename and extra field
|
||||
* lengths to skip past LFH since that may not mirror CDFH (has happened).
|
||||
*
|
||||
* this is called at file-open time instead of while mounting to
|
||||
* reduce seeks: since reading the file will typically follow, the
|
||||
* block cache entirely absorbs the IO cost.
|
||||
**/
|
||||
void FixupEntry(ArchiveEntry& archiveEntry) const
|
||||
{
|
||||
// performance note: this ends up reading one file block, which is
|
||||
// only in the block cache if the file starts in the same block as a
|
||||
// previously read file (i.e. both are small).
|
||||
LFH lfh;
|
||||
LFH_Copier params = { (u8*)&lfh, sizeof(LFH) };
|
||||
THROW_ERR(io_Read(m_file, archiveEntry.ofs, IO_BUF_TEMP, sizeof(LFH), lfh_copier_cb, (uintptr_t)¶ms));
|
||||
|
||||
archiveEntry.ofs += (off_t)lfh.Size();
|
||||
}
|
||||
|
||||
File_Posix m_file;
|
||||
off_t m_fileSize;
|
||||
std::vector<ArchiveEntry> m_entries;
|
||||
};
|
||||
|
||||
|
||||
boost::shared_ptr<IArchiveReader> CreateArchiveReader_Zip(const char* archivePathname)
|
||||
{
|
||||
return boost::shared_ptr<IArchiveReader>(new ArchiveReader_Zip(archivePathname));
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// ArchiveWriterZip
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
static bool IsFileTypeIncompressible(const char* extension)
|
||||
{
|
||||
// file extensions that we don't want to compress
|
||||
static const char* incompressibleExtensions[] =
|
||||
{
|
||||
"zip", "rar",
|
||||
"jpg", "jpeg", "png",
|
||||
"ogg", "mp3"
|
||||
};
|
||||
|
||||
for(uint i = 0; i < ARRAY_SIZE(incompressibleExtensions); i++)
|
||||
{
|
||||
if(!strcasecmp(extension, incompressibleExtensions[i]))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
class ArchiveWriter_Zip : public IArchiveWriter
|
||||
{
|
||||
public:
|
||||
ArchiveWriter_Zip(const char* archivePathname)
|
||||
: m_fileSize(0)
|
||||
, m_numEntries(0)
|
||||
{
|
||||
THROW_ERR(m_file.Open(archivePathname, 'w'));
|
||||
THROW_ERR(pool_create(&m_cdfhPool, 10*MiB, 0));
|
||||
}
|
||||
|
||||
~ArchiveWriter_Zip()
|
||||
{
|
||||
const size_t cd_size = m_cdfhPool.da.pos;
|
||||
|
||||
// append an ECDR to the CDFH list (this allows us to
|
||||
// write out both to the archive file in one burst)
|
||||
ECDR* ecdr = (ECDR*)pool_alloc(&m_cdfhPool, sizeof(ECDR));
|
||||
if(ecdr)
|
||||
{
|
||||
ecdr->Init(m_numEntries, m_fileSize, cd_size);
|
||||
io_Write(m_file, m_fileSize, m_cdfhPool.da.base, cd_size+sizeof(ECDR));
|
||||
}
|
||||
|
||||
(void)pool_destroy(&m_cdfhPool);
|
||||
}
|
||||
|
||||
LibError AddFile(const char* pathname)
|
||||
{
|
||||
FileInfo fileInfo;
|
||||
RETURN_ERR(fsPosix.GetFileInfo(pathname, fileInfo));
|
||||
const off_t usize = fileInfo.Size();
|
||||
// skip 0-length files.
|
||||
// rationale: zip.cpp needs to determine whether a CDFH entry is
|
||||
// a file or directory (the latter are written by some programs but
|
||||
// not needed - they'd only pollute the file table).
|
||||
// it looks like checking for usize=csize=0 is the safest way -
|
||||
// relying on file attributes (which are system-dependent!) is
|
||||
// even less safe.
|
||||
// we thus skip 0-length files to avoid confusing them with directories.
|
||||
if(!usize)
|
||||
return INFO::SKIPPED;
|
||||
|
||||
File_Posix file;
|
||||
RETURN_ERR(file.Open(pathname, 'r'));
|
||||
|
||||
// choose method and the corresponding codec
|
||||
ZipMethod zipMethod;
|
||||
boost::shared_ptr<ICodec> codec;
|
||||
if(IsFileTypeIncompressible(pathname))
|
||||
{
|
||||
zipMethod = ZIP_METHOD_NONE;
|
||||
codec = CreateCodec_ZLibNone();
|
||||
}
|
||||
else
|
||||
{
|
||||
zipMethod = ZIP_METHOD_DEFLATE;
|
||||
codec = CreateCompressor_ZLibDeflate();
|
||||
}
|
||||
|
||||
// allocate memory
|
||||
const size_t pathnameLength = strlen(pathname);
|
||||
const size_t csizeMax = codec.get()->MaxOutputSize(usize);
|
||||
IoBuf buf = io_buf_Allocate(sizeof(LFH) + pathnameLength + csizeMax);
|
||||
|
||||
// read and compress file contents
|
||||
size_t csize; u32 checksum;
|
||||
{
|
||||
u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength;
|
||||
Stream stream(codec);
|
||||
stream.SetOutputBuffer(cdata, csizeMax);
|
||||
RETURN_ERR(io_Read(file, 0, IO_BUF_TEMP, usize, FeedStream, (uintptr_t)&stream));
|
||||
RETURN_ERR(stream.Finish());
|
||||
csize = stream.OutSize();
|
||||
checksum = stream.Checksum();
|
||||
}
|
||||
|
||||
ArchiveEntry archiveEntry(m_fileSize, usize, (off_t)csize, fileInfo.MTime(), checksum, zipMethod, 0);
|
||||
|
||||
// build LFH
|
||||
{
|
||||
LFH* lfh = (LFH*)buf.get();
|
||||
lfh->Init(archiveEntry, pathname, pathnameLength);
|
||||
}
|
||||
|
||||
// write to LFH, pathname and cdata to file
|
||||
const size_t packageSize = sizeof(LFH) + pathnameLength + csize;
|
||||
RETURN_ERR(io_Write(m_file, m_fileSize, buf, packageSize));
|
||||
m_fileSize += (off_t)packageSize;
|
||||
|
||||
// append a CDFH to the central dir (in memory)
|
||||
// .. note: pool_alloc may round size up for padding purposes.
|
||||
const size_t prev_pos = m_cdfhPool.da.pos;
|
||||
const size_t cdfhSize = sizeof(CDFH) + pathnameLength;
|
||||
CDFH* cdfh = (CDFH*)pool_alloc(&m_cdfhPool, cdfhSize);
|
||||
if(!cdfh)
|
||||
WARN_RETURN(ERR::NO_MEM);
|
||||
const size_t slack = m_cdfhPool.da.pos - prev_pos - cdfhSize;
|
||||
cdfh->Init(archiveEntry, pathname, pathnameLength, slack);
|
||||
|
||||
m_numEntries++;
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
File_Posix m_file;
|
||||
off_t m_fileSize;
|
||||
|
||||
Pool m_cdfhPool;
|
||||
uint m_numEntries;
|
||||
};
|
||||
|
||||
boost::shared_ptr<IArchiveWriter> CreateArchiveWriter_Zip(const char* archivePathname)
|
||||
{
|
||||
return boost::shared_ptr<IArchiveWriter>(new ArchiveWriter_Zip(archivePathname));
|
||||
}
|
20
source/lib/file/archive/archive_zip.h
Normal file
20
source/lib/file/archive/archive_zip.h
Normal file
@ -0,0 +1,20 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : archive_zip.h
|
||||
* Project : 0 A.D.
|
||||
* Description : archive backend for Zip files.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_ARCHIVE_ZIP
|
||||
#define INCLUDED_ARCHIVE_ZIP
|
||||
|
||||
struct IArchiveReader;
|
||||
boost::shared_ptr<IArchiveReader> CreateArchiveReader_Zip(const char* archivePathname);
|
||||
|
||||
struct IArchiveWriter;
|
||||
extern boost::shared_ptr<IArchiveWriter> CreateArchiveWriter_Zip(const char* archivePathname);
|
||||
|
||||
#endif // #ifndef INCLUDED_ARCHIVE_ZIP
|
16
source/lib/file/archive/codec.cpp
Normal file
16
source/lib/file/archive/codec.cpp
Normal file
@ -0,0 +1,16 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : codec.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "codec.h"
|
||||
|
||||
ICodec::~ICodec()
|
||||
{
|
||||
}
|
70
source/lib/file/archive/codec.h
Normal file
70
source/lib/file/archive/codec.h
Normal file
@ -0,0 +1,70 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : codec.h
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_CODEC
|
||||
#define INCLUDED_CODEC
|
||||
|
||||
// rationale: this layer allows for other compression methods/libraries
|
||||
// besides ZLib. it also simplifies the interface for user code and
|
||||
// does error checking, etc.
|
||||
|
||||
struct ICodec
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* note: the implementation should not check whether any data remains -
|
||||
* codecs are sometimes destroyed without completing a transfer.
|
||||
**/
|
||||
virtual ~ICodec();
|
||||
|
||||
/**
|
||||
* @return an upper bound on the output size for the given amount of input.
|
||||
* this is used when allocating a single buffer for the whole operation.
|
||||
**/
|
||||
virtual size_t MaxOutputSize(size_t inSize) const = 0;
|
||||
|
||||
/**
|
||||
* clear all previous state and prepare for reuse.
|
||||
*
|
||||
* this is as if the object were destroyed and re-created, but more
|
||||
* efficient since it avoids reallocating a considerable amount of
|
||||
* memory (about 200KB for LZ).
|
||||
**/
|
||||
virtual LibError Reset() = 0;
|
||||
|
||||
/**
|
||||
* process (i.e. compress or decompress) data.
|
||||
*
|
||||
* @param outSize bytes remaining in the output buffer; shall not be zero.
|
||||
* @param inConsumed, outProduced how many bytes in the input and
|
||||
* output buffers were used. either or both of these can be zero if
|
||||
* the input size is small or there's not enough output space.
|
||||
**/
|
||||
virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) = 0;
|
||||
|
||||
/**
|
||||
* flush buffers and make sure all output has been produced.
|
||||
*
|
||||
* @param checksum over all input data.
|
||||
* @return error status for the entire operation.
|
||||
**/
|
||||
virtual LibError Finish(u32& checksum) = 0;
|
||||
|
||||
/**
|
||||
* update a checksum to reflect the contents of a buffer.
|
||||
*
|
||||
* @param checksum the initial value (must be 0 on first call)
|
||||
* @return the new checksum. note: after all data has been seen, this is
|
||||
* identical to the what Finish would return.
|
||||
**/
|
||||
virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const = 0;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_CODEC
|
273
source/lib/file/archive/codec_zlib.cpp
Normal file
273
source/lib/file/archive/codec_zlib.cpp
Normal file
@ -0,0 +1,273 @@
|
||||
#include "precompiled.h"
|
||||
#include "codec_zlib.h"
|
||||
|
||||
#include "codec.h"
|
||||
#include "lib/external_libraries/zlib.h"
|
||||
|
||||
#include "lib/sysdep/cpu.h"
|
||||
|
||||
|
||||
class Codec_ZLib : public ICodec
|
||||
{
|
||||
public:
|
||||
u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const
|
||||
{
|
||||
return (u32)crc32(checksum, in, (uInt)inSize);
|
||||
}
|
||||
|
||||
protected:
|
||||
u32 InitializeChecksum()
|
||||
{
|
||||
return crc32(0, 0, 0);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class Codec_ZLibNone : public Codec_ZLib
|
||||
{
|
||||
public:
|
||||
Codec_ZLibNone()
|
||||
{
|
||||
Reset();
|
||||
}
|
||||
|
||||
virtual ~Codec_ZLibNone()
|
||||
{
|
||||
}
|
||||
|
||||
virtual size_t MaxOutputSize(size_t inSize) const
|
||||
{
|
||||
return inSize;
|
||||
}
|
||||
|
||||
virtual LibError Reset()
|
||||
{
|
||||
m_checksum = InitializeChecksum();
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed)
|
||||
{
|
||||
const size_t transferSize = std::min(inSize, outSize);
|
||||
cpu_memcpy(out, in, transferSize);
|
||||
inConsumed = outConsumed = transferSize;
|
||||
m_checksum = UpdateChecksum(m_checksum, in, inSize);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual LibError Finish(u32& checksum)
|
||||
{
|
||||
checksum = m_checksum;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
u32 m_checksum;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class CodecZLibStream : public Codec_ZLib
|
||||
{
|
||||
protected:
|
||||
CodecZLibStream()
|
||||
{
|
||||
memset(&m_zs, 0, sizeof(m_zs));
|
||||
m_checksum = InitializeChecksum();
|
||||
}
|
||||
|
||||
static LibError LibError_from_zlib(int zlib_ret)
|
||||
{
|
||||
switch(zlib_ret)
|
||||
{
|
||||
case Z_OK:
|
||||
return INFO::OK;
|
||||
case Z_STREAM_END:
|
||||
WARN_RETURN(ERR::FAIL);
|
||||
case Z_MEM_ERROR:
|
||||
WARN_RETURN(ERR::NO_MEM);
|
||||
case Z_DATA_ERROR:
|
||||
WARN_RETURN(ERR::CORRUPTED);
|
||||
case Z_STREAM_ERROR:
|
||||
WARN_RETURN(ERR::INVALID_PARAM);
|
||||
default:
|
||||
WARN_RETURN(ERR::FAIL);
|
||||
}
|
||||
}
|
||||
|
||||
static void WarnIfZLibError(int zlib_ret)
|
||||
{
|
||||
(void)LibError_from_zlib(zlib_ret);
|
||||
}
|
||||
|
||||
typedef int ZEXPORT (*ZLibFunc)(z_streamp strm, int flush);
|
||||
|
||||
LibError Process(ZLibFunc func, int flush, const u8* in, const size_t inSize, u8* out, const size_t outSize, size_t& inConsumed, size_t& outConsumed)
|
||||
{
|
||||
m_zs.next_in = (Byte*)in;
|
||||
m_zs.avail_in = (uInt)inSize;
|
||||
m_zs.next_out = (Byte*)out;
|
||||
m_zs.avail_out = (uInt)outSize;
|
||||
|
||||
int ret = func(&m_zs, flush);
|
||||
// sanity check: if ZLib reports end of stream, all input data
|
||||
// must have been consumed.
|
||||
if(ret == Z_STREAM_END)
|
||||
{
|
||||
debug_assert(m_zs.avail_in == 0);
|
||||
ret = Z_OK;
|
||||
}
|
||||
|
||||
debug_assert(inSize >= m_zs.avail_in && outSize >= m_zs.avail_out);
|
||||
inConsumed = inSize - m_zs.avail_in;
|
||||
outConsumed = outSize - m_zs.avail_out;
|
||||
|
||||
return LibError_from_zlib(ret);
|
||||
}
|
||||
|
||||
mutable z_stream m_zs;
|
||||
|
||||
// note: z_stream does contain an 'adler' checksum field, but that's
|
||||
// not updated in streams lacking a gzip header, so we'll have to
|
||||
// calculate a checksum ourselves.
|
||||
// adler32 is somewhat weaker than CRC32, but a more important argument
|
||||
// is that we should use the latter for compatibility with Zip archives.
|
||||
mutable u32 m_checksum;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class Compressor_ZLib : public CodecZLibStream
|
||||
{
|
||||
public:
|
||||
Compressor_ZLib()
|
||||
{
|
||||
// note: with Z_BEST_COMPRESSION, 78% percent of
|
||||
// archive builder CPU time is spent in ZLib, even though
|
||||
// that is interleaved with IO; everything else is negligible.
|
||||
// we therefore enable this only in final builds; during
|
||||
// development, 1.5% bigger archives are definitely worth much
|
||||
// faster build time.
|
||||
#if CONFIG_FINAL
|
||||
const int level = Z_BEST_COMPRESSION;
|
||||
#else
|
||||
const int level = Z_BEST_SPEED;
|
||||
#endif
|
||||
const int windowBits = -MAX_WBITS; // max window size; omit ZLib header
|
||||
const int memLevel = 9; // max speed; total mem ~= 384KiB
|
||||
const int strategy = Z_DEFAULT_STRATEGY; // normal data - not RLE
|
||||
const int ret = deflateInit2(&m_zs, level, Z_DEFLATED, windowBits, memLevel, strategy);
|
||||
debug_assert(ret == Z_OK);
|
||||
}
|
||||
|
||||
virtual ~Compressor_ZLib()
|
||||
{
|
||||
const int ret = deflateEnd(&m_zs);
|
||||
WarnIfZLibError(ret);
|
||||
}
|
||||
|
||||
virtual size_t MaxOutputSize(size_t inSize) const
|
||||
{
|
||||
return (size_t)deflateBound(&m_zs, (uLong)inSize);
|
||||
}
|
||||
|
||||
virtual LibError Reset()
|
||||
{
|
||||
m_checksum = InitializeChecksum();
|
||||
const int ret = deflateReset(&m_zs);
|
||||
return LibError_from_zlib(ret);
|
||||
}
|
||||
|
||||
virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed)
|
||||
{
|
||||
m_checksum = UpdateChecksum(m_checksum, in, inSize);
|
||||
return CodecZLibStream::Process(deflate, 0, in, inSize, out, outSize, inConsumed, outConsumed);
|
||||
}
|
||||
|
||||
virtual LibError Finish(u32& checksum)
|
||||
{
|
||||
// notify zlib that no more data is forthcoming and have it flush output.
|
||||
// our output buffer has enough space due to use of deflateBound;
|
||||
// therefore, deflate must return Z_STREAM_END.
|
||||
const int ret = deflate(&m_zs, Z_FINISH);
|
||||
debug_assert(ret == Z_STREAM_END);
|
||||
|
||||
checksum = m_checksum;
|
||||
return INFO::OK;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class Decompressor_ZLib : public CodecZLibStream
|
||||
{
|
||||
public:
|
||||
Decompressor_ZLib()
|
||||
{
|
||||
const int windowBits = -MAX_WBITS; // max window size; omit ZLib header
|
||||
const int ret = inflateInit2(&m_zs, windowBits);
|
||||
debug_assert(ret == Z_OK);
|
||||
}
|
||||
|
||||
virtual ~Decompressor_ZLib()
|
||||
{
|
||||
const int ret = inflateEnd(&m_zs);
|
||||
WarnIfZLibError(ret);
|
||||
}
|
||||
|
||||
virtual size_t MaxOutputSize(size_t inSize) const
|
||||
{
|
||||
// relying on an upper bound for the output is a really bad idea for
|
||||
// large files. archive formats store the uncompressed file sizes,
|
||||
// so callers should use that when allocating the output buffer.
|
||||
debug_assert(inSize < 1*MiB);
|
||||
|
||||
// http://www.zlib.org/zlib_tech.html
|
||||
return inSize*1032;
|
||||
}
|
||||
|
||||
virtual LibError Reset()
|
||||
{
|
||||
m_checksum = InitializeChecksum();
|
||||
const int ret = inflateReset(&m_zs);
|
||||
return LibError_from_zlib(ret);
|
||||
}
|
||||
|
||||
virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed)
|
||||
{
|
||||
const LibError ret = CodecZLibStream::Process(inflate, Z_SYNC_FLUSH, in, inSize, out, outSize, inConsumed, outConsumed);
|
||||
m_checksum = UpdateChecksum(m_checksum, in, inSize);
|
||||
return ret;
|
||||
}
|
||||
|
||||
virtual LibError Finish(u32& checksum)
|
||||
{
|
||||
// no action needed - decompression always flushes immediately.
|
||||
|
||||
checksum = m_checksum;
|
||||
return INFO::OK;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
boost::shared_ptr<ICodec> CreateCodec_ZLibNone()
|
||||
{
|
||||
return boost::shared_ptr<ICodec>(new Codec_ZLibNone);
|
||||
}
|
||||
|
||||
boost::shared_ptr<ICodec> CreateCompressor_ZLibDeflate()
|
||||
{
|
||||
return boost::shared_ptr<ICodec>(new Compressor_ZLib);
|
||||
}
|
||||
|
||||
boost::shared_ptr<ICodec> CreateDecompressor_ZLibDeflate()
|
||||
{
|
||||
return boost::shared_ptr<ICodec>(new Decompressor_ZLib);
|
||||
}
|
5
source/lib/file/archive/codec_zlib.h
Normal file
5
source/lib/file/archive/codec_zlib.h
Normal file
@ -0,0 +1,5 @@
|
||||
#include "codec.h"
|
||||
|
||||
extern boost::shared_ptr<ICodec> CreateCodec_ZLibNone();
|
||||
extern boost::shared_ptr<ICodec> CreateCompressor_ZLibDeflate();
|
||||
extern boost::shared_ptr<ICodec> CreateDecompressor_ZLibDeflate();
|
144
source/lib/file/archive/disabled_tests/test_archive_builder.h
Normal file
144
source/lib/file/archive/disabled_tests/test_archive_builder.h
Normal file
@ -0,0 +1,144 @@
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include "lib/base32.h"
|
||||
#include "lib/res/file/path.h"
|
||||
#include "lib/res/file/fp_posix.h"
|
||||
#include "lib/res/file/file_cache.h"
|
||||
#include "lib/res/file/vfs/vfs.h"
|
||||
#include "lib/res/file/archive/archive.h"
|
||||
#include "lib/res/file/archive/archive_builder.h"
|
||||
#include "lib/res/h_mgr.h"
|
||||
#include "lib/res/mem.h"
|
||||
#include "lib/rand.h"
|
||||
|
||||
class TestArchiveBuilder : public CxxTest::TestSuite
|
||||
{
|
||||
const char* const archive_fn;
|
||||
static const size_t NUM_FILES = 30;
|
||||
static const size_t MAX_FILE_SIZE = 20000;
|
||||
|
||||
std::set<const char*> existing_names;
|
||||
const char* gen_random_name()
|
||||
{
|
||||
// 10 chars is enough for (10-1)*5 bits = 45 bits > u32
|
||||
char name_tmp[10];
|
||||
|
||||
for(;;)
|
||||
{
|
||||
u32 rand_num = rand(0, 100000);
|
||||
base32(4, (const u8*)&rand_num, (u8*)name_tmp);
|
||||
|
||||
// store filename in atom pool
|
||||
const char* atom_fn = file_make_unique_fn_copy(name_tmp);
|
||||
// done if the filename is unique (not been generated yet)
|
||||
if(existing_names.find(atom_fn) == existing_names.end())
|
||||
{
|
||||
existing_names.insert(atom_fn);
|
||||
return atom_fn;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestFile
|
||||
{
|
||||
off_t size;
|
||||
u8* data; // must be delete[]-ed after comparing
|
||||
};
|
||||
// (must be separate array and end with NULL entry (see Filenames))
|
||||
const char* filenames[NUM_FILES+1];
|
||||
TestFile files[NUM_FILES];
|
||||
|
||||
void generate_random_files()
|
||||
{
|
||||
for(size_t i = 0; i < NUM_FILES; i++)
|
||||
{
|
||||
const off_t size = rand(0, MAX_FILE_SIZE);
|
||||
u8* data = new u8[size];
|
||||
|
||||
// random data won't compress at all, and we want to exercise
|
||||
// the uncompressed codepath as well => make some of the files
|
||||
// easily compressible (much less values).
|
||||
const bool make_easily_compressible = (rand(0, 100) > 50);
|
||||
if(make_easily_compressible)
|
||||
{
|
||||
for(off_t i = 0; i < size; i++)
|
||||
data[i] = rand() & 0x0F;
|
||||
}
|
||||
else
|
||||
{
|
||||
for(off_t i = 0; i < size; i++)
|
||||
data[i] = rand() & 0xFF;
|
||||
}
|
||||
|
||||
filenames[i] = gen_random_name();
|
||||
files[i].size = size;
|
||||
files[i].data = data;
|
||||
|
||||
ssize_t bytes_written = vfs_store(filenames[i], data, size, FILE_NO_AIO);
|
||||
TS_ASSERT_EQUALS(bytes_written, size);
|
||||
}
|
||||
|
||||
// 0-terminate the list - see Filenames decl.
|
||||
filenames[NUM_FILES] = NULL;
|
||||
}
|
||||
|
||||
public:
|
||||
TestArchiveBuilder()
|
||||
: archive_fn("test_archive_random_data.zip") {}
|
||||
|
||||
void setUp()
|
||||
{
|
||||
(void)path_SetRoot(0, ".");
|
||||
vfs_init();
|
||||
}
|
||||
|
||||
void tearDown()
|
||||
{
|
||||
vfs_shutdown();
|
||||
path_ResetRootDir();
|
||||
}
|
||||
|
||||
void test_create_archive_with_random_files()
|
||||
{
|
||||
if(!file_exists("archivetest")) // don't get stuck if this test fails and never deletes the directory it created
|
||||
TS_ASSERT_OK(dir_create("archivetest"));
|
||||
|
||||
TS_ASSERT_OK(vfs_mount("", "archivetest"));
|
||||
|
||||
generate_random_files();
|
||||
TS_ASSERT_OK(archive_build(archive_fn, filenames));
|
||||
|
||||
// wipe out file cache, otherwise we're just going to get back
|
||||
// the file contents read during archive_build .
|
||||
file_cache_reset();
|
||||
|
||||
// read in each file and compare file contents
|
||||
Handle ha = archive_open(archive_fn);
|
||||
TS_ASSERT(ha > 0);
|
||||
for(size_t i = 0; i < NUM_FILES; i++)
|
||||
{
|
||||
File f;
|
||||
TS_ASSERT_OK(afile_open(ha, filenames[i], 0, 0, &f));
|
||||
FileIOBuf buf = FILE_BUF_ALLOC;
|
||||
ssize_t bytes_read = afile_read(&f, 0, files[i].size, &buf);
|
||||
TS_ASSERT_EQUALS(bytes_read, files[i].size);
|
||||
|
||||
TS_ASSERT_SAME_DATA(buf, files[i].data, files[i].size);
|
||||
|
||||
TS_ASSERT_OK(file_cache_free(buf));
|
||||
TS_ASSERT_OK(afile_close(&f));
|
||||
SAFE_ARRAY_DELETE(files[i].data);
|
||||
}
|
||||
TS_ASSERT_OK(archive_close(ha));
|
||||
|
||||
dir_delete("archivetest");
|
||||
file_delete(archive_fn);
|
||||
}
|
||||
|
||||
void test_multiple_init_shutdown()
|
||||
{
|
||||
// setUp has already vfs_init-ed it and tearDown will vfs_shutdown.
|
||||
vfs_shutdown();
|
||||
vfs_init();
|
||||
}
|
||||
};
|
59
source/lib/file/archive/disabled_tests/test_codec_zlib.h
Normal file
59
source/lib/file/archive/disabled_tests/test_codec_zlib.h
Normal file
@ -0,0 +1,59 @@
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include "lib/self_test.h"
|
||||
#include "lib/res/file/archive/codec_zlib.h"
|
||||
|
||||
class TestCodecZLib : public CxxTest::TestSuite
|
||||
{
|
||||
public:
|
||||
void test_compress_decompress_compare()
|
||||
{
|
||||
size_t inConsumed, outProduced;
|
||||
u32 checksum;
|
||||
|
||||
// generate random input udata
|
||||
// (limit values to 0..7 so that the udata will actually be compressible)
|
||||
const size_t usize = 10000;
|
||||
u8 udata[usize];
|
||||
for(size_t i = 0; i < usize; i++)
|
||||
udata[i] = rand() & 0x07;
|
||||
|
||||
// compress
|
||||
u8* cdata; size_t csize;
|
||||
{
|
||||
boost::shared_ptr<ICodec> compressor_zlib = CreateCompressor_ZLib();
|
||||
ICodec* c = compressor_zlib.get();
|
||||
const size_t csizeMax = c->MaxOutputSize(usize);
|
||||
cdata = new u8[csizeMax];
|
||||
TS_ASSERT_OK(c->Process(udata, usize, cdata, csizeMax, inConsumed, outProduced));
|
||||
TS_ASSERT_EQUALS(inConsumed, usize);
|
||||
TS_ASSERT_LESS_THAN_EQUALS(outProduced, csizeMax);
|
||||
u8* cdata2;
|
||||
TS_ASSERT_OK(c->Finish(cdata2, csize, checksum));
|
||||
TS_ASSERT_EQUALS(cdata, cdata2);
|
||||
TS_ASSERT_EQUALS(csize, outProduced);
|
||||
}
|
||||
|
||||
// make sure the data changed during compression
|
||||
TS_ASSERT(csize != usize || memcmp(udata, cdata, std::min(usize, csize)) != 0);
|
||||
|
||||
// decompress
|
||||
u8 ddata[usize];
|
||||
{
|
||||
boost::shared_ptr<ICodec> decompressor_zlib = CreateDecompressor_ZLib();
|
||||
ICodec* d = decompressor_zlib.get();
|
||||
TS_ASSERT_OK(decompressor_zlib->Process(cdata, csize, ddata, usize, inConsumed, outProduced));
|
||||
TS_ASSERT_EQUALS(inConsumed, csize); // ZLib always outputs as much data as possible
|
||||
TS_ASSERT_EQUALS(outProduced, usize); // .. so these figures are correct before Finish()
|
||||
u8* ddata2; size_t dsize;
|
||||
TS_ASSERT_OK(d->Finish(&ddata2, &dsize, &checksum));
|
||||
TS_ASSERT_EQUALS(ddata, ddata2);
|
||||
TS_ASSERT_EQUALS(dsize, outProduced);
|
||||
}
|
||||
|
||||
// verify udata survived intact
|
||||
TS_ASSERT_SAME_DATA(udata, ddata, usize);
|
||||
|
||||
delete[] cdata;
|
||||
}
|
||||
};
|
54
source/lib/file/archive/disabled_tests/test_compression.h
Normal file
54
source/lib/file/archive/disabled_tests/test_compression.h
Normal file
@ -0,0 +1,54 @@
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include "lib/self_test.h"
|
||||
#include "lib/res/file/archive/compression.h"
|
||||
|
||||
class TestCompression : public CxxTest::TestSuite
|
||||
{
|
||||
public:
|
||||
void test_compress_decompress_compare()
|
||||
{
|
||||
// generate random input data
|
||||
// (limit values to 0..7 so that the data will actually be compressible)
|
||||
const size_t data_size = 10000;
|
||||
u8 data[data_size];
|
||||
for(size_t i = 0; i < data_size; i++)
|
||||
data[i] = rand() & 0x07;
|
||||
|
||||
u8* cdata; size_t csize;
|
||||
u8 udata[data_size];
|
||||
|
||||
// compress
|
||||
uintptr_t c = comp_alloc(CT_COMPRESSION, CM_DEFLATE);
|
||||
{
|
||||
TS_ASSERT(c != 0);
|
||||
const size_t csizeBound = comp_max_output_size(c, data_size);
|
||||
TS_ASSERT_OK(comp_alloc_output(c, csizeBound));
|
||||
const ssize_t cdata_produced = comp_feed(c, data, data_size);
|
||||
TS_ASSERT(cdata_produced >= 0);
|
||||
u32 checksum;
|
||||
TS_ASSERT_OK(comp_finish(c, &cdata, &csize, &checksum));
|
||||
TS_ASSERT(cdata_produced <= (ssize_t)csize); // can't have produced more than total
|
||||
}
|
||||
|
||||
// decompress
|
||||
uintptr_t d = comp_alloc(CT_DECOMPRESSION, CM_DEFLATE);
|
||||
{
|
||||
TS_ASSERT(d != 0);
|
||||
comp_set_output(d, udata, data_size);
|
||||
const ssize_t udata_produced = comp_feed(d, cdata, csize);
|
||||
TS_ASSERT(udata_produced >= 0);
|
||||
u8* udata_final; size_t usize_final; u32 checksum;
|
||||
TS_ASSERT_OK(comp_finish(d, &udata_final, &usize_final, &checksum));
|
||||
TS_ASSERT(udata_produced <= (ssize_t)usize_final); // can't have produced more than total
|
||||
TS_ASSERT_EQUALS(udata_final, udata); // output buffer address is same
|
||||
TS_ASSERT_EQUALS(usize_final, data_size); // correct amount of output
|
||||
}
|
||||
|
||||
comp_free(c);
|
||||
comp_free(d);
|
||||
|
||||
// verify data survived intact
|
||||
TS_ASSERT_SAME_DATA(data, udata, data_size);
|
||||
}
|
||||
};
|
25
source/lib/file/archive/disabled_tests/test_fat_time.h
Normal file
25
source/lib/file/archive/disabled_tests/test_fat_time.h
Normal file
@ -0,0 +1,25 @@
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include <ctime>
|
||||
|
||||
#include "lib/res/file/archive/fat_time.h"
|
||||
|
||||
class TestFatTime: public CxxTest::TestSuite
|
||||
{
|
||||
public:
|
||||
void test_fat_timedate_conversion()
|
||||
{
|
||||
// note: FAT time stores second/2, which means converting may
|
||||
// end up off by 1 second.
|
||||
|
||||
time_t t, converted_t;
|
||||
|
||||
t = time(0);
|
||||
converted_t = time_t_from_FAT(FAT_from_time_t(t));
|
||||
TS_ASSERT_DELTA(t, converted_t, 2);
|
||||
|
||||
t++;
|
||||
converted_t = time_t_from_FAT(FAT_from_time_t(t));
|
||||
TS_ASSERT_DELTA(t, converted_t, 2);
|
||||
}
|
||||
};
|
10
source/lib/file/archive/disabled_tests/test_zip.h
Normal file
10
source/lib/file/archive/disabled_tests/test_zip.h
Normal file
@ -0,0 +1,10 @@
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include <time.h>
|
||||
|
||||
#include "lib/res/file/archive/zip.h"
|
||||
|
||||
class TestZip : public CxxTest::TestSuite
|
||||
{
|
||||
public:
|
||||
};
|
127
source/lib/file/archive/stream.cpp
Normal file
127
source/lib/file/archive/stream.cpp
Normal file
@ -0,0 +1,127 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : stream.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "stream.h"
|
||||
|
||||
#include "lib/allocators/allocators.h" // page_aligned_alloc
|
||||
#include "codec.h"
|
||||
|
||||
|
||||
OutputBufferManager::OutputBufferManager()
|
||||
{
|
||||
Reset();
|
||||
}
|
||||
|
||||
void OutputBufferManager::Reset()
|
||||
{
|
||||
m_buffer = 0;
|
||||
m_size = 0;
|
||||
m_capacity = 0;
|
||||
}
|
||||
|
||||
void OutputBufferManager::SetBuffer(u8* buffer, size_t size)
|
||||
{
|
||||
debug_assert(IsAllowableBuffer(buffer, size));
|
||||
|
||||
m_buffer = buffer;
|
||||
m_size = size;
|
||||
}
|
||||
|
||||
void OutputBufferManager::AllocateBuffer(size_t size)
|
||||
{
|
||||
// notes:
|
||||
// - this implementation allows reusing previous buffers if they
|
||||
// are big enough, which reduces the number of allocations.
|
||||
// - no further attempts to reduce allocations (e.g. by doubling
|
||||
// the current size) are made; this strategy is enough.
|
||||
// - Pool etc. cannot be used because files may be huge (larger
|
||||
// than the address space of 32-bit systems).
|
||||
|
||||
// no buffer or the previous one wasn't big enough: reallocate
|
||||
if(!m_mem.get() || m_capacity < size)
|
||||
{
|
||||
m_mem.reset((u8*)page_aligned_alloc(size), PageAlignedDeleter(size));
|
||||
m_capacity = size;
|
||||
}
|
||||
|
||||
SetBuffer(m_mem.get(), size);
|
||||
}
|
||||
|
||||
bool OutputBufferManager::IsAllowableBuffer(u8* buffer, size_t size)
|
||||
{
|
||||
// none yet established
|
||||
if(m_buffer == 0 && m_size == 0)
|
||||
return true;
|
||||
|
||||
// same as last time (happens with temp buffers)
|
||||
if(m_buffer == buffer && m_size == size)
|
||||
return true;
|
||||
|
||||
// located after the last buffer (note: not necessarily after
|
||||
// the entire buffer; a lack of input can cause the output buffer
|
||||
// to only partially be used before the next call.)
|
||||
if((unsigned)(buffer - m_buffer) <= m_size)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
|
||||
Stream::Stream(boost::shared_ptr<ICodec> codec)
|
||||
: m_codec(codec)
|
||||
, m_inConsumed(0), m_outProduced(0)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void Stream::AllocateOutputBuffer(size_t outSizeMax)
|
||||
{
|
||||
m_outputBufferManager.AllocateBuffer(outSizeMax);
|
||||
}
|
||||
|
||||
|
||||
void Stream::SetOutputBuffer(u8* out, size_t outSize)
|
||||
{
|
||||
m_outputBufferManager.SetBuffer(out, outSize);
|
||||
}
|
||||
|
||||
|
||||
LibError Stream::Feed(const u8* in, size_t inSize, size_t& bytesProcessed)
|
||||
{
|
||||
if(m_outProduced == m_outputBufferManager.Size()) // output buffer full; must not call Process
|
||||
return INFO::OK;
|
||||
|
||||
size_t inConsumed, outProduced;
|
||||
u8* const out = m_outputBufferManager.Buffer();
|
||||
const size_t outSize = m_outputBufferManager.Size() - m_outProduced;
|
||||
RETURN_ERR(m_codec.get()->Process(in, inSize, out, outSize, inConsumed, outProduced));
|
||||
|
||||
m_inConsumed += inConsumed;
|
||||
m_outProduced += outProduced;
|
||||
bytesProcessed = outProduced;
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
|
||||
|
||||
LibError Stream::Finish()
|
||||
{
|
||||
return m_codec.get()->Finish(m_checksum);
|
||||
}
|
||||
|
||||
|
||||
LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize, size_t& bytesProcessed)
|
||||
{
|
||||
Stream& stream = *(Stream*)cbData;
|
||||
return stream.Feed(in, inSize, bytesProcessed);
|
||||
}
|
103
source/lib/file/archive/stream.h
Normal file
103
source/lib/file/archive/stream.h
Normal file
@ -0,0 +1,103 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : stream.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_STREAM
|
||||
#define INCLUDED_STREAM
|
||||
|
||||
struct ICodec;
|
||||
|
||||
// note: this is similar in function to std::vector, but we don't need
|
||||
// iterators etc. and would prefer to avoid initializing each byte.
|
||||
class OutputBufferManager
|
||||
{
|
||||
public:
|
||||
OutputBufferManager();
|
||||
|
||||
void Reset();
|
||||
void SetBuffer(u8* buffer, size_t size);
|
||||
|
||||
/**
|
||||
* allocate a new output buffer.
|
||||
*
|
||||
* @param size [bytes] to allocate.
|
||||
*
|
||||
* notes:
|
||||
* - if a buffer had previously been allocated and is large enough,
|
||||
* it is reused (this reduces the number of allocations).
|
||||
* - this class manages the lifetime of the buffer.
|
||||
**/
|
||||
void AllocateBuffer(size_t size);
|
||||
|
||||
u8* Buffer() const
|
||||
{
|
||||
return m_buffer;
|
||||
}
|
||||
|
||||
size_t Size() const
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
|
||||
private:
|
||||
bool IsAllowableBuffer(u8* buffer, size_t size);
|
||||
|
||||
u8* m_buffer;
|
||||
size_t m_size;
|
||||
|
||||
boost::shared_array<u8> m_mem;
|
||||
// size of m_mem. allows reusing previously allocated buffers
|
||||
// (user-specified buffers can't be reused because we have no control
|
||||
// over their lifetime)
|
||||
size_t m_capacity;
|
||||
};
|
||||
|
||||
|
||||
class Stream
|
||||
{
|
||||
public:
|
||||
Stream(boost::shared_ptr<ICodec> codec);
|
||||
|
||||
void SetOutputBuffer(u8* out, size_t outSize);
|
||||
|
||||
void AllocateOutputBuffer(size_t outSizeMax);
|
||||
|
||||
/**
|
||||
* 'feed' the codec with a data block.
|
||||
*
|
||||
* @param bytesProcessed receives the number of output bytes produced.
|
||||
* it can legitimately be 0 - this happens if the input buffer is small
|
||||
* and the codec hasn't produced any output.
|
||||
**/
|
||||
LibError Feed(const u8* in, size_t inSize, size_t& bytesProcessed);
|
||||
|
||||
LibError Finish();
|
||||
|
||||
size_t OutSize() const
|
||||
{
|
||||
return m_outProduced;
|
||||
}
|
||||
|
||||
u32 Checksum() const
|
||||
{
|
||||
return m_checksum;
|
||||
}
|
||||
|
||||
private:
|
||||
boost::shared_ptr<ICodec> m_codec;
|
||||
OutputBufferManager m_outputBufferManager;
|
||||
|
||||
size_t m_inConsumed;
|
||||
size_t m_outProduced;
|
||||
u32 m_checksum;
|
||||
};
|
||||
|
||||
extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize, size_t& bytesProcessed);
|
||||
|
||||
#endif // #ifndef INCLUDED_STREAM
|
235
source/lib/file/archive/trace.cpp
Normal file
235
source/lib/file/archive/trace.cpp
Normal file
@ -0,0 +1,235 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : trace.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : allows recording and 'playing back' a sequence of
|
||||
* : I/Os - useful for benchmarking and archive builder.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "trace.h"
|
||||
|
||||
#include "lib/allocators.h"
|
||||
#include "lib/timer.h"
|
||||
#include "lib/sysdep/cpu.h"
|
||||
|
||||
// one run per file
|
||||
|
||||
|
||||
/*virtual*/ ITrace::~ITrace()
|
||||
{
|
||||
}
|
||||
|
||||
class Trace_Dummy : public ITrace
|
||||
{
|
||||
public:
|
||||
Trace_Dummy(size_t UNUSED(maxSize))
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
virtual void NotifyLoad(const char* UNUSED(pathname), size_t UNUSED(size))
|
||||
{
|
||||
}
|
||||
|
||||
virtual void NotifyStore(const char* UNUSED(pathname), size_t UNUSED(size))
|
||||
{
|
||||
}
|
||||
|
||||
virtual void NotifyFree(const char* UNUSED(pathname))
|
||||
{
|
||||
}
|
||||
|
||||
virtual LibError Load(const char* UNUSED(vfsPathname))
|
||||
{
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual LibError Store(const char* UNUSED(vfsPathname)) const
|
||||
{
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual const TraceEntry* Entries() const
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual size_t NumEntries() const
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class Trace : public ITrace
|
||||
{
|
||||
public:
|
||||
Trace(size_t maxSize)
|
||||
{
|
||||
(void)pool_create(&m_pool, maxSize, sizeof(TraceEntry));
|
||||
}
|
||||
|
||||
virtual ~Trace()
|
||||
{
|
||||
(void)pool_destroy(&m_pool);
|
||||
}
|
||||
|
||||
virtual void NotifyLoad(const char* pathname, size_t size)
|
||||
{
|
||||
AddEntry(TO_LOAD, pathname, size);
|
||||
}
|
||||
|
||||
virtual void NotifyStore(const char* pathname, size_t size)
|
||||
{
|
||||
AddEntry(TO_STORE, pathname, size);
|
||||
}
|
||||
|
||||
virtual void NotifyFree(const char* pathname)
|
||||
{
|
||||
const size_t size = 0; // meaningless for TO_FREE
|
||||
AddEntry(TO_FREE, pathname, size);
|
||||
}
|
||||
|
||||
virtual LibError Load(const char* osPathname)
|
||||
{
|
||||
pool_free_all(&m_pool);
|
||||
|
||||
FILE* f = fopen(osPathname, "rt");
|
||||
if(!f)
|
||||
WARN_RETURN(ERR::FILE_ACCESS);
|
||||
for(size_t i = 0; ; i++)
|
||||
{
|
||||
TraceEntry* t = (TraceEntry*)pool_alloc(&m_pool, 0);
|
||||
t->Read(f);
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual LibError Store(const char* osPathname) const
|
||||
{
|
||||
FILE* f = fopen(osPathname, "at");
|
||||
if(!f)
|
||||
WARN_RETURN(ERR::FILE_ACCESS);
|
||||
for(size_t i = 0; i < NumEntries(); i++)
|
||||
Entries()[i]->Write(f);
|
||||
(void)fclose(f);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual const TraceEntry* Entries() const
|
||||
{
|
||||
return (const TraceEntry*)m_pool.da.base;
|
||||
}
|
||||
|
||||
virtual size_t NumEntries() const
|
||||
{
|
||||
return m_pool.da.pos / sizeof(TraceEntry);
|
||||
}
|
||||
|
||||
private:
|
||||
void AddEntry(TraceOp op, const char* pathname, size_t size, double timestamp = 0.0)
|
||||
{
|
||||
TraceEntry* t = (TraceEntry*)pool_alloc(&m_pool, 0);
|
||||
debug_assert(t);
|
||||
t->timestamp = timestamp == 0.0? get_time() : timestamp;
|
||||
t->vfsPathname = path_Pool()->UniqueCopy(pathname);
|
||||
t->size = size;
|
||||
t->op = op;
|
||||
}
|
||||
|
||||
Pool m_pool;
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// parse lines and stuff them in m_pool
|
||||
// (as if they had been AddEntry-ed; replaces any existing data)
|
||||
// .. bake PATH_MAX limit into string.
|
||||
char fmt[30];
|
||||
snprintf(fmt, ARRAY_SIZE(fmt), "%%lf: %%c \"%%%d[^\"]\" %%d %%04x\n", PATH_MAX);
|
||||
|
||||
double timestamp; char opcode; char P_path[PATH_MAX]; size_t size;
|
||||
int fieldsRead = fscanf(f, fmt, ×tamp, &opcode, P_path, &size);
|
||||
if(fieldsRead == EOF)
|
||||
break;
|
||||
debug_assert(fieldsRead == 4);
|
||||
|
||||
|
||||
|
||||
static char CharFromTraceOp(TraceOp op)
|
||||
{
|
||||
switch(op)
|
||||
{
|
||||
case TO_LOAD:
|
||||
return 'L';
|
||||
case TO_STORE:
|
||||
return 'S';
|
||||
case TO_FREE:
|
||||
return 'F';
|
||||
default:
|
||||
DEBUG_WARN_ERR(ERR::CORRUPTED);
|
||||
}
|
||||
}
|
||||
|
||||
static TraceOp TraceOpFromChar(char c)
|
||||
{
|
||||
switch(c)
|
||||
{
|
||||
case 'L':
|
||||
return TO_LOAD;
|
||||
case 'S':
|
||||
return TO_STORE;
|
||||
case 'F':
|
||||
return TO_FREE;
|
||||
default:
|
||||
DEBUG_WARN_ERR(ERR::CORRUPTED);
|
||||
}
|
||||
}
|
||||
|
||||
static void write_entry(FILE* f, const TraceEntry* ent)
|
||||
{
|
||||
fprintf(f, "%#010f: %c \"%s\" %d\n", ent->timestamp, CharFromTraceOp(ent->op), ent->vfsPathname, ent->size);
|
||||
}
|
||||
|
||||
{
|
||||
// carry out this entry's operation
|
||||
switch(ent->op)
|
||||
{
|
||||
// do not 'run' writes - we'd destroy the existing data.
|
||||
case TO_STORE:
|
||||
break;
|
||||
case TO_LOAD:
|
||||
{
|
||||
IoBuf buf; size_t size;
|
||||
(void)vfs_load(ent->vfsPathname, buf, size, ent->flags);
|
||||
break;
|
||||
}
|
||||
case TO_FREE:
|
||||
fileCache.Release(ent->vfsPathname);
|
||||
break;
|
||||
default:
|
||||
debug_warn("unknown TraceOp");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
LibError trace_run(const char* osPathname)
|
||||
{
|
||||
Trace trace;
|
||||
RETURN_ERR(trace.Load(osPathname));
|
||||
for(uint i = 0; i < trace.NumEntries(); i++)
|
||||
trace.Entries()[i]->Run();
|
||||
return INFO::OK;
|
||||
}
|
66
source/lib/file/archive/trace.h
Normal file
66
source/lib/file/archive/trace.h
Normal file
@ -0,0 +1,66 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : trace.h
|
||||
* Project : 0 A.D.
|
||||
* Description : allows recording and 'playing back' a sequence of
|
||||
* : I/Os - useful for benchmarking and archive builder.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_TRACE
|
||||
#define INCLUDED_TRACE
|
||||
|
||||
// TraceEntry operation type.
|
||||
// note: rather than only a list of accessed files, we also need to
|
||||
// know the application's behavior WRT caching (e.g. when it releases
|
||||
// cached buffers). this is necessary so that our simulation can
|
||||
// yield the same behavior.
|
||||
enum TraceOp
|
||||
{
|
||||
TO_LOAD,
|
||||
TO_STORE,
|
||||
TO_FREE,
|
||||
};
|
||||
|
||||
// stores one event that is relevant for file IO / caching.
|
||||
//
|
||||
// size-optimized a bit since these are all kept in memory
|
||||
// (to prevent trace file writes from affecting other IOs)
|
||||
struct TraceEntry
|
||||
{
|
||||
// note: float instead of double for nice 16 byte struct size
|
||||
float timestamp; // returned by get_time before operation starts
|
||||
const char* vfsPathname;
|
||||
// rationale: store size in the trace because other applications
|
||||
// that use this trace format but not our IO code wouldn't know
|
||||
// size (since they cannot retrieve the file info given vfsPathname).
|
||||
size_t size; // of IO (usually the entire file)
|
||||
TraceOp op;
|
||||
};
|
||||
|
||||
struct ITrace
|
||||
{
|
||||
virtual ~ITrace();
|
||||
|
||||
virtual void NotifyLoad(const char* vfsPathname, size_t size);
|
||||
virtual void NotifyStore(const char* vfsPathname, size_t size);
|
||||
virtual void NotifyFree(const char* vfsPathname);
|
||||
|
||||
// note: file format is text-based to allow human inspection and because
|
||||
// storing filenames would be a bit awkward
|
||||
virtual LibError Load(const char* osPathname);
|
||||
virtual LibError Store(const char* osPathname) const;
|
||||
|
||||
virtual const TraceEntry* Entries() const;
|
||||
virtual size_t NumEntries() const;
|
||||
};
|
||||
|
||||
|
||||
// enabled by default. by the time we can decide whether a trace needs to
|
||||
// be generated (see should_rebuild_main_archive), file accesses will
|
||||
// already have occurred; hence default enabled and disable if not needed.
|
||||
|
||||
|
||||
#endif // #ifndef INCLUDED_TRACE
|
@ -21,30 +21,30 @@
|
||||
|
||||
bool dir_FileExists(IFilesystem* fs, const char* pathname)
|
||||
{
|
||||
FilesystemEntry fsEntry;
|
||||
if(fs->GetEntry(pathname, fsEntry) < 0)
|
||||
FileInfo fileInfo;
|
||||
if(fs->GetEntry(pathname, fileInfo) < 0)
|
||||
return false;
|
||||
debug_assert(!fsEntry.IsDirectory());
|
||||
debug_assert(!fileInfo.IsDirectory());
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool dir_DirectoryExists(IFilesystem* fs, const char* dirPath)
|
||||
{
|
||||
FilesystemEntry fsEntry;
|
||||
if(fs->GetEntry(dirPath, fsEntry) < 0)
|
||||
FileInfo fileInfo;
|
||||
if(fs->GetEntry(dirPath, fileInfo) < 0)
|
||||
return false;
|
||||
debug_assert(fsEntry.IsDirectory());
|
||||
debug_assert(fileInfo.IsDirectory());
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
struct FsEntryNameLess : public std::binary_function<const FilesystemEntry, const FilesystemEntry, bool>
|
||||
struct FsEntryNameLess : public std::binary_function<const FileInfo, const FileInfo, bool>
|
||||
{
|
||||
bool operator()(const FilesystemEntry& fsEntry1, const FilesystemEntry& fsEntry2) const
|
||||
bool operator()(const FileInfo& fileInfo1, const FileInfo& fileInfo2) const
|
||||
{
|
||||
return strcmp(fsEntry1.name, fsEntry2.name) < 0;
|
||||
return strcmp(fileInfo1.name, fileInfo2.name) < 0;
|
||||
}
|
||||
};
|
||||
|
||||
@ -53,14 +53,14 @@ LibError dir_GatherSortedEntries(IFilesystem* fs, const char* dirPath, Filesyste
|
||||
DirectoryIterator di(fs, dirPath);
|
||||
fsEntries.reserve(50); // preallocate for efficiency
|
||||
|
||||
FilesystemEntry fsEntry;
|
||||
FileInfo fileInfo;
|
||||
for(;;)
|
||||
{
|
||||
LibError ret = di.NextEntry(fsEntry);
|
||||
LibError ret = di.NextEntry(fileInfo);
|
||||
if(ret == ERR::DIR_END)
|
||||
break;
|
||||
RETURN_ERR(ret);
|
||||
fsEntries.push_back(fsEntry);
|
||||
fsEntries.push_back(fileInfo);
|
||||
}
|
||||
|
||||
std::sort(fsEntries.begin(), fsEntries.end(), FsEntryNameLess());
|
||||
@ -79,10 +79,10 @@ LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, const DirC
|
||||
|
||||
for(FilesystemEntries::const_iterator it = fsEntries.begin(); it != fsEntries.end(); ++it)
|
||||
{
|
||||
const FilesystemEntry& fsEntry = *it;
|
||||
path_package_append_file(&pp, fsEntry.name);
|
||||
const FileInfo& fileInfo = *it;
|
||||
path_package_append_file(&pp, fileInfo.name);
|
||||
|
||||
LibError ret = cb(pp.path, fsEntry, cbData);
|
||||
LibError ret = cb(pp.path, fileInfo, cbData);
|
||||
if(ret != INFO::CB_CONTINUE)
|
||||
return ret;
|
||||
}
|
||||
@ -91,7 +91,7 @@ LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, const DirC
|
||||
}
|
||||
|
||||
|
||||
LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry, const char* filter)
|
||||
LibError dir_filtered_next_ent(DirectoryIterator& di, FileInfo& fileInfo, const char* filter)
|
||||
{
|
||||
bool want_dir = true;
|
||||
if(filter)
|
||||
@ -108,12 +108,12 @@ LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry,
|
||||
want_dir = false;
|
||||
}
|
||||
|
||||
// loop until fsEntry matches what is requested, or end of directory.
|
||||
// loop until fileInfo matches what is requested, or end of directory.
|
||||
for(;;)
|
||||
{
|
||||
RETURN_ERR(di.NextEntry(fsEntry));
|
||||
RETURN_ERR(di.NextEntry(fileInfo));
|
||||
|
||||
if(fsEntry.IsDirectory())
|
||||
if(fileInfo.IsDirectory())
|
||||
{
|
||||
if(want_dir)
|
||||
break;
|
||||
@ -121,7 +121,7 @@ LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry,
|
||||
else
|
||||
{
|
||||
// (note: filter = 0 matches anything)
|
||||
if(match_wildcard(fsEntry.name, filter))
|
||||
if(match_wildcard(fileInfo.name, filter))
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -130,11 +130,11 @@ LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry,
|
||||
}
|
||||
|
||||
|
||||
// call <cb> for each fsEntry matching <user_filter> (see vfs_next_dirent) in
|
||||
// call <cb> for each fileInfo matching <user_filter> (see vfs_next_dirent) in
|
||||
// directory <path>; if flags & VFS_DIR_RECURSIVE, entries in
|
||||
// subdirectories are also returned.
|
||||
//
|
||||
// note: EnumDirEntsCB path and fsEntry are only valid during the callback.
|
||||
// note: EnumDirEntsCB path and fileInfo are only valid during the callback.
|
||||
LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint flags, const char* user_filter, DirCallback cb, uintptr_t cbData)
|
||||
{
|
||||
debug_assert((flags & ~(VFS_DIR_RECURSIVE)) == 0);
|
||||
@ -162,7 +162,7 @@ LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint fla
|
||||
// directory consecutively)
|
||||
|
||||
std::queue<const char*> dir_queue;
|
||||
dir_queue.push(path_UniqueCopy(dirPath));
|
||||
dir_queue.push(path_Pool()->UniqueCopy(dirPath));
|
||||
|
||||
// for each directory:
|
||||
do
|
||||
@ -176,24 +176,24 @@ LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint fla
|
||||
|
||||
DirectoryIterator di(fs, pp.path);
|
||||
|
||||
// for each fsEntry (file, subdir) in directory:
|
||||
FilesystemEntry fsEntry;
|
||||
while(dir_filtered_next_ent(di, fsEntry, filter) == 0)
|
||||
// for each fileInfo (file, subdir) in directory:
|
||||
FileInfo fileInfo;
|
||||
while(dir_filtered_next_ent(di, fileInfo, filter) == 0)
|
||||
{
|
||||
// build complete path (FilesystemEntry only stores fsEntry name)
|
||||
(void)path_package_append_file(&pp, fsEntry.name);
|
||||
const char* atom_path = path_UniqueCopy(pp.path);
|
||||
// build complete path (FileInfo only stores fileInfo name)
|
||||
(void)path_package_append_file(&pp, fileInfo.name);
|
||||
const char* atom_path = path_Pool()->UniqueCopy(pp.path);
|
||||
|
||||
if(fsEntry.IsDirectory())
|
||||
if(fileInfo.IsDirectory())
|
||||
{
|
||||
if(recursive)
|
||||
dir_queue.push(atom_path);
|
||||
|
||||
if(user_filter_wants_dirs)
|
||||
cb(atom_path, fsEntry, cbData);
|
||||
cb(atom_path, fileInfo, cbData);
|
||||
}
|
||||
else
|
||||
cb(atom_path, fsEntry, cbData);
|
||||
cb(atom_path, fileInfo, cbData);
|
||||
}
|
||||
}
|
||||
while(!dir_queue.empty());
|
||||
@ -207,9 +207,6 @@ LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint fla
|
||||
// the next numbered filename according to the pattern defined by V_fn_fmt.
|
||||
// <state> must be initially zeroed (e.g. by defining as static) and passed
|
||||
// each time.
|
||||
// if <use_vfs> (default), the paths are treated as VFS paths; otherwise,
|
||||
// file.cpp's functions are used. this is necessary because one of
|
||||
// our callers needs a filename for VFS archive files.
|
||||
//
|
||||
// this function is useful when creating new files which are not to
|
||||
// overwrite the previous ones, e.g. screenshots.
|
||||
@ -229,10 +226,10 @@ void dir_NextNumberedFilename(IFilesystem* fs, const char* fn_fmt, NextNumberedF
|
||||
|
||||
int max_num = -1; int num;
|
||||
DirectoryIterator di(fs, dirPath);
|
||||
FilesystemEntry fsEntry;
|
||||
while(di.NextEntry(fsEntry) == INFO::OK)
|
||||
FileInfo fileInfo;
|
||||
while(di.NextEntry(fileInfo) == INFO::OK)
|
||||
{
|
||||
if(!fsEntry.IsDirectory() && sscanf(fsEntry.name, name_fmt, &num) == 1)
|
||||
if(!fileInfo.IsDirectory() && sscanf(fileInfo.name, name_fmt, &num) == 1)
|
||||
max_num = std::max(num, max_num);
|
||||
}
|
||||
|
||||
|
@ -17,7 +17,7 @@ extern bool dir_FileExists(IFilesystem* fs, const char* pathname);
|
||||
extern bool dir_DirectoryExists(IFilesystem* fs, const char* dirPath);
|
||||
|
||||
|
||||
typedef std::vector<FilesystemEntry> FilesystemEntries;
|
||||
typedef std::vector<FileInfo> FilesystemEntries;
|
||||
|
||||
// enumerate all directory entries in <P_path>; add to container and
|
||||
// then sort it by filename.
|
||||
@ -27,7 +27,7 @@ extern LibError dir_GatherSortedEntries(IFilesystem* fs, const char* dirPath, Fi
|
||||
// called by dir_ForEachSortedEntry for each entry in the directory.
|
||||
// return INFO::CB_CONTINUE to continue calling; anything else will cause
|
||||
// dir_ForEachSortedEntry to abort and immediately return that value.
|
||||
typedef LibError (*DirCallback)(const char* pathname, const FilesystemEntry& fsEntry, const uintptr_t cbData);
|
||||
typedef LibError (*DirCallback)(const char* pathname, const FileInfo& fileInfo, const uintptr_t cbData);
|
||||
|
||||
|
||||
// call <cb> for each file and subdirectory in <dir> (alphabetical order),
|
||||
@ -49,6 +49,33 @@ typedef LibError (*DirCallback)(const char* pathname, const FilesystemEntry& fsE
|
||||
extern LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, DirCallback cb, uintptr_t cbData);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* (mostly) insulating concrete class providing iterator access to
|
||||
* directory entries.
|
||||
* this is usable for posix, VFS, etc.; instances are created via IFilesystem.
|
||||
**/
|
||||
class DirectoryIterator
|
||||
{
|
||||
public:
|
||||
DirectoryIterator(IFilesystem* fs, const char* dirPath)
|
||||
: m_impl(fs->OpenDirectory(dirPath))
|
||||
{
|
||||
}
|
||||
|
||||
// return ERR::DIR_END if all entries have already been returned once,
|
||||
// another negative error code, or INFO::OK on success, in which case <fileInfo>
|
||||
// describes the next (order is unspecified) directory entry.
|
||||
LibError NextEntry(FileInfo& fileInfo)
|
||||
{
|
||||
return m_impl.get()->NextEntry(fileInfo);
|
||||
}
|
||||
|
||||
private:
|
||||
boost::shared_ptr<IDirectoryIterator> m_impl;
|
||||
};
|
||||
|
||||
|
||||
// retrieve the next (order is unspecified) dir entry matching <filter>.
|
||||
// return 0 on success, ERR::DIR_END if no matching entry was found,
|
||||
// or a negative error code on failure.
|
||||
@ -65,8 +92,8 @@ extern LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, Dir
|
||||
// rationale: we do not sort directory entries alphabetically here.
|
||||
// most callers don't need it and the overhead is considerable
|
||||
// (we'd have to store all entries in a vector). it is left up to
|
||||
// higher-level code such as VfsUtil.
|
||||
extern LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry, const char* filter);
|
||||
// other routines.
|
||||
extern LibError dir_filtered_next_ent(DirectoryIterator& di, FileInfo& fileInfo, const char* filter);
|
||||
|
||||
|
||||
|
||||
|
@ -30,31 +30,27 @@ public:
|
||||
|
||||
// file_make_full_*_path is left untested (hard to do so)
|
||||
|
||||
void test_atom()
|
||||
void test_pool()
|
||||
{
|
||||
// file_make_unique_fn_copy
|
||||
|
||||
// .. return same address for same string?
|
||||
const char* atom1 = file_make_unique_fn_copy("a/bc/def");
|
||||
const char* atom2 = file_make_unique_fn_copy("a/bc/def");
|
||||
const char* atom1 = path_Pool->UniqueCopy("a/bc/def");
|
||||
const char* atom2 = path_Pool->UniqueCopy("a/bc/def");
|
||||
TS_ASSERT_EQUALS(atom1, atom2);
|
||||
|
||||
// .. early out (already in pool) check works?
|
||||
const char* atom3 = file_make_unique_fn_copy(atom1);
|
||||
const char* atom3 = path_Pool->UniqueCopy(atom1);
|
||||
TS_ASSERT_EQUALS(atom3, atom1);
|
||||
|
||||
|
||||
// path_is_atom_fn
|
||||
// is it reported as in pool?
|
||||
TS_ASSERT(path_is_atom_fn(atom1));
|
||||
TS_ASSERT(path_Pool()->Contains(atom1));
|
||||
|
||||
// file_get_random_name
|
||||
// path_Pool()->RandomString
|
||||
// see if the atom added above eventually comes out when a
|
||||
// random one is returned from the pool.
|
||||
int tries_left;
|
||||
for(tries_left = 1000; tries_left != 0; tries_left--)
|
||||
{
|
||||
const char* random_name = file_get_random_name();
|
||||
const char* random_name = path_Pool->RandomString();
|
||||
if(random_name == atom1)
|
||||
break;
|
||||
}
|
||||
|
@ -1,380 +0,0 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : file_cache.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : cache of file contents (supports zero-copy IO)
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "file_cache.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "path.h"
|
||||
#include "file_stats.h"
|
||||
#include "archive/trace.h"
|
||||
#include "lib/cache_adt.h" // Cache
|
||||
#include "lib/lockfree.h"
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "lib/allocators/allocators.h"
|
||||
#include "lib/allocators/headerless.h"
|
||||
#include "lib/allocators/mem_util.h" // mem_PageSize
|
||||
|
||||
|
||||
// >= sys_max_sector_size or else waio will have to realign.
|
||||
// chosen as exactly 1 page: this allows write-protecting file buffers
|
||||
// without worrying about their (non-page-aligned) borders.
|
||||
// internal fragmentation is considerable but acceptable.
|
||||
static const size_t alignment = mem_PageSize();
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// allocator
|
||||
|
||||
/*
|
||||
the biggest worry of a file cache is external fragmentation. there are two
|
||||
basic ways to combat this:
|
||||
1) 'defragment' periodically - move blocks around to increase
|
||||
size of available 'holes'.
|
||||
2) prevent fragmentation from occurring at all via
|
||||
deliberate alloc/free policy.
|
||||
|
||||
file contents are returned directly to the user (zero-copy IO), so only
|
||||
currently unreferenced blocks can be moved. it is believed that this would
|
||||
severely hamper defragmentation; we therefore go with the latter approach.
|
||||
|
||||
the basic insight is: fragmentation occurs when a block is freed whose
|
||||
neighbors are not free (thus preventing coalescing). this can be prevented by
|
||||
allocating objects of similar lifetimes together. typical workloads
|
||||
(uniform access frequency) already show such behavior: the Landlord cache
|
||||
manager evicts files in an LRU manner, which matches the allocation policy.
|
||||
|
||||
references:
|
||||
"The Memory Fragmentation Problem - Solved?" (Johnstone and Wilson)
|
||||
"Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson)
|
||||
*/
|
||||
|
||||
class Allocator
|
||||
{
|
||||
public:
|
||||
Allocator(size_t maxSize)
|
||||
: m_allocator(maxSize)
|
||||
{
|
||||
}
|
||||
|
||||
IoBuf Allocate(size_t size)
|
||||
{
|
||||
const size_t alignedSize = round_up(size, alignment);
|
||||
stats_buf_alloc(size, alignedSize);
|
||||
|
||||
void* p = m_allocator.Allocate(alignedSize);
|
||||
#ifndef NDEBUG
|
||||
m_checker.notify_alloc(p, alignedSize);
|
||||
#endif
|
||||
|
||||
return (IoBuf)p;
|
||||
}
|
||||
|
||||
void Deallocate(IoBuf buf, size_t size)
|
||||
{
|
||||
void* const p = (void*)buf;
|
||||
|
||||
// (re)allow writes. it would be nice to un-map the buffer, but this is
|
||||
// not possible because HeaderlessAllocator needs to affix boundary tags.
|
||||
(void)mprotect(p, size, PROT_READ|PROT_WRITE);
|
||||
|
||||
const size_t alignedSize = round_up(size, alignment);
|
||||
#ifndef NDEBUG
|
||||
m_checker.notify_free(p, alignedSize);
|
||||
#endif
|
||||
m_allocator.Deallocate(p, alignedSize);
|
||||
|
||||
stats_buf_free();
|
||||
}
|
||||
|
||||
private:
|
||||
HeaderlessAllocator m_allocator;
|
||||
|
||||
#ifndef NDEBUG
|
||||
static AllocatorChecker m_checker;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
typedef LF_RefCountedMemRange FileContents;
|
||||
|
||||
/**
|
||||
* manages the active FileContents referenced by users.
|
||||
* ("active" means between Reserve() and the final Release())
|
||||
**/
|
||||
class ActiveList
|
||||
{
|
||||
public:
|
||||
~ActiveList()
|
||||
{
|
||||
// display leaks
|
||||
debug_printf("file_cache leaks:\n");
|
||||
for(MapIt it = m_map.begin(); it != m_map.end(); ++it)
|
||||
{
|
||||
const char* atom_fn = it->first;
|
||||
FileContents& fc = it->second;
|
||||
debug_printf(" %s (0x%P 0x%08x)\n", atom_fn, fc.mem, fc.size);
|
||||
}
|
||||
debug_printf("--------\n");
|
||||
}
|
||||
|
||||
void Add(const char* atom_fn, FileContents& fc)
|
||||
{
|
||||
const PairIB ret = m_map.insert(std::make_pair(atom_fn, fc));
|
||||
debug_assert(ret.second); // complain if already existed
|
||||
}
|
||||
|
||||
void Remove(const char* atom_fn)
|
||||
{
|
||||
const size_t numRemoved = m_map.erase(atom_fn);
|
||||
debug_assert(numRemoved == 1);
|
||||
}
|
||||
|
||||
FileContents* Find(const char* atom_fn)
|
||||
{
|
||||
MapIt it = m_map.find(atom_fn);
|
||||
if(it == m_map.end())
|
||||
return 0; // not an error
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
// (called by FileCache::Impl::AllocateCacheSpace; we can't pass
|
||||
// atom_fn because Cache only knows about buf and size.)
|
||||
bool Contains(IoBuf buf) const
|
||||
{
|
||||
for(MapCIt it = m_map.begin(); it != m_map.end(); ++it)
|
||||
{
|
||||
const FileContents& fc = it->second;
|
||||
if(fc.mem == buf)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
typedef std::map<const char*, FileContents> Map;
|
||||
typedef Map::iterator MapIt;
|
||||
typedef Map::const_iterator MapCIt;
|
||||
typedef std::pair<MapIt, bool> PairIB;
|
||||
Map m_map;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// FileCache::Impl
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// the organization of this cache is somewhat counterintuitive. one might
|
||||
// expect a simple mapping of filename to FileContents. however, since users
|
||||
// are strongly encouraged to only load/process one file at a time, there
|
||||
// will only be a few active references. with the cache holding many more
|
||||
// entries, looking up files there is more expensive than consulting a
|
||||
// separate list of active FileContents.
|
||||
// this list (the "manager") and the cache contents are not necessarily
|
||||
// related; no inclusion relation need hold. the only requirement is that
|
||||
// each consult the other on ownership issues. if the cache decides a file
|
||||
// should be evicted while references to it are active, or users release a
|
||||
// reference to FileContents that the cache wants to keep, the memory must
|
||||
// not actually be freed. (it is then logically 'owned' by the other)
|
||||
|
||||
class FileCache::Impl
|
||||
{
|
||||
public:
|
||||
Impl(size_t size)
|
||||
: m_allocator(size)
|
||||
{
|
||||
}
|
||||
|
||||
IoBuf Reserve(const char* atom_fn, size_t size)
|
||||
{
|
||||
// (this probably indicates a bug; caching 0-length files would
|
||||
// have no benefit, anyway)
|
||||
debug_assert(size != 0);
|
||||
|
||||
IoBuf buf = AllocateCacheSpace(size);
|
||||
|
||||
FileContents fc;
|
||||
fc.refs.AcquireExclusiveAccess();
|
||||
fc.mem = (void*)buf;
|
||||
fc.size = size;
|
||||
m_activeList.Add(atom_fn, fc);
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
void MarkComplete(const char* atom_fn, uint cost)
|
||||
{
|
||||
FileContents* fc = m_activeList.Find(atom_fn);
|
||||
debug_assert(fc);
|
||||
|
||||
fc->refs.RelinquishExclusiveAccess();
|
||||
|
||||
// zero-copy cache => all users share the contents => must not
|
||||
// allow changes. this will be reverted when the buffer is freed.
|
||||
(void)mprotect(fc->mem, fc->size, PROT_READ);
|
||||
|
||||
m_cache.add(atom_fn, (IoBuf)fc->mem, fc->size, cost);
|
||||
}
|
||||
|
||||
IoBuf Retrieve(const char* atom_fn, size_t& size)
|
||||
{
|
||||
IoBuf buf;
|
||||
if(!m_cache.retrieve(atom_fn, buf, &size))
|
||||
return 0;
|
||||
|
||||
FileContents* pfc = m_activeList.Find(atom_fn);
|
||||
// was already active; add a reference.
|
||||
if(pfc)
|
||||
pfc->refs.AddReference();
|
||||
// in cache, but no active references; add to list.
|
||||
else
|
||||
{
|
||||
FileContents fc;
|
||||
fc.refs.AddReference();
|
||||
fc.mem = (void*)buf;
|
||||
fc.size = size;
|
||||
m_activeList.Add(atom_fn, fc);
|
||||
}
|
||||
|
||||
stats_buf_ref();
|
||||
return buf;
|
||||
}
|
||||
|
||||
void Release(const char* atom_fn)
|
||||
{
|
||||
FileContents* fc = m_activeList.Find(atom_fn);
|
||||
debug_assert(fc);
|
||||
|
||||
fc->refs.Release();
|
||||
if(fc->refs.ReferenceCount() == 0)
|
||||
{
|
||||
trace_notify_free(atom_fn);
|
||||
|
||||
if(!IsInCache(atom_fn))
|
||||
m_allocator.Deallocate((IoBuf)fc->mem, fc->size);
|
||||
|
||||
m_activeList.Remove(atom_fn);
|
||||
}
|
||||
}
|
||||
|
||||
LibError Invalidate(const char* atom_fn)
|
||||
{
|
||||
// remove from cache
|
||||
IoBuf cachedBuf; size_t cachedSize;
|
||||
if(m_cache.peek(atom_fn, cachedBuf, &cachedSize))
|
||||
{
|
||||
m_cache.remove(atom_fn);
|
||||
// note: we ensure cachedBuf is not active below.
|
||||
m_allocator.Deallocate(cachedBuf, cachedSize);
|
||||
}
|
||||
|
||||
// this could happen if a hotload notification comes while someone
|
||||
// is holding a reference to the file contents. atom_fn has been
|
||||
// removed from the cache, so subsequent Retrieve() calls will not
|
||||
// return old data. however, (re)loading the file would fail because
|
||||
// Reserve() ensures there's not already an extant buffer.
|
||||
// the correct way to handle this is to delay or cancel the reload,
|
||||
// so we notify our caller accordingly.
|
||||
if(IsActive(cachedBuf))
|
||||
WARN_RETURN(ERR::AGAIN); // if this actually happens, remove the warning.
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
bool IsActive(IoBuf buf) const
|
||||
{
|
||||
return m_activeList.Contains(buf);
|
||||
}
|
||||
|
||||
bool IsInCache(const char* atom_fn) const
|
||||
{
|
||||
IoBuf cachedBuf; size_t cachedSize; // unused
|
||||
return m_cache.peek(atom_fn, cachedBuf, &cachedSize);
|
||||
}
|
||||
|
||||
IoBuf AllocateCacheSpace(size_t size)
|
||||
{
|
||||
uint attempts = 0;
|
||||
for(;;)
|
||||
{
|
||||
IoBuf buf = m_allocator.Allocate(size);
|
||||
if(buf)
|
||||
return buf;
|
||||
|
||||
// remove least valuable entry from cache
|
||||
IoBuf discardedBuf; size_t discardedSize;
|
||||
bool removed = m_cache.remove_least_valuable(&discardedBuf, &discardedSize);
|
||||
// only false if cache is empty, which can't be the case because
|
||||
// allocation failed.
|
||||
debug_assert(removed);
|
||||
|
||||
// someone is holding a reference; we must not free the
|
||||
// underlying memory, nor count this iteration.
|
||||
if(IsActive(discardedBuf))
|
||||
continue;
|
||||
|
||||
m_allocator.Deallocate(discardedBuf, discardedSize);
|
||||
|
||||
// note: this may seem hefty, but 300 is known to be reached.
|
||||
// (after building an archive, the file cache will be full;
|
||||
// attempting to allocate a few MB can take a while if only
|
||||
// small scattered blocks are freed.)
|
||||
debug_assert(++attempts < 500); // otherwise: failed to make room in cache?!
|
||||
}
|
||||
}
|
||||
|
||||
ActiveList m_activeList;
|
||||
|
||||
// HACK: due to atom_fn, we are assured that strings are equal iff their
|
||||
// addresses match. however, Cache's STL (hash_)map stupidly assumes that
|
||||
// const char* keys are "strings". to avoid this behavior, we specify the
|
||||
// key as const void*.
|
||||
static Cache<const void*, IoBuf> m_cache;
|
||||
|
||||
Allocator m_allocator;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
FileCache::FileCache(size_t size)
|
||||
: impl(new Impl(size))
|
||||
{
|
||||
}
|
||||
|
||||
IoBuf FileCache::Reserve(const char* atom_fn, size_t size)
|
||||
{
|
||||
return impl.get()->Reserve(atom_fn, size);
|
||||
}
|
||||
|
||||
void FileCache::MarkComplete(const char* atom_fn, uint cost)
|
||||
{
|
||||
impl.get()->MarkComplete(atom_fn, cost);
|
||||
}
|
||||
|
||||
IoBuf FileCache::Retrieve(const char* atom_fn, size_t& size)
|
||||
{
|
||||
return impl.get()->Retrieve(atom_fn, size);
|
||||
}
|
||||
|
||||
void FileCache::Release(const char* atom_fn)
|
||||
{
|
||||
impl.get()->Release(atom_fn);
|
||||
}
|
||||
|
||||
LibError FileCache::Invalidate(const char* atom_fn)
|
||||
{
|
||||
return impl.get()->Invalidate(atom_fn);
|
||||
}
|
@ -1,111 +0,0 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : file_cache.h
|
||||
* Project : 0 A.D.
|
||||
* Description : cache of file contents (supports zero-copy IO)
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_FILE_CACHE
|
||||
#define INCLUDED_FILE_CACHE
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include "io_buf.h"
|
||||
|
||||
|
||||
/**
|
||||
* cache of file contents with support for zero-copy IO.
|
||||
* this works by reserving a region of the cache, using it as the IO buffer,
|
||||
* and returning the memory directly to users. optional write-protection
|
||||
* via MMU ensures that the shared contents aren't inadvertently changed.
|
||||
*
|
||||
* to ensure efficient operation and prevent fragmentation, only one
|
||||
* reference should be active at a time. in other words, read a file,
|
||||
* process it, and only then start reading the next file.
|
||||
*
|
||||
* rationale: this is very similar to BlockCache; however, the differences
|
||||
* (Reserve's size and MarkComplete's cost parameters and different eviction
|
||||
* policies) are enough to warrant separate implementations.
|
||||
**/
|
||||
class FileCache
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* @param size maximum amount [bytes] of memory to use for the cache.
|
||||
* (managed as a virtual memory region that's committed on-demand)
|
||||
**/
|
||||
FileCache(size_t size);
|
||||
|
||||
/**
|
||||
* Allocate an IO buffer in the cache's memory region.
|
||||
*
|
||||
* @param atom_fn pathname of the file that is to be read; this is
|
||||
* the key that will be used to Retrieve the file contents.
|
||||
* @param size required number of bytes (more may be allocated due to
|
||||
* alignment and/or internal fragmentation)
|
||||
* @return suitably aligned memory; never fails.
|
||||
*
|
||||
* no further operations with the same atom_fn are allowed to succeed
|
||||
* until MarkComplete has been called.
|
||||
**/
|
||||
IoBuf Reserve(const char* atom_fn, size_t size);
|
||||
|
||||
/**
|
||||
* Indicate that IO into the buffer has completed.
|
||||
*
|
||||
* this allows the cache to satisfy subsequent Retrieve() calls by
|
||||
* returning this buffer; if CONFIG_READ_ONLY_CACHE, the buffer is
|
||||
* made read-only. if need be and no references are currently attached
|
||||
* to it, the memory can also be commandeered by Reserve().
|
||||
*
|
||||
* @param cost is the expected cost of retrieving the file again and
|
||||
* influences how/when it is evicted from the cache.
|
||||
**/
|
||||
void MarkComplete(const char* atom_fn, uint cost = 1);
|
||||
|
||||
/**
|
||||
* Attempt to retrieve a file's contents from the file cache.
|
||||
*
|
||||
* @return 0 if not in cache or its IO is still pending, otherwise a
|
||||
* pointer to its (read-only) contents.
|
||||
*
|
||||
* if successful, the size is passed back and a reference is added to
|
||||
* the file contents.
|
||||
*
|
||||
* note: does not call stats_cache because it does not know the file size
|
||||
* in case of a cache miss; doing so is left to the caller.
|
||||
**/
|
||||
const u8* Retrieve(const char* atom_fn, size_t& size);
|
||||
|
||||
/**
|
||||
* Indicate the file contents are no longer needed.
|
||||
*
|
||||
* this decreases the reference count; the memory can only be reused
|
||||
* if it reaches 0. the contents remain in cache until they are evicted
|
||||
* by a subsequent Reserve() call.
|
||||
*
|
||||
* note: fails (raises a warning) if called for an file that is
|
||||
* currently between Reserve and MarkComplete operations.
|
||||
**/
|
||||
void Release(const char* atom_fn);
|
||||
|
||||
/**
|
||||
* Invalidate the cached contents of a file.
|
||||
*
|
||||
* this ensures subsequent reads of the files see the current (presumably
|
||||
* recently changed) contents of the file. has no effect if the file is
|
||||
* not cached at the moment.
|
||||
*
|
||||
* this would typically be called in response to a notification that a
|
||||
* file has changed.
|
||||
**/
|
||||
LibError Invalidate(const char* atom_fn);
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
boost::shared_ptr<Impl> impl;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_FILE_CACHE
|
@ -29,6 +29,8 @@ extern void stats_vfs_init_start();
|
||||
extern void stats_vfs_init_finish();
|
||||
|
||||
// file
|
||||
|
||||
// currently not called because string_pool is now in lib/allocators
|
||||
extern void stats_unique_name(size_t name_len);
|
||||
extern void stats_open(const char* atom_fn, size_t file_size);
|
||||
extern void stats_close();
|
||||
|
@ -4,16 +4,12 @@
|
||||
#include "path.h"
|
||||
|
||||
ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES);
|
||||
ERROR_ASSOCIATE(ERR::DIR_END, "End of directory reached (no more files)", -1);
|
||||
ERROR_ASSOCIATE(ERR::DIR_NOT_FOUND, "Directory not found", ENOENT);
|
||||
ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO);
|
||||
ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1);
|
||||
|
||||
|
||||
// rationale for out-of-line dtors: see [Lakos]
|
||||
|
||||
IDirectoryIterator::~IDirectoryIterator()
|
||||
{
|
||||
}
|
||||
// rationale for out-of-line dtor: see [Lakos]
|
||||
|
||||
IFilesystem::~IFilesystem()
|
||||
{
|
||||
|
@ -11,104 +11,49 @@
|
||||
#ifndef INCLUDED_FILESYSTEM
|
||||
#define INCLUDED_FILESYSTEM
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
|
||||
#include "io/io_manager.h" // IoCallback
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError FILE_ACCESS = -110000;
|
||||
const LibError DIR_NOT_FOUND = -110002;
|
||||
const LibError DIR_END = -110003;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* information describing filesystem entries (i.e. files or directories)
|
||||
* information describing file system entries (i.e. files or directories)
|
||||
*
|
||||
* note: don't be extravagant with memory - dir_ForEachSortedEntry allocates
|
||||
* one instance of this per directory entry.
|
||||
**/
|
||||
struct FilesystemEntry
|
||||
class FileInfo
|
||||
{
|
||||
off_t size;
|
||||
time_t mtime;
|
||||
public:
|
||||
FileInfo()
|
||||
{
|
||||
}
|
||||
|
||||
FileInfo(const char* name, off_t size, time_t mtime)
|
||||
: m_name(name), m_size(size), m_mtime(mtime)
|
||||
{
|
||||
}
|
||||
|
||||
const char* Name() const
|
||||
{
|
||||
return m_name;
|
||||
}
|
||||
|
||||
off_t Size() const
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
|
||||
time_t MTime() const
|
||||
{
|
||||
return m_mtime;
|
||||
}
|
||||
|
||||
private:
|
||||
/**
|
||||
* name of the entry; does not include a path.
|
||||
* the underlying storage is guaranteed to remain valid and must not
|
||||
* be freed/modified.
|
||||
**/
|
||||
const char* name;
|
||||
|
||||
// only defined for VFS files; points to their TMount.
|
||||
const void* mount;
|
||||
|
||||
bool IsDirectory() const
|
||||
{
|
||||
return (size == -1);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// (note: this is defined in the public header to promote inlining of the
|
||||
// DirectoryIterator (concrete class) member functions.)
|
||||
// documentation: see DirectoryIterator
|
||||
struct IDirectoryIterator
|
||||
{
|
||||
virtual ~IDirectoryIterator();
|
||||
virtual LibError NextEntry(FilesystemEntry& fsEntry) = 0;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* flags controlling file IO and caching behavior.
|
||||
**/
|
||||
enum FileFlags
|
||||
{
|
||||
// translate newlines: convert from/to native representation when
|
||||
// reading/writing. this is useful if files we create need to be
|
||||
// edited externally - e.g. Notepad requires \r\n.
|
||||
// caveats:
|
||||
// - FILE_NO_AIO must be set; translation is done by OS read()/write().
|
||||
// - not supported by POSIX, so this currently only has meaning on Win32.
|
||||
FILE_TEXT = 0x01,
|
||||
|
||||
// skip the aio path and use the OS-provided synchronous blocking
|
||||
// read()/write() calls. this avoids the need for buffer alignment
|
||||
// set out below, so it's useful for writing small text files.
|
||||
// note: despite its more heavyweight operation, aio is still
|
||||
// worthwhile for small files, so it is not automatically disabled.
|
||||
FILE_NO_AIO = 0x02,
|
||||
|
||||
// do not add the (entire) contents of this file to the cache.
|
||||
// this flag should be specified when the data is cached at a higher
|
||||
// level (e.g. OpenGL textures) to avoid wasting precious cache space.
|
||||
FILE_NO_CACHE = 0x04,
|
||||
|
||||
// enable caching individual blocks read from a file. the block cache
|
||||
// is small, organized as LRU and incurs some copying overhead, so it
|
||||
// should only be enabled when needed. this is the case for archives,
|
||||
// where the cache absorbs overhead of block-aligning all IOs.
|
||||
FILE_CACHE_BLOCK = 0x08,
|
||||
|
||||
// instruct file_open not to set FileCommon.atom_fn.
|
||||
// this is a slight optimization used by VFS code: file_open
|
||||
// would store the portable name, which is only used when calling
|
||||
// the OS's open(); this would unnecessarily waste atom_fn memory.
|
||||
//
|
||||
// note: other file.cpp functions require atom_fn to be set,
|
||||
// so this behavior is only triggered via flag (caller is
|
||||
// promising they will set atom_fn).
|
||||
FILE_DONT_SET_FN = 0x20,
|
||||
|
||||
// (only relevant for VFS) file will be written into the
|
||||
// appropriate subdirectory of the mount point established by
|
||||
// vfs_set_write_target. see documentation there.
|
||||
FILE_WRITE_TO_TARGET = 0x40,
|
||||
|
||||
// sum of all flags above. used when validating flag parameters.
|
||||
FILE_FLAG_ALL = 0x7F
|
||||
const char* m_name;
|
||||
off_t m_size;
|
||||
time_t m_mtime;
|
||||
};
|
||||
|
||||
|
||||
@ -116,70 +61,11 @@ struct IFilesystem
|
||||
{
|
||||
virtual ~IFilesystem();
|
||||
|
||||
/**
|
||||
* @return a single character identifying the filesystem.
|
||||
*
|
||||
* this is useful for VFS directory listings, where an indication is
|
||||
* made of where the file is actually stored.
|
||||
**/
|
||||
virtual char IdentificationCode() const = 0;
|
||||
virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const = 0;
|
||||
|
||||
/**
|
||||
* @return a number that represents the precedence of this filesystem.
|
||||
*
|
||||
* when mounting into the VFS, entries from a filesystem with higher
|
||||
* precedence override otherwise equivalent files.
|
||||
**/
|
||||
virtual int Precedence() const = 0;
|
||||
|
||||
virtual LibError GetEntry(const char* pathname, FilesystemEntry& fsEntry) const = 0;
|
||||
|
||||
virtual LibError CreateDirectory(const char* dirPath) = 0;
|
||||
virtual LibError DeleteDirectory(const char* dirPath) = 0;
|
||||
virtual IDirectoryIterator* OpenDirectory(const char* dirPath) const = 0;
|
||||
|
||||
// note: only allowing either reads or writes simplifies file cache
|
||||
// coherency (need only invalidate when closing a FILE_WRITE file).
|
||||
virtual LibError CreateFile(const char* pathname, const u8* buf, size_t size, uint flags = 0) = 0;
|
||||
virtual LibError DeleteFile(const char* pathname) = 0;
|
||||
|
||||
// read the entire file.
|
||||
// return number of bytes transferred (see above), or a negative error code.
|
||||
//
|
||||
// if non-NULL, <cb> is called for each block transferred, passing <cbData>.
|
||||
// it returns how much data was actually transferred, or a negative error
|
||||
// code (in which case we abort the transfer and return that value).
|
||||
// the callback mechanism is useful for user progress notification or
|
||||
// processing data while waiting for the next I/O to complete
|
||||
// (quasi-parallel, without the complexity of threads).
|
||||
virtual LibError LoadFile(const char* pathname, const u8*& buf, size_t size, uint flags = 0, IoCallback cb = 0, uintptr_t cbData = 0) = 0;
|
||||
};
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* (mostly) insulating concrete class providing iterator access to
|
||||
* directory entries.
|
||||
* this is usable for posix, VFS, etc.; instances are created via IFilesystem.
|
||||
**/
|
||||
class DirectoryIterator
|
||||
{
|
||||
public:
|
||||
DirectoryIterator(IFilesystem* fs, const char* dirPath)
|
||||
: m_impl(fs->OpenDirectory(dirPath))
|
||||
{
|
||||
}
|
||||
|
||||
// return ERR::DIR_END if all entries have already been returned once,
|
||||
// another negative error code, or INFO::OK on success, in which case <fsEntry>
|
||||
// describes the next (order is unspecified) directory entry.
|
||||
LibError NextEntry(FilesystemEntry& fsEntry)
|
||||
{
|
||||
return m_impl.get()->NextEntry(fsEntry);
|
||||
}
|
||||
|
||||
private:
|
||||
boost::shared_ptr<IDirectoryIterator> m_impl;
|
||||
// note: this interface avoids having to lock a directory while an
|
||||
// iterator is extant.
|
||||
virtual LibError GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const = 0;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_FILESYSTEM
|
||||
|
@ -25,9 +25,9 @@ BlockId::BlockId()
|
||||
|
||||
BlockId::BlockId(const char* atom_fn, off_t ofs)
|
||||
{
|
||||
debug_assert(ofs <= (u64)blockSize * 0xFFFFFFFF); // ensure value fits in m_blockIndex
|
||||
debug_assert(ofs <= (u64)BLOCK_SIZE * 0xFFFFFFFF); // ensure value fits in m_blockIndex
|
||||
m_atom_fn = atom_fn; // unique (by definition)
|
||||
m_blockIndex = (u32)(ofs / blockSize);
|
||||
m_blockIndex = (u32)(ofs / BLOCK_SIZE);
|
||||
}
|
||||
|
||||
bool BlockId::operator==(const BlockId& rhs) const
|
||||
@ -43,7 +43,16 @@ bool BlockId::operator!=(const BlockId& rhs) const
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
typedef LF_RefCountedMemRange Block;
|
||||
struct Block
|
||||
{
|
||||
Block()
|
||||
{
|
||||
buf = io_buf_Allocate(BLOCK_SIZE);
|
||||
}
|
||||
|
||||
IoBuf buf;
|
||||
LF_ReferenceCounter refs;
|
||||
};
|
||||
|
||||
class BlockManager
|
||||
{
|
||||
@ -51,14 +60,6 @@ public:
|
||||
BlockManager(size_t numBlocks)
|
||||
: m_ids(numBlocks), m_blocks(numBlocks), m_oldestIndex(0)
|
||||
{
|
||||
for(size_t i = 0; i < m_blocks.size(); i++)
|
||||
m_blocks[i].mem = (void*)io_buf_Allocate(blockSize);
|
||||
}
|
||||
|
||||
~BlockManager()
|
||||
{
|
||||
for(size_t i = 0; i < m_blocks.size(); i++)
|
||||
io_buf_Deallocate((IoBuf)m_blocks[i].mem, blockSize);
|
||||
}
|
||||
|
||||
// (linear search is ok since we only expect to manage a few blocks)
|
||||
@ -123,7 +124,7 @@ class BlockCache::Impl
|
||||
{
|
||||
public:
|
||||
Impl(size_t cacheSize)
|
||||
: m_blockManager(cacheSize / blockSize)
|
||||
: m_blockManager(cacheSize / BLOCK_SIZE)
|
||||
{
|
||||
}
|
||||
|
||||
@ -134,10 +135,10 @@ public:
|
||||
Block* block = m_blockManager.AcquireOldestAvailableBlock(id);
|
||||
|
||||
#if CONFIG_READ_ONLY_CACHE
|
||||
mprotect(block->mem, block->size, PROT_WRITE|PROT_READ);
|
||||
mprotect((void*)block->buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ);
|
||||
#endif
|
||||
|
||||
return (IoBuf)block->mem;
|
||||
return block->buf;
|
||||
}
|
||||
|
||||
void MarkComplete(BlockId id)
|
||||
@ -148,20 +149,21 @@ public:
|
||||
block->refs.RelinquishExclusiveAccess();
|
||||
|
||||
#if CONFIG_READ_ONLY_CACHE
|
||||
mprotect(block.mem, block.size, PROT_READ);
|
||||
mprotect((void*)block->buf.get(), BLOCK_SIZE, PROT_READ);
|
||||
#endif
|
||||
}
|
||||
|
||||
IoBuf Retrieve(BlockId id)
|
||||
bool Retrieve(BlockId id, IoBuf& buf)
|
||||
{
|
||||
Block* block = m_blockManager.Find(id);
|
||||
if(!block) // not found
|
||||
return 0;
|
||||
return false;
|
||||
|
||||
if(!block->refs.AddReference()) // contents are not yet valid
|
||||
return 0; // (this can happen due to multithreaded IOs)
|
||||
if(!block->refs.AddReference()) // contents are not yet valid (can happen due to multithreaded IOs)
|
||||
return false;
|
||||
|
||||
return (IoBuf)block->mem;
|
||||
buf = block->buf;
|
||||
return true;
|
||||
}
|
||||
|
||||
void Release(BlockId id)
|
||||
@ -201,9 +203,9 @@ void BlockCache::MarkComplete(BlockId id)
|
||||
impl.get()->Reserve(id);
|
||||
}
|
||||
|
||||
IoBuf BlockCache::Retrieve(BlockId id)
|
||||
bool BlockCache::Retrieve(BlockId id, IoBuf& buf)
|
||||
{
|
||||
return impl.get()->Retrieve(id);
|
||||
return impl.get()->Retrieve(id, buf);
|
||||
}
|
||||
|
||||
void BlockCache::Release(BlockId id)
|
||||
|
@ -11,7 +11,6 @@
|
||||
#ifndef INCLUDED_BLOCK_CACHE
|
||||
#define INCLUDED_BLOCK_CACHE
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include "io_buf.h"
|
||||
|
||||
/**
|
||||
@ -82,12 +81,13 @@ public:
|
||||
/**
|
||||
* Attempt to retrieve a block the file cache.
|
||||
*
|
||||
* @return 0 if not in cache or its IO is still pending, otherwise a
|
||||
* pointer to its (read-only) contents.
|
||||
* @return false if not in cache or its IO is still pending,
|
||||
* otherwise true.
|
||||
*
|
||||
* if successful, a reference is added to the block.
|
||||
* if successful, a reference is added to the block and its
|
||||
* buffer is returned.
|
||||
**/
|
||||
IoBuf Retrieve(BlockId id);
|
||||
bool Retrieve(BlockId id, IoBuf& buf);
|
||||
|
||||
/**
|
||||
* Indicate the block contents are no longer needed.
|
||||
@ -118,4 +118,4 @@ private:
|
||||
boost::shared_ptr<Impl> impl;
|
||||
};
|
||||
|
||||
#endif * #ifndef INCLUDED_BLOCK_CACHE
|
||||
#endif // #ifndef INCLUDED_BLOCK_CACHE
|
||||
|
@ -2,27 +2,49 @@
|
||||
#include "io_buf.h"
|
||||
|
||||
#include "lib/allocators/allocators.h" // AllocatorChecker
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "block_cache.h" // BLOCK_SIZE
|
||||
|
||||
|
||||
#ifndef NDEBUG
|
||||
static AllocatorChecker allocatorChecker;
|
||||
#endif
|
||||
|
||||
|
||||
class IoBufDeleter
|
||||
{
|
||||
public:
|
||||
IoBufDeleter(size_t paddedSize)
|
||||
: m_paddedSize(paddedSize)
|
||||
{
|
||||
debug_assert(m_paddedSize != 0);
|
||||
}
|
||||
|
||||
void operator()(const u8* mem)
|
||||
{
|
||||
debug_assert(m_paddedSize != 0);
|
||||
#ifndef NDEBUG
|
||||
allocatorChecker.notify_free((void*)mem, m_paddedSize);
|
||||
#endif
|
||||
page_aligned_free((void*)mem, m_paddedSize);
|
||||
m_paddedSize = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t m_paddedSize;
|
||||
};
|
||||
|
||||
|
||||
IoBuf io_buf_Allocate(size_t size)
|
||||
{
|
||||
void* p = page_aligned_alloc(size);
|
||||
if(!p)
|
||||
const size_t paddedSize = (size_t)round_up(size, BLOCK_SIZE);
|
||||
const u8* mem = (const u8*)page_aligned_alloc(paddedSize);
|
||||
if(!mem)
|
||||
throw std::bad_alloc();
|
||||
#ifndef NDEBUG
|
||||
allocatorChecker.notify_alloc(p, size);
|
||||
#endif
|
||||
return (IoBuf)p;
|
||||
}
|
||||
|
||||
void io_buf_Deallocate(IoBuf buf, size_t size)
|
||||
{
|
||||
void* p = (void*)buf;
|
||||
#ifndef NDEBUG
|
||||
allocatorChecker.notify_free(p, size);
|
||||
allocatorChecker.notify_alloc((void*)mem, paddedSize);
|
||||
#endif
|
||||
page_aligned_free(p, size);
|
||||
|
||||
return IoBuf(mem, IoBufDeleter(paddedSize));
|
||||
}
|
||||
|
@ -11,14 +11,12 @@
|
||||
#ifndef INCLUDED_IO_BUF
|
||||
#define INCLUDED_IO_BUF
|
||||
|
||||
typedef const u8* IoBuf;
|
||||
const IoBuf IO_BUF_TEMP = 0;
|
||||
const IoBuf IO_BUF_ALLOC = (IoBuf)1;
|
||||
typedef boost::shared_ptr<const u8> IoBuf;
|
||||
const IoBuf IO_BUF_TEMP((const u8*)0);
|
||||
|
||||
// memory will be allocated from the heap, not the (limited) file cache.
|
||||
// this makes sense for write buffers that are never used again,
|
||||
// because we avoid having to displace some other cached items.
|
||||
extern IoBuf io_buf_Allocate(size_t size);
|
||||
extern void io_buf_Deallocate(IoBuf buf, size_t size);
|
||||
|
||||
#endif // #ifndef INCLUDED_IO_BUF
|
||||
|
@ -11,8 +11,6 @@
|
||||
#include "precompiled.h"
|
||||
#include "io_manager.h"
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
|
||||
#include "../posix/io_posix.h"
|
||||
#include "../file_stats.h"
|
||||
#include "block_cache.h"
|
||||
@ -48,12 +46,17 @@
|
||||
// (apparently since NTFS files are sector-padded anyway?)
|
||||
|
||||
|
||||
LibError io_InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed)
|
||||
// helper routine used by functions that call back to a IoCallback.
|
||||
//
|
||||
// bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE }
|
||||
// note: don't abort if = 0: zip callback may not actually
|
||||
// output anything if passed very little data.
|
||||
static LibError InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed)
|
||||
{
|
||||
if(cb)
|
||||
{
|
||||
stats_cb_start();
|
||||
LibError ret = cb(cbData, block, size, &bytesProcessed);
|
||||
LibError ret = cb(cbData, block, size, bytesProcessed);
|
||||
stats_cb_finish();
|
||||
|
||||
// failed - reset byte count in case callback didn't
|
||||
@ -78,7 +81,7 @@ class BlockIo
|
||||
{
|
||||
public:
|
||||
BlockIo()
|
||||
: m_blockId(), cachedBlock(0), tempBlock(0), m_posixIo()
|
||||
: m_blockId(), m_cachedBlock(0), m_tempBlock(0), m_posixIo()
|
||||
{
|
||||
}
|
||||
|
||||
@ -87,8 +90,7 @@ public:
|
||||
m_blockId = BlockId(file.Pathname(), ofs);
|
||||
|
||||
// block already available in cache?
|
||||
cachedBlock = s_blockCache.Retrieve(m_blockId);
|
||||
if(cachedBlock)
|
||||
if(s_blockCache.Retrieve(m_blockId, m_cachedBlock))
|
||||
{
|
||||
stats_block_cache(CR_HIT);
|
||||
return INFO::OK;
|
||||
@ -99,16 +101,16 @@ public:
|
||||
|
||||
// use a temporary block if not writing to a preallocated buffer.
|
||||
if(!buf)
|
||||
buf = tempBlock = s_blockCache.Reserve(m_blockId);
|
||||
buf = m_tempBlock = s_blockCache.Reserve(m_blockId);
|
||||
|
||||
return m_posixIo.Issue(file, ofs, buf, size);
|
||||
}
|
||||
|
||||
LibError WaitUntilComplete(const u8*& block, size_t& blockSize)
|
||||
{
|
||||
if(cachedBlock)
|
||||
block = m_cachedBlock.get();
|
||||
if(block)
|
||||
{
|
||||
block = (u8*)cachedBlock;
|
||||
blockSize = BLOCK_SIZE;
|
||||
return INFO::OK;
|
||||
}
|
||||
@ -118,17 +120,17 @@ public:
|
||||
|
||||
void Discard()
|
||||
{
|
||||
if(cachedBlock)
|
||||
if(m_cachedBlock)
|
||||
{
|
||||
s_blockCache.Release(m_blockId);
|
||||
cachedBlock = 0;
|
||||
m_cachedBlock = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if(tempBlock)
|
||||
if(m_tempBlock)
|
||||
{
|
||||
s_blockCache.MarkComplete(m_blockId);
|
||||
tempBlock = 0;
|
||||
m_tempBlock = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@ -136,9 +138,9 @@ private:
|
||||
static BlockCache s_blockCache;
|
||||
|
||||
BlockId m_blockId;
|
||||
IoBuf cachedBlock;
|
||||
IoBuf m_cachedBlock;
|
||||
|
||||
IoBuf tempBlock;
|
||||
IoBuf m_tempBlock;
|
||||
|
||||
Io_Posix m_posixIo;
|
||||
};
|
||||
@ -197,7 +199,7 @@ private:
|
||||
|
||||
// we have useable data from a previous temp buffer,
|
||||
// but it needs to be copied into the user's buffer
|
||||
if(blockIo.cachedBlock && pbuf != IO_BUF_TEMP)
|
||||
if(blockIo.m_cachedBlock && pbuf != IO_BUF_TEMP)
|
||||
cpu_memcpy((char*)*pbuf+ofs_misalign+m_totalTransferred, block, blockSize);
|
||||
|
||||
m_totalTransferred += blockSize;
|
||||
@ -350,3 +352,11 @@ size_t size;
|
||||
//RingBuf<BlockIo, MAX_PENDING_IOS> queue;
|
||||
std::deque<BlockIo> queue;
|
||||
};
|
||||
|
||||
LibError io(File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb, uintptr_t cbData)
|
||||
{
|
||||
debug_printf("IO| size=%d\n", size);
|
||||
|
||||
|
||||
return ERR::NOT_IMPLEMENTED;
|
||||
}
|
@ -11,16 +11,10 @@
|
||||
#ifndef INCLUDED_IO_MANAGER
|
||||
#define INCLUDED_IO_MANAGER
|
||||
|
||||
#include "../posix/io_posix.h"
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError IO = -110100;
|
||||
const LibError IO_EOF = -110101;
|
||||
}
|
||||
class File_Posix;
|
||||
|
||||
// called by file_io after a block IO has completed.
|
||||
// *bytesProcessed must be set; file_io will return the sum of these values.
|
||||
// bytesProcessed must be set; file_io will return the sum of these values.
|
||||
// example: when reading compressed data and decompressing in the callback,
|
||||
// indicate #bytes decompressed.
|
||||
// return value: INFO::CB_CONTINUE to continue calling; anything else:
|
||||
@ -30,15 +24,12 @@ namespace ERR
|
||||
// for the entire IO. we do not split into fake blocks because it is
|
||||
// advantageous (e.g. for decompressors) to have all data at once, if available
|
||||
// anyway.
|
||||
typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t size, size_t* bytesProcessed);
|
||||
typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t size, size_t& bytesProcessed);
|
||||
|
||||
extern LibError io_Read(const File_Posix& file, off_t ofs, u8* buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
extern LibError io_Read(const File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
|
||||
// helper routine used by functions that call back to a IoCallback.
|
||||
//
|
||||
// bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE }
|
||||
// note: don't abort if = 0: zip callback may not actually
|
||||
// output anything if passed very little data.
|
||||
extern LibError io_InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed);
|
||||
|
||||
extern LibError io_Write(File_Posix& file, off_t ofs, const u8* data, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
extern LibError io_Write(File_Posix& file, off_t ofs, IoBuf data, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
|
||||
#endif // #ifndef INCLUDED_IO_MANAGER
|
||||
|
@ -2,7 +2,7 @@
|
||||
* =========================================================================
|
||||
* File : path.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : helper functions for VFS paths.
|
||||
* Description : manage paths relative to a root directory
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
@ -14,303 +14,117 @@
|
||||
#include <string.h>
|
||||
|
||||
#include "lib/posix/posix_filesystem.h"
|
||||
#include "lib/adts.h"
|
||||
#include "lib/rand.h"
|
||||
#include "lib/allocators/pool.h"
|
||||
#include "lib/sysdep/sysdep.h"
|
||||
#include "lib/path_util.h"
|
||||
#include "lib/sysdep/cpu.h" // cpu_memcpy
|
||||
#include "file_stats.h"
|
||||
#include "lib/sysdep/sysdep.h" // SYS_DIR_SEP
|
||||
#include "lib/path_util.h" // ERR::PATH_LENGTH
|
||||
|
||||
|
||||
ERROR_ASSOCIATE(ERR::ROOT_DIR_ALREADY_SET, "Attempting to set FS root dir more than once", -1);
|
||||
ERROR_ASSOCIATE(ERR::NOT_IN_ROOT_DIR, "Accessing a file that's outside of the root dir", -1);
|
||||
|
||||
|
||||
|
||||
// path types:
|
||||
// p_*: posix (e.g. mount object name or for open())
|
||||
// v_*: vfs (e.g. mount point)
|
||||
// fn : filename only (e.g. from readdir)
|
||||
// dir_name: directory only, no path (e.g. subdir name)
|
||||
//
|
||||
// all paths must be relative (no leading '/'); components are separated
|
||||
// by '/'; no ':', '\\', "." or ".." allowed; root dir is "".
|
||||
//
|
||||
// grammar:
|
||||
// path ::= dir*file?
|
||||
// dir ::= name/
|
||||
// file ::= name
|
||||
// name ::= [^/]
|
||||
|
||||
enum Conversion
|
||||
{
|
||||
TO_NATIVE,
|
||||
TO_PORTABLE
|
||||
};
|
||||
|
||||
static LibError convert_path(char* dst, const char* src, Conversion conv = TO_NATIVE)
|
||||
{
|
||||
// SYS_DIR_SEP is assumed to be a single character!
|
||||
|
||||
const char* s = src;
|
||||
char* d = dst;
|
||||
|
||||
char from = SYS_DIR_SEP, to = '/';
|
||||
if(conv == TO_NATIVE)
|
||||
from = '/', to = SYS_DIR_SEP;
|
||||
|
||||
size_t len = 0;
|
||||
|
||||
for(;;)
|
||||
{
|
||||
len++;
|
||||
if(len >= PATH_MAX)
|
||||
WARN_RETURN(ERR::PATH_LENGTH);
|
||||
|
||||
char c = *s++;
|
||||
|
||||
if(c == from)
|
||||
c = to;
|
||||
|
||||
*d++ = c;
|
||||
|
||||
// end of string - done
|
||||
if(c == '\0')
|
||||
return INFO::OK;
|
||||
}
|
||||
}
|
||||
ERROR_ASSOCIATE(ERR::PATH_ROOT_DIR_ALREADY_SET, "Attempting to set FS root dir more than once", -1);
|
||||
ERROR_ASSOCIATE(ERR::PATH_NOT_IN_ROOT_DIR, "Accessing a file that's outside of the root dir", -1);
|
||||
|
||||
|
||||
// set by path_SetRoot
|
||||
static char n_root_dir[PATH_MAX];
|
||||
static size_t n_root_dir_len;
|
||||
static char osRootPath[PATH_MAX];
|
||||
static size_t osRootPathLength;
|
||||
|
||||
|
||||
// return the native equivalent of the given relative portable path
|
||||
// (i.e. convert all '/' to the platform's directory separator)
|
||||
// makes sure length < PATH_MAX.
|
||||
LibError file_make_native_path(const char* path, char* n_path)
|
||||
{
|
||||
return convert_path(n_path, path, TO_NATIVE);
|
||||
}
|
||||
|
||||
// return the portable equivalent of the given relative native path
|
||||
// (i.e. convert the platform's directory separators to '/')
|
||||
// makes sure length < PATH_MAX.
|
||||
LibError file_make_portable_path(const char* n_path, char* path)
|
||||
{
|
||||
return convert_path(path, n_path, TO_PORTABLE);
|
||||
}
|
||||
|
||||
|
||||
// return the native equivalent of the given portable path
|
||||
// (i.e. convert all '/' to the platform's directory separator).
|
||||
// also prepends current directory => n_full_path is absolute.
|
||||
// makes sure length < PATH_MAX.
|
||||
LibError file_make_full_native_path(const char* path, char* n_full_path)
|
||||
{
|
||||
debug_assert(path != n_full_path); // doesn't work in-place
|
||||
|
||||
strcpy_s(n_full_path, PATH_MAX, n_root_dir);
|
||||
return convert_path(n_full_path+n_root_dir_len, path, TO_NATIVE);
|
||||
}
|
||||
|
||||
// return the portable equivalent of the given relative native path
|
||||
// (i.e. convert the platform's directory separators to '/')
|
||||
// n_full_path is absolute; if it doesn't match the current dir, fail.
|
||||
// (note: portable paths are always relative to the file root dir).
|
||||
// makes sure length < PATH_MAX.
|
||||
LibError file_make_full_portable_path(const char* n_full_path, char* path)
|
||||
{
|
||||
debug_assert(path != n_full_path); // doesn't work in-place
|
||||
|
||||
if(strncmp(n_full_path, n_root_dir, n_root_dir_len) != 0)
|
||||
WARN_RETURN(ERR::NOT_IN_ROOT_DIR);
|
||||
return convert_path(path, n_full_path+n_root_dir_len, TO_PORTABLE);
|
||||
}
|
||||
|
||||
|
||||
// security check: only allow attempting to chdir once, so that malicious
|
||||
// code cannot circumvent the VFS checks that disallow access to anything
|
||||
// above the current directory (set here).
|
||||
// this routine is called early at startup, so any subsequent attempts
|
||||
// security check: only allow path_SetRoot once so that malicious code
|
||||
// cannot circumvent the VFS checks that disallow access to anything above
|
||||
// the current directory (set here).
|
||||
// path_SetRoot is called early at startup, so any subsequent attempts
|
||||
// are likely bogus.
|
||||
// we provide for resetting this from the self-test to allow clean
|
||||
// re-init of the individual tests.
|
||||
static bool root_dir_established;
|
||||
static bool isRootDirEstablished;
|
||||
|
||||
|
||||
// establish the root directory from <rel_path>, which is treated as
|
||||
// relative to the executable's directory (determined via argv[0]).
|
||||
// all relative file paths passed to this module will be based from
|
||||
// this root dir.
|
||||
//
|
||||
// example: executable in "$install_dir/system"; desired root dir is
|
||||
// "$install_dir/data" => rel_path = "../data".
|
||||
//
|
||||
// argv[0] is necessary because the current directory is unknown at startup
|
||||
// (e.g. it isn't set when invoked via batch file), and this is the
|
||||
// easiest portable way to find our install directory.
|
||||
//
|
||||
// can only be called once, by design (see below). rel_path is trusted.
|
||||
LibError path_SetRoot(const char* argv0, const char* rel_path)
|
||||
{
|
||||
if(root_dir_established)
|
||||
WARN_RETURN(ERR::ROOT_DIR_ALREADY_SET);
|
||||
root_dir_established = true;
|
||||
if(isRootDirEstablished)
|
||||
WARN_RETURN(ERR::PATH_ROOT_DIR_ALREADY_SET);
|
||||
isRootDirEstablished = true;
|
||||
|
||||
// get full path to executable
|
||||
char n_path[PATH_MAX];
|
||||
char osPathname[PATH_MAX];
|
||||
// .. first try safe, but system-dependent version
|
||||
if(sys_get_executable_name(n_path, PATH_MAX) < 0)
|
||||
if(sys_get_executable_name(osPathname, PATH_MAX) < 0)
|
||||
{
|
||||
// .. failed; use argv[0]
|
||||
if(!realpath(argv0, n_path))
|
||||
errno = 0;
|
||||
if(!realpath(argv0, osPathname))
|
||||
return LibError_from_errno();
|
||||
}
|
||||
|
||||
// make sure it's valid
|
||||
if(access(n_path, X_OK) < 0)
|
||||
errno = 0;
|
||||
if(access(osPathname, X_OK) < 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
// strip executable name, append rel_path, convert to native
|
||||
char* start_of_fn = (char*)path_name_only(n_path);
|
||||
RETURN_ERR(file_make_native_path(rel_path, start_of_fn));
|
||||
// strip executable name
|
||||
char* name = (char*)path_name_only(osPathname);
|
||||
*name = '\0';
|
||||
|
||||
// get actual root dir - previous n_path may include ".."
|
||||
strcat_s(osPathname, ARRAY_SIZE(osRootPath), rel_path);
|
||||
|
||||
// get actual root dir - previous osPathname may include ".."
|
||||
// (slight optimization, speeds up path lookup)
|
||||
if(!realpath(n_path, n_root_dir))
|
||||
errno = 0;
|
||||
if(!realpath(osPathname, osRootPath))
|
||||
return LibError_from_errno();
|
||||
|
||||
// .. append SYS_DIR_SEP to simplify code that uses n_root_dir
|
||||
n_root_dir_len = strlen(n_root_dir)+1; // +1 for trailing SYS_DIR_SEP
|
||||
debug_assert((n_root_dir_len+1) < sizeof(n_root_dir)); // Just checking
|
||||
n_root_dir[n_root_dir_len-1] = SYS_DIR_SEP;
|
||||
// You might think that n_root_dir is already 0-terminated, since it's
|
||||
// static - but that might not be true after calling file_reset_root_dir!
|
||||
n_root_dir[n_root_dir_len] = 0;
|
||||
|
||||
// .. append SYS_DIR_SEP to simplify code that uses osRootPath
|
||||
osRootPathLength = strlen(osRootPath)+1; // +1 for trailing SYS_DIR_SEP
|
||||
debug_assert((osRootPathLength+1) < sizeof(osRootPath)); // Just checking
|
||||
osRootPath[osRootPathLength-1] = SYS_DIR_SEP;
|
||||
// You might think that osRootPath is already 0-terminated, since it's
|
||||
// static - but that might not be true after calling path_ResetRootDir!
|
||||
osRootPath[osRootPathLength] = 0;
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
void path_ResetRootDir()
|
||||
{
|
||||
// see comment at root_dir_established.
|
||||
debug_assert(root_dir_established);
|
||||
n_root_dir[0] = '\0';
|
||||
n_root_dir_len = 0;
|
||||
root_dir_established = false;
|
||||
// see comment at isRootDirEstablished.
|
||||
debug_assert(isRootDirEstablished);
|
||||
osRootPath[0] = '\0';
|
||||
osRootPathLength = 0;
|
||||
isRootDirEstablished = false;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// storage for path strings
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// rationale: we want a constant-time IsAtomFn(string pointer) lookup:
|
||||
// this avoids any overhead of calling path_UniqueCopy on
|
||||
// already-atomized strings. that requires allocating from one contiguous
|
||||
// arena, which is also more memory-efficient than the heap (no headers).
|
||||
static Pool atom_pool;
|
||||
|
||||
typedef DynHashTbl<const char*, const char*> AtomMap;
|
||||
static AtomMap atom_map;
|
||||
|
||||
static void InitPool()
|
||||
// (this assumes SYS_DIR_SEP is a single character)
|
||||
static void ConvertSlashCharacters(char* dst, const char* src, char from, char to)
|
||||
{
|
||||
static bool initialized = false;
|
||||
if(initialized)
|
||||
return;
|
||||
pool_create(&atom_pool, 8*MiB, POOL_VARIABLE_ALLOCS);
|
||||
initialized = true;
|
||||
|
||||
// TODO: we currently leak the pool. the following would have to run,
|
||||
// but it's not clear when that would be possible+safe.
|
||||
// atom_map.clear();
|
||||
// (void)pool_destroy(&atom_pool);
|
||||
|
||||
}
|
||||
|
||||
bool path_is_atom_fn(const char* fn)
|
||||
{
|
||||
InitPool();
|
||||
return pool_contains(&atom_pool, (void*)fn);
|
||||
}
|
||||
|
||||
// allocate a copy of P_fn in our string pool. strings are equal iff
|
||||
// their addresses are equal, thus allowing fast comparison.
|
||||
//
|
||||
// if the (generous) filename storage is full, 0 is returned.
|
||||
// this is not ever expected to happen; callers need not check the
|
||||
// return value because a warning is raised anyway.
|
||||
const char* path_UniqueCopy(const char* P_fn)
|
||||
{
|
||||
InitPool();
|
||||
|
||||
// early out: if already an atom, return immediately.
|
||||
if(path_is_atom_fn(P_fn))
|
||||
return P_fn;
|
||||
|
||||
const size_t fn_len = strlen(P_fn);
|
||||
const char* unique_fn;
|
||||
|
||||
// check if already allocated; return existing copy if so.
|
||||
//
|
||||
// rationale: the entire storage could be done via container,
|
||||
// rather than simply using it as a lookup mapping.
|
||||
// however, DynHashTbl together with Pool (see above) is more efficient.
|
||||
unique_fn = atom_map.find(P_fn);
|
||||
if(unique_fn)
|
||||
return unique_fn;
|
||||
|
||||
unique_fn = (const char*)pool_alloc(&atom_pool, fn_len+1);
|
||||
if(!unique_fn)
|
||||
for(size_t len = 0; len < PATH_MAX; len++)
|
||||
{
|
||||
DEBUG_WARN_ERR(ERR::NO_MEM);
|
||||
return 0;
|
||||
}
|
||||
cpu_memcpy((void*)unique_fn, P_fn, fn_len);
|
||||
((char*)unique_fn)[fn_len] = '\0';
|
||||
char c = *src++;
|
||||
if(c == from)
|
||||
c = to;
|
||||
*dst++ = c;
|
||||
|
||||
atom_map.insert(unique_fn, unique_fn);
|
||||
|
||||
stats_unique_name(fn_len);
|
||||
return unique_fn;
|
||||
}
|
||||
|
||||
|
||||
const char* file_get_random_name()
|
||||
{
|
||||
InitPool();
|
||||
|
||||
// there had better be names in atom_pool, else this will fail.
|
||||
debug_assert(atom_pool.da.pos != 0);
|
||||
|
||||
again:
|
||||
const size_t start_ofs = (size_t)rand(0, (uint)atom_pool.da.pos);
|
||||
|
||||
// scan back to start of string (don't scan ahead; this must
|
||||
// work even if atom_pool only contains one entry).
|
||||
const char* start = (const char*)atom_pool.da.base+start_ofs;
|
||||
for(size_t i = 0; i < start_ofs; i++)
|
||||
{
|
||||
if(*start == '\0')
|
||||
break;
|
||||
start--;
|
||||
// end of string - done
|
||||
if(c == '\0')
|
||||
return;
|
||||
}
|
||||
|
||||
// skip past the '\0' we found. loop is needed because there may be
|
||||
// several if we land in padding (due to pool alignment).
|
||||
size_t chars_left = atom_pool.da.pos - start_ofs;
|
||||
for(; *start == '\0'; start++)
|
||||
{
|
||||
// we had landed in padding at the end of the buffer.
|
||||
if(chars_left-- == 0)
|
||||
goto again;
|
||||
}
|
||||
|
||||
const char* next_name = start;
|
||||
return next_name;
|
||||
DEBUG_WARN_ERR(ERR::PATH_LENGTH);
|
||||
}
|
||||
|
||||
|
||||
void path_MakeAbsolute(const char* path, char* osPath)
|
||||
{
|
||||
debug_assert(path != osPath); // doesn't work in-place
|
||||
|
||||
strcpy_s(osPath, PATH_MAX, osRootPath);
|
||||
ConvertSlashCharacters(osPath+osRootPathLength, path, '/', SYS_DIR_SEP);
|
||||
}
|
||||
|
||||
|
||||
void path_MakeRelative(const char* osPath, char* path)
|
||||
{
|
||||
debug_assert(path != osPath); // doesn't work in-place
|
||||
|
||||
if(strncmp(osPath, osRootPath, osRootPathLength) != 0)
|
||||
DEBUG_WARN_ERR(ERR::PATH_NOT_IN_ROOT_DIR);
|
||||
ConvertSlashCharacters(path, osPath+osRootPathLength, SYS_DIR_SEP, '/');
|
||||
}
|
||||
|
@ -2,82 +2,47 @@
|
||||
* =========================================================================
|
||||
* File : path.h
|
||||
* Project : 0 A.D.
|
||||
* Description : helper functions for VFS paths.
|
||||
* Description : manage paths relative to a root directory
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
// path types:
|
||||
|
||||
// tag type type separator
|
||||
// portable relative /
|
||||
// os native absolute SYS_DIR_SEP
|
||||
// vfs vfs absolute /
|
||||
|
||||
// the vfs root directory is "". no ':', '\\', "." or ".." are allowed.
|
||||
|
||||
#ifndef INCLUDED_PATH
|
||||
#define INCLUDED_PATH
|
||||
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError ROOT_DIR_ALREADY_SET = -110200;
|
||||
const LibError NOT_IN_ROOT_DIR = -110201;
|
||||
const LibError PATH_ROOT_DIR_ALREADY_SET = -110200;
|
||||
const LibError PATH_NOT_IN_ROOT_DIR = -110201;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// establish the root directory from <rel_path>, which is treated as
|
||||
// relative to the executable's directory (determined via argv[0]).
|
||||
// all relative file paths passed to this module will be based from
|
||||
// this root dir.
|
||||
//
|
||||
// example: executable in "$install_dir/system"; desired root dir is
|
||||
// "$install_dir/data" => rel_path = "../data".
|
||||
//
|
||||
// argv[0] is necessary because the current directory is unknown at startup
|
||||
// (e.g. it isn't set when invoked via batch file), and this is the
|
||||
// easiest portable way to find our install directory.
|
||||
//
|
||||
// can only be called once, by design (see below). rel_path is trusted.
|
||||
/**
|
||||
* establish the root OS directory (portable paths are relative to it)
|
||||
*
|
||||
* @param argv0 the value of argv[0] (used to determine the location
|
||||
* of the executable in case sys_get_executable_path fails). note that
|
||||
* the current directory cannot be used because it's not set when
|
||||
* starting via batch file.
|
||||
* @param rel_path root directory relative to the executable's directory.
|
||||
* the value is considered trusted since it will typically be hard-coded.
|
||||
*
|
||||
* example: executable in "$install_dir/system"; desired root dir is
|
||||
* "$install_dir/data" => rel_path = "../data".
|
||||
*
|
||||
* can only be called once unless path_ResetRootDir is called.
|
||||
**/
|
||||
extern LibError path_SetRoot(const char* argv0, const char* rel_path);
|
||||
|
||||
|
||||
//
|
||||
// path conversion functions (native <--> portable),
|
||||
// for external libraries that require the real filename.
|
||||
//
|
||||
// replaces '/' with platform's directory separator and vice versa.
|
||||
// verifies path length < PATH_MAX (otherwise return ERR::PATH_LENGTH).
|
||||
//
|
||||
|
||||
// relative paths (relative to root dir)
|
||||
extern LibError file_make_native_path(const char* path, char* n_path);
|
||||
extern LibError file_make_portable_path(const char* n_path, char* path);
|
||||
|
||||
// as above, but with full native paths (portable paths are always relative).
|
||||
// prepends current directory, resp. makes sure it matches the given path.
|
||||
extern LibError file_make_full_native_path(const char* path, char* n_full_path);
|
||||
extern LibError file_make_full_portable_path(const char* n_full_path, char* path);
|
||||
|
||||
|
||||
|
||||
|
||||
#define VFS_PATH_IS_DIR(path) (*path == '\0' || path[strlen(path)-1] == '/')
|
||||
|
||||
|
||||
extern bool path_is_atom_fn(const char* fn);
|
||||
|
||||
|
||||
|
||||
// allocate a copy of P_fn in our string pool. strings are equal iff
|
||||
// their addresses are equal, thus allowing fast comparison.
|
||||
//
|
||||
// if the (generous) filename storage is full, 0 is returned.
|
||||
// this is not ever expected to happen; callers need not check the
|
||||
// return value because a warning is raised anyway.
|
||||
extern const char* path_UniqueCopy(const char* P_fn);
|
||||
|
||||
extern const char* file_get_random_name();
|
||||
|
||||
|
||||
extern const char* file_get_random_name();
|
||||
|
||||
|
||||
/**
|
||||
* reset root directory that was previously established via path_SetRoot.
|
||||
*
|
||||
@ -87,4 +52,19 @@ extern const char* file_get_random_name();
|
||||
**/
|
||||
extern void path_ResetRootDir();
|
||||
|
||||
|
||||
/**
|
||||
* return the absolute OS path for a given relative portable path.
|
||||
*
|
||||
* this is useful for external libraries that require the real filename.
|
||||
**/
|
||||
extern void path_MakeAbsolute(const char* path, char* osPath);
|
||||
|
||||
/**
|
||||
* return the relative portable path for a given absolute OS path.
|
||||
*
|
||||
* this is useful when receiving paths from external libraries (e.g. FAM).
|
||||
**/
|
||||
extern void path_MakeRelative(const char* osPath, char* path);
|
||||
|
||||
#endif // #ifndef INCLUDED_PATH
|
||||
|
@ -1,6 +1,6 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : fp_posix.cpp
|
||||
* File : fs_posix.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : file layer on top of POSIX. avoids the need for
|
||||
* : absolute paths and provides fast I/O.
|
||||
@ -16,217 +16,167 @@
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
|
||||
#include "../path.h"
|
||||
#include "lib/path_util.h"
|
||||
#include "lib/file/path.h"
|
||||
#include "lib/posix/posix_filesystem.h"
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// DirectoryIterator_Posix
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
DirectoryIterator_Posix::DirectoryIterator_Posix(const char* P_path)
|
||||
struct DirDeleter
|
||||
{
|
||||
char N_path[PATH_MAX];
|
||||
(void)file_make_full_native_path(P_path, N_path);
|
||||
m_osDir = opendir(N_path);
|
||||
|
||||
// note: copying to N_path and then &m_pp.path is inefficient but
|
||||
// more clear/robust. this is only called a few hundred times anyway.
|
||||
(void)path_package_set_dir(&m_pp, N_path);
|
||||
}
|
||||
|
||||
|
||||
DirectoryIterator_Posix::~DirectoryIterator_Posix()
|
||||
{
|
||||
if(m_osDir)
|
||||
void operator()(DIR* osDir) const
|
||||
{
|
||||
const int ret = closedir(m_osDir);
|
||||
const int ret = closedir(osDir);
|
||||
debug_assert(ret == 0);
|
||||
}
|
||||
};
|
||||
|
||||
// is name "." or ".."?
|
||||
static bool IsDummyDirectory(const char* name)
|
||||
{
|
||||
if(name[0] != '.')
|
||||
return false;
|
||||
return (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'));
|
||||
}
|
||||
|
||||
|
||||
LibError DirectoryIterator_Posix::NextEntry(FilesystemEntry& fsEntry)
|
||||
/*virtual*/ LibError Filesystem_Posix::GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories)
|
||||
{
|
||||
if(!m_osDir)
|
||||
return ERR::DIR_NOT_FOUND;
|
||||
|
||||
get_another_entry:
|
||||
// open directory
|
||||
char osPath[PATH_MAX];
|
||||
(void)path_MakeAbsolute(path, osPath);
|
||||
errno = 0;
|
||||
struct dirent* osEnt = readdir(m_osDir);
|
||||
if(!osEnt)
|
||||
boost::shared_ptr<DIR> osDir(opendir(osPath), DirDeleter());
|
||||
if(!osDir.get())
|
||||
return LibError_from_errno();
|
||||
|
||||
// (we need absolute paths below; using this instead of path_append avoids
|
||||
// a strlen call for each entry.)
|
||||
PathPackage pp;
|
||||
(void)path_package_set_dir(&pp, osPath);
|
||||
|
||||
for(;;)
|
||||
{
|
||||
// no error, just no more entries to return
|
||||
if(!errno)
|
||||
return ERR::DIR_END; // NOWARN
|
||||
return LibError_from_errno();
|
||||
}
|
||||
errno = 0;
|
||||
struct dirent* osEnt = readdir(osDir.get());
|
||||
if(!osEnt)
|
||||
{
|
||||
// no error, just no more entries to return
|
||||
if(!errno)
|
||||
return INFO::OK;
|
||||
return LibError_from_errno();
|
||||
}
|
||||
|
||||
// copying into the global filename storage avoids the need for users to
|
||||
// free fsEntry.name and is convenient+safe.
|
||||
const char* atom_fn = path_UniqueCopy(osEnt->d_name);
|
||||
const char* name = path_Pool()->UniqueCopy(osEnt->d_name);
|
||||
RETURN_ERR(path_component_validate(name));
|
||||
|
||||
// get file information (mode, size, mtime)
|
||||
struct stat s;
|
||||
// get file information (mode, size, mtime)
|
||||
struct stat s;
|
||||
#if OS_WIN
|
||||
// .. wposix readdir has enough information to return dirent
|
||||
// status directly (much faster than calling stat).
|
||||
RETURN_ERR(readdir_stat_np(m_osDir, &s));
|
||||
// .. wposix readdir has enough information to return dirent
|
||||
// status directly (much faster than calling stat).
|
||||
RETURN_ERR(readdir_stat_np(osDir.get(), &s));
|
||||
#else
|
||||
// .. call regular stat().
|
||||
errno = 0;
|
||||
// (we need the full pathname; don't use path_append because it would
|
||||
// unnecessarily call strlen.)
|
||||
path_package_append_file(&m_pp, atom_fn);
|
||||
if(stat(&m_pp->path, &s) != 0)
|
||||
return LibError_from_errno();
|
||||
// .. call regular stat().
|
||||
errno = 0;
|
||||
path_package_append_file(&pp, name);
|
||||
if(stat(pp->path, &s) != 0)
|
||||
return LibError_from_errno();
|
||||
#endif
|
||||
|
||||
// skip "undesirable" entries that POSIX readdir returns:
|
||||
if(S_ISDIR(s.st_mode))
|
||||
{
|
||||
// .. dummy directory entries ("." and "..")
|
||||
if(atom_fn[0] == '.' && (atom_fn[1] == '\0' || (atom_fn[1] == '.' && atom_fn[2] == '\0')))
|
||||
goto get_another_entry;
|
||||
|
||||
s.st_size = -1; // our way of indicating it's a directory
|
||||
if(files && S_ISREG(s.st_mode))
|
||||
files->push_back(FileInfo(name, s.st_size, s.st_mtime));
|
||||
else if(subdirectories && S_ISDIR(s.st_mode) && !IsDummyDirectory(name))
|
||||
subdirectories->push_back(name);
|
||||
}
|
||||
// .. neither dir nor file
|
||||
else if(!S_ISREG(s.st_mode))
|
||||
goto get_another_entry;
|
||||
|
||||
fsEntry.size = s.st_size;
|
||||
fsEntry.mtime = s.st_mtime;
|
||||
fsEntry.name = atom_fn;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Filesystem_Posix
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
LibError Filesystem_Posix::GetEntry(const char* P_pathname, FilesystemEntry& fsEntry) const
|
||||
LibError Filesystem_Posix::GetFileInfo(const char* pathname, FileInfo& fileInfo) const
|
||||
{
|
||||
char N_pathname[PATH_MAX];
|
||||
RETURN_ERR(file_make_full_native_path(P_pathname, N_pathname));
|
||||
char osPathname[PATH_MAX];
|
||||
RETURN_ERR(path_MakeAbsolute(pathname, osPathname));
|
||||
|
||||
// if path ends in slash, remove it (required by stat)
|
||||
char* last_char = N_pathname+strlen(N_pathname)-1;
|
||||
char* last_char = osPathname+strlen(osPathname)-1;
|
||||
if(path_is_dir_sep(*last_char))
|
||||
*last_char = '\0';
|
||||
|
||||
errno = 0;
|
||||
struct stat s;
|
||||
memset(&s, 0, sizeof(s));
|
||||
if(stat(N_pathname, &s) != 0)
|
||||
if(stat(osPathname, &s) != 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
fsEntry.size = s.st_size;
|
||||
fsEntry.mtime = s.st_mtime;
|
||||
fsEntry.name = path_UniqueCopy(path_name_only(N_pathname));
|
||||
fsEntry.mount = 0;
|
||||
const char* name = path_Pool()->UniqueCopy(path_name_only(osPathname));
|
||||
fileInfo = FileInfo(name, s.st_size, s.st_mtime);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError Filesystem_Posix::CreateDirectory(const char* P_dirPath)
|
||||
LibError Filesystem_Posix::DeleteFile(const char* pathname)
|
||||
{
|
||||
char N_dirPath[PATH_MAX];
|
||||
RETURN_ERR(file_make_full_native_path(P_dirPath, N_dirPath));
|
||||
char osPathname[PATH_MAX+1];
|
||||
RETURN_ERR(path_MakeAbsolute(pathname, osPathname));
|
||||
|
||||
errno = 0;
|
||||
if(unlink(osPathname) != 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError Filesystem_Posix::CreateDirectory(const char* path)
|
||||
{
|
||||
char osPath[PATH_MAX];
|
||||
RETURN_ERR(path_MakeAbsolute(path, osPath));
|
||||
|
||||
errno = 0;
|
||||
struct stat s;
|
||||
if(stat(N_dirPath, &s) != 0)
|
||||
if(stat(osPath, &s) != 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
errno = 0;
|
||||
if(mkdir(N_dirPath, S_IRWXO|S_IRWXU|S_IRWXG) != 0)
|
||||
if(mkdir(osPath, S_IRWXO|S_IRWXU|S_IRWXG) != 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError Filesystem_Posix::DeleteDirectory(const char* P_dirPath)
|
||||
LibError Filesystem_Posix::DeleteDirectory(const char* path)
|
||||
{
|
||||
// note: we have to recursively empty the directory before it can
|
||||
// be deleted (required by Windows and POSIX rmdir()).
|
||||
|
||||
char N_dirPath[PATH_MAX];
|
||||
RETURN_ERR(file_make_full_native_path(P_dirPath, N_dirPath));
|
||||
PathPackage N_pp;
|
||||
RETURN_ERR(path_package_set_dir(&N_pp, N_dirPath));
|
||||
char osPath[PATH_MAX];
|
||||
RETURN_ERR(path_MakeAbsolute(path, osPath));
|
||||
PathPackage pp;
|
||||
RETURN_ERR(path_package_set_dir(&pp, osPath));
|
||||
|
||||
std::vector<FileInfo> files;
|
||||
std::vector<const char*> subdirectories;
|
||||
RETURN_ERR(GetDirectoryEntries(path, &files, &subdirectories));
|
||||
|
||||
// delete files
|
||||
for(size_t i = 0; i < files.size(); i++)
|
||||
{
|
||||
// (must go out of scope before rmdir)
|
||||
DirectoryIterator_Posix di(P_dirPath);
|
||||
for(;;)
|
||||
{
|
||||
FilesystemEntry fsEntry;
|
||||
LibError err = di.NextEntry(fsEntry);
|
||||
if(err == ERR::DIR_END)
|
||||
break;
|
||||
RETURN_ERR(err);
|
||||
RETURN_ERR(path_package_append_file(&pp, files[i].Name()));
|
||||
errno = 0;
|
||||
if(unlink(pp.path) != 0)
|
||||
return LibError_from_errno();
|
||||
}
|
||||
|
||||
if(fsEntry.IsDirectory())
|
||||
{
|
||||
char P_subdirPath[PATH_MAX];
|
||||
RETURN_ERR(path_append(P_subdirPath, P_dirPath, fsEntry.name));
|
||||
RETURN_ERR(DeleteDirectory(P_subdirPath));
|
||||
}
|
||||
else
|
||||
{
|
||||
RETURN_ERR(path_package_append_file(&N_pp, fsEntry.name));
|
||||
|
||||
errno = 0;
|
||||
if(unlink(N_pp.path) != 0)
|
||||
return LibError_from_errno();
|
||||
}
|
||||
}
|
||||
// recurse over subdirectories
|
||||
for(size_t i = 0; i < subdirectories.size(); i++)
|
||||
{
|
||||
char subdirectoryPath[PATH_MAX];
|
||||
path_append(subdirectoryPath, path, subdirectories[i]);
|
||||
RETURN_ERR(DeleteDirectory(subdirectoryPath));
|
||||
}
|
||||
|
||||
errno = 0;
|
||||
if(rmdir(N_dirPath) != 0)
|
||||
if(rmdir(osPath) != 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
IDirectoryIterator* Filesystem_Posix::OpenDirectory(const char* P_dirPath) const
|
||||
{
|
||||
return new DirectoryIterator_Posix(P_dirPath);
|
||||
}
|
||||
|
||||
|
||||
LibError CreateFile(const char* P_pathname, const u8* buf, size_t size, uint flags = 0)
|
||||
{
|
||||
File_Posix file;
|
||||
RETURN_ERR(file.Open(P_pathname, 'w', flags));
|
||||
RETURN_ERR(io(file, 0, buf, size));
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError Filesystem_Posix::DeleteFile(const char* P_pathname)
|
||||
{
|
||||
char N_pathname[PATH_MAX+1];
|
||||
RETURN_ERR(file_make_full_native_path(P_pathname, N_pathname));
|
||||
|
||||
errno = 0;
|
||||
if(unlink(N_pathname) != 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError LoadFile(const char* P_pathname, const u8*& buf, size_t size, uint flags = 0, IoCallback cb = 0, uintptr_t cbData = 0)
|
||||
{
|
||||
File_Posix file;
|
||||
RETURN_ERR(file.Open(P_pathname, 'r', flags));
|
||||
RETURN_ERR(io(file, 0, buf, size, cb, cbData));
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : fp_posix.h
|
||||
* File : fs_posix.h
|
||||
* Project : 0 A.D.
|
||||
* Description : file layer on top of POSIX. avoids the need for
|
||||
* : absolute paths and provides fast I/O.
|
||||
@ -12,49 +12,16 @@
|
||||
#ifndef INCLUDED_FS_POSIX
|
||||
#define INCLUDED_FS_POSIX
|
||||
|
||||
#include "../filesystem.h"
|
||||
|
||||
#include "lib/path_util.h" // PathPackage
|
||||
#include "../io/io_manager.h" // IoCallback
|
||||
|
||||
// layer on top of POSIX opendir/readdir/closedir that converts paths to
|
||||
// the native representation and ignores non-file/directory entries.
|
||||
class DirectoryIterator_Posix : public IDirectoryIterator
|
||||
{
|
||||
public:
|
||||
DirectoryIterator_Posix(const char* P_path);
|
||||
virtual ~DirectoryIterator_Posix();
|
||||
virtual LibError NextEntry(FilesystemEntry& fsEntry);
|
||||
|
||||
private:
|
||||
DIR* m_osDir;
|
||||
|
||||
PathPackage m_pp;
|
||||
};
|
||||
|
||||
#include "lib/file/filesystem.h"
|
||||
|
||||
struct Filesystem_Posix : public IFilesystem
|
||||
{
|
||||
virtual char IdentificationCode() const
|
||||
{
|
||||
return 'F';
|
||||
}
|
||||
virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const;
|
||||
virtual LibError GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
|
||||
|
||||
virtual int Precedence() const
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
virtual LibError GetEntry(const char* pathname, FilesystemEntry& fsEntry) const = 0;
|
||||
|
||||
virtual LibError CreateDirectory(const char* dirPath);
|
||||
virtual LibError DeleteDirectory(const char* dirPath);
|
||||
virtual IDirectoryIterator* OpenDirectory(const char* dirPath) const;
|
||||
|
||||
virtual LibError CreateFile(const char* pathname, const u8* buf, size_t size, uint flags = 0);
|
||||
virtual LibError DeleteFile(const char* pathname);
|
||||
virtual ssize_t LoadFile(const char* pathname, IoBuf& buf, uint flags = 0, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
LibError DeleteFile(const char* pathname);
|
||||
LibError CreateDirectory(const char* dirPath);
|
||||
LibError DeleteDirectory(const char* dirPath);
|
||||
};
|
||||
|
||||
|
||||
#endif // #ifndef INCLUDED_FS_POSIX
|
||||
|
@ -30,34 +30,22 @@ File_Posix::~File_Posix()
|
||||
}
|
||||
|
||||
|
||||
LibError File_Posix::Open(const char* P_pathname, char mode, uint flags)
|
||||
LibError File_Posix::Open(const char* P_pathname, char mode)
|
||||
{
|
||||
debug_assert(mode == 'w' || mode == 'r');
|
||||
debug_assert(flags <= FILE_FLAG_ALL);
|
||||
|
||||
m_pathname = path_UniqueCopy(P_pathname);
|
||||
m_pathname = path_Pool()->UniqueCopy(P_pathname);
|
||||
m_mode = mode;
|
||||
m_flags = flags;
|
||||
|
||||
char N_pathname[PATH_MAX];
|
||||
(void)file_make_full_native_path(P_pathname, N_pathname);
|
||||
(void)path_MakeAbsolute(P_pathname, N_pathname);
|
||||
|
||||
int oflag = (mode == 'r')? O_RDONLY : O_WRONLY|O_CREAT|O_TRUNC;
|
||||
|
||||
#if OS_WIN
|
||||
if(flags & FILE_TEXT)
|
||||
oflag |= O_TEXT_NP;
|
||||
else
|
||||
oflag |= O_BINARY_NP;
|
||||
|
||||
// if AIO is disabled at user's behest, inform wposix.
|
||||
if(flags & FILE_NO_AIO)
|
||||
oflag |= O_NO_AIO_NP;
|
||||
#endif
|
||||
|
||||
m_fd = open(N_pathname, oflag, S_IRWXO|S_IRWXU|S_IRWXG);
|
||||
if(m_fd < 0)
|
||||
RETURN_ERR(ERR::FILE_ACCESS);
|
||||
WARN_RETURN(ERR::FILE_ACCESS);
|
||||
|
||||
stats_open(m_pathname, Size());
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
@ -72,15 +60,13 @@ void File_Posix::Close()
|
||||
|
||||
LibError File_Posix::Validate() const
|
||||
{
|
||||
if(path_UniqueCopy(m_pathname) != m_pathname)
|
||||
if(path_Pool()->UniqueCopy(m_pathname) != m_pathname)
|
||||
WARN_RETURN(ERR::_1);
|
||||
if((m_mode != 'w' && m_mode != 'r'))
|
||||
WARN_RETURN(ERR::_2);
|
||||
if(m_flags > FILE_FLAG_ALL)
|
||||
WARN_RETURN(ERR::_3);
|
||||
// >= 0x100 is not necessarily bogus, but suspicious.
|
||||
if(!(3 <= m_fd && m_fd < 0x100))
|
||||
WARN_RETURN(ERR::_4);
|
||||
WARN_RETURN(ERR::_3);
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
@ -11,8 +11,6 @@
|
||||
#ifndef INCLUDED_IO_POSIX
|
||||
#define INCLUDED_IO_POSIX
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
|
||||
#include "../io/io_buf.h"
|
||||
|
||||
// rationale for using aio instead of mmap:
|
||||
@ -64,11 +62,17 @@
|
||||
// idle time to satisfy potential future IOs) requires extra buffers;
|
||||
// this is a bit more complicated than just using the cache as storage.
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError FILE_ACCESS = -110200;
|
||||
const LibError IO = -110201;
|
||||
const LibError IO_EOF = -110202;
|
||||
}
|
||||
|
||||
namespace INFO
|
||||
{
|
||||
const LibError IO_PENDING = +110200;
|
||||
const LibError IO_COMPLETE = +110201;
|
||||
const LibError IO_PENDING = +110203;
|
||||
const LibError IO_COMPLETE = +110204;
|
||||
}
|
||||
|
||||
|
||||
@ -80,7 +84,7 @@ public:
|
||||
File_Posix();
|
||||
~File_Posix();
|
||||
|
||||
LibError Open(const char* pathname, char mode, uint flags);
|
||||
LibError Open(const char* pathname, char mode);
|
||||
void Close();
|
||||
|
||||
const char* Pathname() const
|
||||
@ -93,11 +97,6 @@ public:
|
||||
return m_mode;
|
||||
}
|
||||
|
||||
uint Flags() const
|
||||
{
|
||||
return m_flags;
|
||||
}
|
||||
|
||||
int Handle() const
|
||||
{
|
||||
return m_fd;
|
||||
@ -108,7 +107,6 @@ public:
|
||||
private:
|
||||
const char* m_pathname;
|
||||
char m_mode;
|
||||
uint m_flags;
|
||||
int m_fd;
|
||||
};
|
||||
|
||||
|
233
source/lib/file/vfs/file_cache.cpp
Normal file
233
source/lib/file/vfs/file_cache.cpp
Normal file
@ -0,0 +1,233 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : file_cache.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : cache of file contents (supports zero-copy IO)
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "file_cache.h"
|
||||
|
||||
#include "path.h"
|
||||
#include "file_stats.h"
|
||||
#include "archive/trace.h"
|
||||
#include "lib/cache_adt.h" // Cache
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "lib/allocators/allocators.h"
|
||||
#include "lib/allocators/headerless.h"
|
||||
#include "lib/allocators/mem_util.h" // mem_PageSize
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// allocator
|
||||
|
||||
/*
|
||||
the biggest worry of a file cache is external fragmentation. there are two
|
||||
basic ways to combat this:
|
||||
1) 'defragment' periodically - move blocks around to increase
|
||||
size of available 'holes'.
|
||||
2) prevent fragmentation from occurring at all via
|
||||
deliberate alloc/free policy.
|
||||
|
||||
file contents are returned directly to the user (zero-copy IO), so only
|
||||
currently unreferenced blocks can be moved. it is believed that this would
|
||||
severely hamper defragmentation; we therefore go with the latter approach.
|
||||
|
||||
the basic insight is: fragmentation occurs when a block is freed whose
|
||||
neighbors are not free (thus preventing coalescing). this can be prevented by
|
||||
allocating objects of similar lifetimes together. typical workloads
|
||||
(uniform access frequency) already show such behavior: the Landlord cache
|
||||
manager evicts files in an LRU manner, which matches the allocation policy.
|
||||
|
||||
references:
|
||||
"The Memory Fragmentation Problem - Solved?" (Johnstone and Wilson)
|
||||
"Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson)
|
||||
*/
|
||||
|
||||
class Allocator;
|
||||
|
||||
class Deleter
|
||||
{
|
||||
public:
|
||||
Deleter(size_t size, Allocator* allocator, const char* owner)
|
||||
: m_size(size), m_allocator(allocator), m_owner(owner)
|
||||
{
|
||||
}
|
||||
|
||||
// (this must come after Allocator because it calls Deallocate())
|
||||
void operator()(const u8* data) const;
|
||||
|
||||
private:
|
||||
size_t m_size;
|
||||
Allocator* m_allocator;
|
||||
const char* m_owner;
|
||||
};
|
||||
|
||||
// >= sys_max_sector_size or else waio will have to realign.
|
||||
// chosen as exactly 1 page: this allows write-protecting file buffers
|
||||
// without worrying about their (non-page-aligned) borders.
|
||||
// internal fragmentation is considerable but acceptable.
|
||||
static const size_t alignment = mem_PageSize();
|
||||
|
||||
class Allocator
|
||||
{
|
||||
public:
|
||||
Allocator(size_t maxSize)
|
||||
: m_allocator(maxSize)
|
||||
{
|
||||
}
|
||||
|
||||
FileCacheData Allocate(size_t size, const char* owner)
|
||||
{
|
||||
const size_t alignedSize = round_up(size, alignment);
|
||||
stats_buf_alloc(size, alignedSize);
|
||||
|
||||
const u8* data = (const u8*)m_allocator.Allocate(alignedSize);
|
||||
#ifndef NDEBUG
|
||||
m_checker.notify_alloc((void*)data, alignedSize);
|
||||
#endif
|
||||
|
||||
return FileCacheData(data, Deleter(size, this, owner));
|
||||
}
|
||||
|
||||
void Deallocate(const u8* data, size_t size, const char* owner)
|
||||
{
|
||||
void* const p = (void*)data;
|
||||
const size_t alignedSize = round_up(size, alignment);
|
||||
|
||||
// (re)allow writes. it would be nice to un-map the buffer, but this is
|
||||
// not possible because HeaderlessAllocator needs to affix boundary tags.
|
||||
(void)mprotect(p, size, PROT_READ|PROT_WRITE);
|
||||
|
||||
#ifndef NDEBUG
|
||||
m_checker.notify_free(p, alignedSize);
|
||||
#endif
|
||||
m_allocator.Deallocate(p, alignedSize);
|
||||
|
||||
stats_buf_free();
|
||||
trace_notify_free(owner);
|
||||
}
|
||||
|
||||
private:
|
||||
HeaderlessAllocator m_allocator;
|
||||
|
||||
#ifndef NDEBUG
|
||||
AllocatorChecker m_checker;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
void Deleter::operator()(const u8* data) const
|
||||
{
|
||||
m_allocator->Deallocate(data, m_size, m_owner);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// FileCache::Impl
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// since users are strongly encouraged to only load/process one file at a
|
||||
// time, there won't be many active references to cache entries. we could
|
||||
// take advantage of this with a separate extant list, but the cache's
|
||||
// hash map should be fast enough and this way is less work than maintaining
|
||||
// (possibly disjunct) cached and extant lists.
|
||||
|
||||
class FileCache::Impl
|
||||
{
|
||||
public:
|
||||
Impl(size_t size)
|
||||
: m_allocator(size)
|
||||
{
|
||||
}
|
||||
|
||||
FileCacheData Reserve(const char* vfsPathname, size_t size)
|
||||
{
|
||||
// (this probably indicates a bug; caching 0-length files would
|
||||
// have no benefit, anyway)
|
||||
debug_assert(size != 0);
|
||||
|
||||
// (300 iterations have been observed when reserving several MB
|
||||
// of space in a full cache)
|
||||
for(;;)
|
||||
{
|
||||
FileCacheData data = m_allocator.Allocate(size, vfsPathname);
|
||||
if(data.get())
|
||||
return data;
|
||||
|
||||
// remove least valuable entry from cache (if users are holding
|
||||
// references, the contents won't actually be deallocated)
|
||||
FileCacheData discardedData; size_t discardedSize;
|
||||
bool removed = m_cache.remove_least_valuable(&discardedData, &discardedSize);
|
||||
// only false if cache is empty, which can't be the case because
|
||||
// allocation failed.
|
||||
debug_assert(removed);
|
||||
}
|
||||
}
|
||||
|
||||
void Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost)
|
||||
{
|
||||
// zero-copy cache => all users share the contents => must not
|
||||
// allow changes. this will be reverted when deallocating.
|
||||
(void)mprotect((void*)data.get(), size, PROT_READ);
|
||||
|
||||
m_cache.add(vfsPathname, data, size, cost);
|
||||
}
|
||||
|
||||
bool Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size)
|
||||
{
|
||||
// (note: don't call stats_cache because we don't know the file size
|
||||
// in case of a cache miss; doing so is left to the caller.)
|
||||
stats_buf_ref();
|
||||
|
||||
return m_cache.retrieve(vfsPathname, data, &size);
|
||||
}
|
||||
|
||||
void Remove(const char* vfsPathname)
|
||||
{
|
||||
m_cache.remove(vfsPathname);
|
||||
|
||||
// note: we could check if someone is still holding a reference
|
||||
// to the contents, but that currently doesn't matter.
|
||||
}
|
||||
|
||||
private:
|
||||
// HACK: due to vfsPathname, we are assured that strings are equal iff their
|
||||
// addresses match. however, Cache's STL (hash_)map stupidly assumes that
|
||||
// const char* keys are "strings". to avoid this behavior, we specify the
|
||||
// key as const void*.
|
||||
static Cache<const void*, FileCacheData> m_cache;
|
||||
|
||||
Allocator m_allocator;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
FileCache::FileCache(size_t size)
|
||||
: impl(new Impl(size))
|
||||
{
|
||||
}
|
||||
|
||||
FileCacheData FileCache::Reserve(const char* vfsPathname, size_t size)
|
||||
{
|
||||
return impl.get()->Reserve(vfsPathname, size);
|
||||
}
|
||||
|
||||
void FileCache::Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost)
|
||||
{
|
||||
impl.get()->Add(vfsPathname, data, size, cost);
|
||||
}
|
||||
|
||||
void FileCache::Remove(const char* vfsPathname)
|
||||
{
|
||||
impl.get()->Remove(vfsPathname);
|
||||
}
|
||||
|
||||
bool FileCache::Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size)
|
||||
{
|
||||
return impl.get()->Retrieve(vfsPathname, data, size);
|
||||
}
|
91
source/lib/file/vfs/file_cache.h
Normal file
91
source/lib/file/vfs/file_cache.h
Normal file
@ -0,0 +1,91 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : file_cache.h
|
||||
* Project : 0 A.D.
|
||||
* Description : cache of file contents (supports zero-copy IO)
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_FILE_CACHE
|
||||
#define INCLUDED_FILE_CACHE
|
||||
|
||||
typedef boost::shared_ptr<const u8> FileCacheData;
|
||||
|
||||
/**
|
||||
* cache of file contents with support for zero-copy IO.
|
||||
* this works by reserving a region of the cache, using it as the IO buffer,
|
||||
* and returning the memory directly to users. optional write-protection
|
||||
* via MMU ensures that the shared contents aren't inadvertently changed.
|
||||
*
|
||||
* (unique copies of) VFS pathnames are used as lookup key and owner tag.
|
||||
*
|
||||
* to ensure efficient operation and prevent fragmentation, only one
|
||||
* reference should be active at a time. in other words, read a file,
|
||||
* process it, and only then start reading the next file.
|
||||
*
|
||||
* rationale: this is rather similar to BlockCache; however, the differences
|
||||
* (Reserve's size parameter, Add vs. MarkComplete, different eviction
|
||||
* policies) are enough to warrant separate implementations.
|
||||
**/
|
||||
class FileCache
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* @param size maximum amount [bytes] of memory to use for the cache.
|
||||
* (managed as a virtual memory region that's committed on-demand)
|
||||
**/
|
||||
FileCache(size_t size);
|
||||
|
||||
/**
|
||||
* Reserve a chunk of the cache's memory region.
|
||||
*
|
||||
* @param vfsPathname pathname of the file that is to be read; this is
|
||||
* the key that will be used to Retrieve the file contents.
|
||||
* @param size required number of bytes (more may be allocated due to
|
||||
* alignment and/or internal fragmentation)
|
||||
* @return memory suitably aligned for IO; never fails.
|
||||
*
|
||||
* it is expected that this data will be Add()-ed once its IO completes.
|
||||
**/
|
||||
FileCacheData Reserve(const char* vfsPathname, size_t size);
|
||||
|
||||
/**
|
||||
* Add a file's contents to the cache.
|
||||
*
|
||||
* the cache will be able to satisfy subsequent Retrieve() calls by
|
||||
* returning this data; if CONFIG_READ_ONLY_CACHE, the buffer is made
|
||||
* read-only. if need be and no references are currently attached to it,
|
||||
* the memory can also be commandeered by Reserve().
|
||||
*
|
||||
* @param cost is the expected cost of retrieving the file again and
|
||||
* influences how/when it is evicted from the cache.
|
||||
**/
|
||||
void Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost = 1);
|
||||
|
||||
/**
|
||||
* Remove a file's contents from the cache (if it exists).
|
||||
*
|
||||
* this ensures subsequent reads of the files see the current, presumably
|
||||
* recently changed, contents of the file.
|
||||
*
|
||||
* this would typically be called in response to a notification that a
|
||||
* file has changed.
|
||||
**/
|
||||
void Remove(const char* vfsPathname);
|
||||
|
||||
/**
|
||||
* Attempt to retrieve a file's contents from the file cache.
|
||||
*
|
||||
* @return whether the contents were successfully retrieved; if so,
|
||||
* data references the read-only file contents.
|
||||
**/
|
||||
bool Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size);
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
boost::shared_ptr<Impl> impl;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_FILE_CACHE
|
185
source/lib/file/vfs/vfs.cpp
Normal file
185
source/lib/file/vfs/vfs.cpp
Normal file
@ -0,0 +1,185 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "vfs.h"
|
||||
|
||||
#include "lib/path_util.h"
|
||||
#include "lib/file/file_stats.h"
|
||||
#include "file_cache.h"
|
||||
#include "vfs_tree.h"
|
||||
#include "vfs_path.h"
|
||||
#include "vfs_mount.h"
|
||||
|
||||
|
||||
class Filesystem_VFS::Impl : public IFilesystem
|
||||
{
|
||||
public:
|
||||
Impl()
|
||||
: m_fileCache(ChooseCacheSize())
|
||||
{
|
||||
}
|
||||
|
||||
~Impl()
|
||||
{
|
||||
trace_shutdown();
|
||||
mount_shutdown();
|
||||
}
|
||||
|
||||
LibError Mount(const char* vfsPath, const char* path, uint flags /* = 0 */, uint priority /* = 0 */)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
void Unmount(const char* path)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const
|
||||
{
|
||||
VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
|
||||
if(!file)
|
||||
return ERR::VFS_FILE_NOT_FOUND; // NOWARN
|
||||
file->GetFileInfo(fileInfo);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
virtual LibError GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
|
||||
{
|
||||
VfsDirectory* directory = LookupDirectory(vfsPath, &m_tree.Root());
|
||||
if(!directory)
|
||||
WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
|
||||
directory->GetEntries(files, subdirectories);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
// note: only allowing either reads or writes simplifies file cache
|
||||
// coherency (need only invalidate when closing a FILE_WRITE file).
|
||||
LibError CreateFile(const char* vfsPathname, const u8* buf, size_t size)
|
||||
{
|
||||
VfsDirectory* directory = LookupDirectory(vfsPathname, &m_tree.Root());
|
||||
if(!directory)
|
||||
WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
|
||||
const char* name = path_name_only(vfsPathname);
|
||||
directory->CreateFile(name, buf, size);
|
||||
|
||||
// wipe out any cached blocks. this is necessary to cover the (rare) case
|
||||
// of file cache contents predating the file write.
|
||||
m_fileCache.Remove(vfsPathname);
|
||||
}
|
||||
|
||||
// read the entire file.
|
||||
// return number of bytes transferred (see above), or a negative error code.
|
||||
//
|
||||
// if non-NULL, <cb> is called for each block transferred, passing <cbData>.
|
||||
// it returns how much data was actually transferred, or a negative error
|
||||
// code (in which case we abort the transfer and return that value).
|
||||
// the callback mechanism is useful for user progress notification or
|
||||
// processing data while waiting for the next I/O to complete
|
||||
// (quasi-parallel, without the complexity of threads).
|
||||
LibError LoadFile(const char* vfsPathname, FileContents& contents, size_t& size)
|
||||
{
|
||||
vfsPathname = path_Pool()->UniqueCopy(vfsPathname);
|
||||
debug_printf("VFS| load %s\n", vfsPathname);
|
||||
|
||||
const bool isCacheHit = m_fileCache.Retrieve(vfsPathname, contents, size);
|
||||
if(!isCacheHit)
|
||||
{
|
||||
VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
|
||||
if(!file)
|
||||
WARN_RETURN(ERR::VFS_FILE_NOT_FOUND);
|
||||
contents = m_fileCache.Reserve(vfsPathname, file->Size());
|
||||
RETURN_ERR(file->Load((u8*)contents.get()));
|
||||
m_fileCache.Add(vfsPathname, contents, size);
|
||||
}
|
||||
|
||||
stats_io_user_request(size);
|
||||
stats_cache(isCacheHit? CR_HIT : CR_MISS, size, vfsPathname);
|
||||
trace_notify_io(vfsPathname, size);
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
void RefreshFileInfo(const char* pathname)
|
||||
{
|
||||
//VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
|
||||
}
|
||||
|
||||
void Display()
|
||||
{
|
||||
m_tree.Display();
|
||||
}
|
||||
|
||||
private:
|
||||
static size_t ChooseCacheSize()
|
||||
{
|
||||
return 96*MiB;
|
||||
}
|
||||
|
||||
void Rebuild()
|
||||
{
|
||||
m_tree.Clear();
|
||||
m_mounts.RedoAll();
|
||||
}
|
||||
|
||||
VfsTree m_tree;
|
||||
FileCache m_fileCache;
|
||||
};
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
Filesystem_VFS::Filesystem_VFS(void* trace)
|
||||
{
|
||||
}
|
||||
|
||||
/*virtual*/ Filesystem_VFS::~Filesystem_VFS()
|
||||
{
|
||||
}
|
||||
|
||||
/*virtual*/ LibError Filesystem_VFS::GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const
|
||||
{
|
||||
return impl.get()->GetFileInfo(vfsPathname, fileInfo);
|
||||
}
|
||||
|
||||
/*virtual*/ LibError Filesystem_VFS::GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
|
||||
{
|
||||
return impl.get()->GetDirectoryEntries(vfsPath, files, subdirectories);
|
||||
}
|
||||
|
||||
LibError Filesystem_VFS::CreateFile(const char* vfsPathname, const u8* data, size_t size)
|
||||
{
|
||||
return impl.get()->CreateFile(vfsPathname, data, size);
|
||||
}
|
||||
|
||||
LibError Filesystem_VFS::LoadFile(const char* vfsPathname, FileContents& contents, size_t& size)
|
||||
{
|
||||
return impl.get()->LoadFile(vfsPathname, contents, size);
|
||||
}
|
||||
|
||||
LibError Filesystem_VFS::Mount(const char* vfsPath, const char* path, uint flags, uint priority)
|
||||
{
|
||||
return impl.get()->Mount(vfsPath, path, flags, priority);
|
||||
}
|
||||
|
||||
void Filesystem_VFS::Unmount(const char* path)
|
||||
{
|
||||
return impl.get()->Unmount(path);
|
||||
}
|
||||
|
||||
void Filesystem_VFS::RefreshFileInfo(const char* pathname)
|
||||
{
|
||||
impl.get()->RefreshFileInfo(pathname);
|
||||
}
|
||||
|
||||
void Filesystem_VFS::Display() const
|
||||
{
|
||||
impl.get()->Display();
|
||||
}
|
78
source/lib/file/vfs/vfs.h
Normal file
78
source/lib/file/vfs/vfs.h
Normal file
@ -0,0 +1,78 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs.h
|
||||
* Project : 0 A.D.
|
||||
* Description : Virtual File System API - allows transparent access to
|
||||
* : files in archives and modding via multiple mount points.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_VFS
|
||||
#define INCLUDED_VFS
|
||||
|
||||
#include "lib/file/filesystem.h"
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError VFS_DIR_NOT_FOUND = -110100;
|
||||
const LibError VFS_FILE_NOT_FOUND = -110101;
|
||||
}
|
||||
|
||||
typedef boost::shared_ptr<const u8> FileContents;
|
||||
|
||||
class Filesystem_VFS : public IFilesystem
|
||||
{
|
||||
public:
|
||||
Filesystem_VFS(void* trace);
|
||||
|
||||
/**
|
||||
* mount a directory into the VFS.
|
||||
*
|
||||
* @param vfsPath mount point (created if it does not already exist)
|
||||
* @param path real directory path
|
||||
*
|
||||
* if files are encountered that already exist in the VFS (sub)directories,
|
||||
* the most recent / highest priority/precedence version is preferred.
|
||||
*
|
||||
* the contents of archives in this directory (but not its subdirectories!)
|
||||
* are added as well; they are processed in alphabetical order.
|
||||
**/
|
||||
LibError Mount(const char* vfsPath, const char* path, uint flags = 0, uint priority = 0);
|
||||
|
||||
/**
|
||||
* unmount a previously mounted path and rebuild the VFS afterwards.
|
||||
**/
|
||||
void Unmount(const char* path);
|
||||
|
||||
// (see IFilesystem)
|
||||
virtual ~Filesystem_VFS();
|
||||
virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const;
|
||||
virtual LibError GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
|
||||
|
||||
// note: only allowing either reads or writes simplifies file cache
|
||||
// coherency (need only invalidate when closing a FILE_WRITE file).
|
||||
LibError CreateFile(const char* vfsPathname, const u8* data, size_t size);
|
||||
|
||||
// read the entire file.
|
||||
// return number of bytes transferred (see above), or a negative error code.
|
||||
//
|
||||
// if non-NULL, <cb> is called for each block transferred, passing <cbData>.
|
||||
// it returns how much data was actually transferred, or a negative error
|
||||
// code (in which case we abort the transfer and return that value).
|
||||
// the callback mechanism is useful for user progress notification or
|
||||
// processing data while waiting for the next I/O to complete
|
||||
// (quasi-parallel, without the complexity of threads).
|
||||
LibError LoadFile(const char* vfsPathname, FileContents& contents, size_t& size);
|
||||
|
||||
void RefreshFileInfo(const char* pathname);
|
||||
|
||||
void Display() const;
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
boost::shared_ptr<Impl> impl;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_VFS
|
293
source/lib/file/vfs/vfs_mount.cpp
Normal file
293
source/lib/file/vfs/vfs_mount.cpp
Normal file
@ -0,0 +1,293 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_mount.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : mounts files and archives into VFS
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "vfs_mount.h"
|
||||
|
||||
#include "lib/file/archive/archive_zip.h"
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// ArchiveEnumerator
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class ArchiveEnumerator
|
||||
{
|
||||
public:
|
||||
ArchiveEnumerator(VfsDirectory* directory)
|
||||
: m_startDirectory(directory), m_previousPath(0), m_previousDirectory(0)
|
||||
{
|
||||
}
|
||||
|
||||
LibError Next(const char* pathname, const ArchiveEntry& archiveEntry)
|
||||
{
|
||||
vfs_opt_notify_archived_file(pathname);
|
||||
|
||||
const char* name = path_name_only(pathname);
|
||||
const char* path = path_dir_only2(pathname);
|
||||
|
||||
// into which directory should the file be inserted?
|
||||
VfsDirectory* directory = m_previousDirectory;
|
||||
if(path != m_previousPath)
|
||||
{
|
||||
// we have to create them if missing, since we can't rely on the
|
||||
// archiver placing directories before subdirs or files that
|
||||
// reference them (WinZip doesn't always).
|
||||
|
||||
VfsFile* file;
|
||||
TraverseAndCreate(path, m_startDirectory, directory, file);
|
||||
debug_assert(file == 0); // should not be a file on the path
|
||||
|
||||
m_previousPath = path;
|
||||
m_previousDirectory = directory;
|
||||
}
|
||||
|
||||
FileInfo fileInfo;
|
||||
fileInfo.name = name;
|
||||
fileInfo.size = archiveEntry.usize;
|
||||
fileInfo.mtime = archiveEntry.mtime;
|
||||
directory->AddFile(name, fileInfo);
|
||||
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
|
||||
private:
|
||||
VfsDirectory* m_startDirectory;
|
||||
|
||||
// optimization: looking up each full path is rather slow, so we
|
||||
// cache the previous directory and use it if the path string
|
||||
// addresses match.
|
||||
const char* m_previousPath;
|
||||
VfsDirectory* m_previousDirectory;
|
||||
};
|
||||
|
||||
static LibError ArchiveEnumeratorCallback(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData)
|
||||
{
|
||||
ArchiveEnumerator* archiveEnumerator = (ArchiveEnumerator*)cbData;
|
||||
return ArchiveEnumerator->Next(pathname, archiveEntry);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Mount
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// not many instances => don't worry about efficiency.
|
||||
// note: only check for archives in the root directory of the mounting
|
||||
// because doing so for every single file would entail serious overhead
|
||||
class Mount
|
||||
{
|
||||
Mount(const char* vfsPath, const char* path, uint flags, uint priority)
|
||||
: m_vfsPath(vfsPath), m_path(path), m_flags(flags), m_priority(priority)
|
||||
{
|
||||
}
|
||||
|
||||
const char* VfsPath() const
|
||||
{
|
||||
return m_vfsPath;
|
||||
}
|
||||
|
||||
const char* Path() const
|
||||
{
|
||||
return m_path;
|
||||
}
|
||||
|
||||
// actually mount the specified entry. split out of vfs_mount,
|
||||
// because when invalidating (reloading) the VFS, we need to
|
||||
// be able to mount without changing the mount list.
|
||||
LibError Mount(VfsDirectory* rootDirectory)
|
||||
{
|
||||
VfsDirectory* directory; VfsFile* file;
|
||||
TraverseAndCreate(m_vfsPath, rootDirectory, directory, file);
|
||||
debug_assert(file == 0); // should not be a file on the path
|
||||
|
||||
directory->AssociateWithRealDirectory(m_path);
|
||||
|
||||
// add archives
|
||||
directory->Populate();
|
||||
std::vector<FileInfo> files;
|
||||
directory->GetEntries(&files, 0);
|
||||
for(size_t i = 0; i < files.size(); i++)
|
||||
{
|
||||
FileInfo& fileInfo = files[i];
|
||||
const char* pathname = path_append2(m_path, fileInfo.Name());
|
||||
const char* extension = path_extension(fileInfo.Name());
|
||||
boost::shared_ptr<IArchiveReader> archiveReader;
|
||||
if(strcasecmp(extension, "zip") == 0)
|
||||
archiveReader = CreateArchiveReader_Zip(pathname);
|
||||
else
|
||||
continue; // not a (supported) archive file
|
||||
|
||||
ArchiveEnumerator archiveEnumerator(directory);
|
||||
RETURN_ERR(archiveReader->ReadEntries(ArchiveEnumeratorCallback, (uintptr_t)&archiveEnumerator));
|
||||
m_archiveReaders.push_back(archiveReader);
|
||||
}
|
||||
}
|
||||
|
||||
// "backs off of" all archives - closes their files and allows them to
|
||||
// be rewritten or deleted (required by archive builder).
|
||||
// must call mount_rebuild when done with the rewrite/deletes,
|
||||
// because this call leaves the VFS in limbo!!
|
||||
void ReleaseArchives()
|
||||
{
|
||||
m_archiveReaders.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
bool IsArchivable() const
|
||||
{
|
||||
return (m_flags & MOUNT_ARCHIVABLE) != 0;
|
||||
}
|
||||
|
||||
// mounting into this VFS directory;
|
||||
// must end in '/' (unless if root td, i.e. "")
|
||||
const char* m_vfsPath;
|
||||
|
||||
const char* m_path;
|
||||
|
||||
std::vector<boost::shared_ptr<IArchiveReader> > m_archiveReaders;
|
||||
|
||||
uint m_flags; // MountFlags
|
||||
|
||||
uint m_priority;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// MountManager
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
|
||||
class MountManager
|
||||
{
|
||||
public:
|
||||
// mount <P_real_path> into the VFS at <V_mount_point>,
|
||||
// which is created if it does not yet exist.
|
||||
// files in that directory override the previous VFS contents if
|
||||
// <pri>(ority) is not lower.
|
||||
// all archives in <P_real_path> are also mounted, in alphabetical order.
|
||||
//
|
||||
// flags determines extra actions to perform; see VfsMountFlags.
|
||||
//
|
||||
// P_real_path = "." or "./" isn't allowed - see implementation for rationale.
|
||||
LibError Add(const char* vfsPath, const char* path, uint flags, uint priority)
|
||||
{
|
||||
// make sure caller didn't forget the required trailing '/'.
|
||||
debug_assert(VFS_PATH_IS_DIR(vfsPath));
|
||||
|
||||
// make sure it's not already mounted, i.e. in mounts.
|
||||
// also prevents mounting a parent directory of a previously mounted
|
||||
// directory, or vice versa. example: mount $install/data and then
|
||||
// $install/data/mods/official - mods/official would also be accessible
|
||||
// from the first mount point - bad.
|
||||
char existingVfsPath[PATH_MAX];
|
||||
if(GetVfsPath(path, existingVfsPath))
|
||||
WARN_RETURN(ERR::ALREADY_MOUNTED);
|
||||
|
||||
// disallow "." because "./" isn't supported on Windows.
|
||||
// "./" and "/." are caught by CHECK_PATH.
|
||||
if(!strcmp(path, "."))
|
||||
WARN_RETURN(ERR::PATH_NON_CANONICAL);
|
||||
|
||||
// (count this as "init" to obviate a separate timer)
|
||||
stats_vfs_init_start();
|
||||
mounts[path] = Mount(vfsPath, path, flags, priority);
|
||||
LibError ret = remount(m);
|
||||
stats_vfs_init_finish();
|
||||
return ret;
|
||||
}
|
||||
|
||||
void Remove(const char* path)
|
||||
{
|
||||
const size_t numRemoved = mounts.erase(path);
|
||||
debug_assert(numRemoved == 1);
|
||||
}
|
||||
|
||||
// rebuild the VFS, i.e. re-mount everything. open files are not affected.
|
||||
// necessary after loose files or directories change, so that the VFS
|
||||
// "notices" the changes and updates file locations. res calls this after
|
||||
// dir_watch reports changes; can also be called from the console after a
|
||||
// rebuild command. there is no provision for updating single VFS dirs -
|
||||
// it's not worth the trouble.
|
||||
void RedoAll()
|
||||
{
|
||||
for(MountIt it = mounts.begin(); it != mounts.end(); ++it)
|
||||
{
|
||||
const Mount& mount = it->second;
|
||||
mount.Apply();
|
||||
}
|
||||
}
|
||||
|
||||
// if <path> or its ancestors are mounted,
|
||||
// return a VFS path that accesses it.
|
||||
// used when receiving paths from external code.
|
||||
bool GetVfsPath(const char* path, char* vfsPath) const
|
||||
{
|
||||
for(MountIt it = mounts.begin(); it != mounts.end(); ++it)
|
||||
{
|
||||
const Mount& mount = it->second;
|
||||
|
||||
const char* remove = mount.Path();
|
||||
const char* replace = mount.VfsPath();
|
||||
if(path_replace(vfsPath, path, remove, replace) == INFO::OK)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void ReleaseArchives()
|
||||
{
|
||||
// "backs off of" all archives - closes their files and allows them to
|
||||
// be rewritten or deleted (required by archive builder).
|
||||
// must call mount_rebuild when done with the rewrite/deletes,
|
||||
// because this call leaves the VFS in limbo!!
|
||||
|
||||
}
|
||||
|
||||
private:
|
||||
typedef std::map<const char*, Mount> Mounts;
|
||||
typedef Mounts::iterator MountIt;
|
||||
Mounts mounts;
|
||||
};
|
||||
|
||||
|
||||
|
||||
// 2006-05-09 JW note: we are wanting to move XMB files into a separate
|
||||
// folder tree (no longer interspersed with XML), so that deleting them is
|
||||
// easier and dirs are less cluttered.
|
||||
//
|
||||
// if several mods are active, VFS would have several OS directories mounted
|
||||
// into a VfsDirectory and could no longer automatically determine the write target.
|
||||
//
|
||||
// one solution would be to use a set_write_target API to choose the
|
||||
// correct dir; however, XMB files may be generated whilst editing
|
||||
// (which also requires a write_target to write files that are actually
|
||||
// currently in archives), so we'd need to save/restore write_target.
|
||||
// this would't be thread-safe => disaster.
|
||||
//
|
||||
// a vfs_store_to(filename, flags, N_actual_path) API would work, but it'd
|
||||
// impose significant burden on users (finding the actual native dir),
|
||||
// and be prone to abuse. additionally, it would be difficult to
|
||||
// propagate N_actual_path to VFile_reload where it is needed;
|
||||
// this would end up messy.
|
||||
//
|
||||
// instead, we'll write XMB files into VFS path "mods/$MODNAME/..",
|
||||
// into which the realdir of the same name (located in some writable folder)
|
||||
// is mounted; VFS therefore can write without problems.
|
||||
//
|
||||
// however, other code (e.g. archive builder) doesn't know about this
|
||||
// trick - it only sees the flat VFS namespace, which doesn't
|
||||
// include mods/$MODNAME (that is hidden). to solve this, we also mount
|
||||
// any active mod's XMB dir into VFS root for read access.
|
||||
//
|
||||
// write_targets are no longer necessary since files and directories are
|
||||
// directly associated with a real directory; when writing files, we can
|
||||
// do so directly.
|
37
source/lib/file/vfs/vfs_mount.h
Normal file
37
source/lib/file/vfs/vfs_mount.h
Normal file
@ -0,0 +1,37 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_mount.h
|
||||
* Project : 0 A.D.
|
||||
* Description : mounts files and archives into VFS; provides x_* API
|
||||
* : that dispatches to file or archive implementation.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_VFS_MOUNT
|
||||
#define INCLUDED_VFS_MOUNT
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError ALREADY_MOUNTED = -110700;
|
||||
const LibError NOT_MOUNTED = -110701;
|
||||
const LibError MOUNT_INVALID_TYPE = -110702;
|
||||
}
|
||||
|
||||
// (recursive mounting and mounting archives are no longer optional since they don't hurt)
|
||||
enum MountFlags
|
||||
{
|
||||
// all real directories mounted during this operation will be watched
|
||||
// for changes. this flag is provided to avoid watches in output-only
|
||||
// directories, e.g. screenshots/ (only causes unnecessary overhead).
|
||||
MOUNT_WATCH = 4,
|
||||
|
||||
// anything mounted from here should be added to archive when
|
||||
// building via vfs_optimizer.
|
||||
MOUNT_ARCHIVABLE = 8
|
||||
};
|
||||
|
||||
|
||||
|
||||
#endif // #ifndef INCLUDED_VFS_MOUNT
|
122
source/lib/file/vfs/vfs_path.cpp
Normal file
122
source/lib/file/vfs/vfs_path.cpp
Normal file
@ -0,0 +1,122 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_path.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "vfs_path.h"
|
||||
|
||||
#include "lib/path_util.h" // path_foreach_component
|
||||
#include "vfs_tree.h"
|
||||
|
||||
|
||||
static bool IsVfsPath(const char* path)
|
||||
{
|
||||
if(path[0] == '\0') // root dir
|
||||
return true;
|
||||
if(path[strlen(path)-1] == '/') // ends in slash => directory path
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
class PathResolver
|
||||
{
|
||||
public:
|
||||
enum Flags
|
||||
{
|
||||
// when encountering subdirectory components in the path(name) that
|
||||
// don't (yet) exist, add them.
|
||||
CreateMissingDirectories = 1,
|
||||
};
|
||||
|
||||
PathResolver(VfsDirectory* startDirectory, uint flags = 0)
|
||||
: m_currentDirectory(startDirectory), m_file(0)
|
||||
, m_createMissingDirectories((flags & CreateMissingDirectories) != 0)
|
||||
{
|
||||
}
|
||||
|
||||
LibError Next(const char* component, bool isDir) const
|
||||
{
|
||||
m_currentDirectory->Populate();
|
||||
|
||||
if(isDir)
|
||||
{
|
||||
if(m_createMissingDirectories)
|
||||
m_currentDirectory->AddSubdirectory(component);
|
||||
|
||||
m_currentDirectory = m_currentDirectory->GetSubdirectory(component);
|
||||
if(!m_currentDirectory)
|
||||
WARN_RETURN(ERR::FAIL);
|
||||
}
|
||||
else
|
||||
{
|
||||
debug_assert(m_file == 0); // can't have encountered any files yet
|
||||
m_file = m_currentDirectory->GetFile(component);
|
||||
if(!m_file)
|
||||
WARN_RETURN(ERR::FAIL);
|
||||
}
|
||||
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
|
||||
VfsDirectory* Directory() const
|
||||
{
|
||||
return m_currentDirectory;
|
||||
}
|
||||
|
||||
VfsFile* File() const
|
||||
{
|
||||
return m_file;
|
||||
}
|
||||
|
||||
private:
|
||||
mutable VfsDirectory* m_currentDirectory;
|
||||
mutable VfsFile* m_file;
|
||||
bool m_createMissingDirectories;
|
||||
};
|
||||
|
||||
|
||||
static LibError PathResolverCallback(const char* component, bool isDir, uintptr_t cbData)
|
||||
{
|
||||
PathResolver* pathResolver = (PathResolver*)cbData;
|
||||
return pathResolver->Next(component, isDir);
|
||||
}
|
||||
|
||||
|
||||
VfsFile* LookupFile(const char* vfsPathname, VfsDirectory* startDirectory)
|
||||
{
|
||||
debug_assert(!IsVfsPath(vfsPathname));
|
||||
|
||||
PathResolver pathResolver(startDirectory);
|
||||
if(path_foreach_component(vfsPathname, PathResolverCallback, (uintptr_t)&pathResolver) != INFO::OK)
|
||||
return 0;
|
||||
return pathResolver.File();
|
||||
}
|
||||
|
||||
|
||||
VfsDirectory* LookupDirectory(const char* vfsPath, VfsDirectory* startDirectory)
|
||||
{
|
||||
debug_assert(IsVfsPath(vfsPath));
|
||||
|
||||
PathResolver pathResolver(startDirectory);
|
||||
if(path_foreach_component(vfsPath, PathResolverCallback, (uintptr_t)&pathResolver) != INFO::OK)
|
||||
return 0;
|
||||
return pathResolver.Directory();
|
||||
}
|
||||
|
||||
|
||||
void TraverseAndCreate(const char* vfsPath, VfsDirectory* startDirectory, VfsDirectory*& lastDirectory, VfsFile*& file)
|
||||
{
|
||||
PathResolver pathResolver(startDirectory, PathResolver::CreateMissingDirectories);
|
||||
LibError ret = path_foreach_component(vfsPath, PathResolverCallback, (uintptr_t)&pathResolver);
|
||||
debug_assert(ret == INFO::OK); // should never fail
|
||||
lastDirectory = pathResolver.Directory();
|
||||
file = pathResolver.File();
|
||||
}
|
58
source/lib/file/vfs/vfs_path.h
Normal file
58
source/lib/file/vfs/vfs_path.h
Normal file
@ -0,0 +1,58 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_path.h
|
||||
* Project : 0 A.D.
|
||||
* Description :
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_VFS_PATH
|
||||
#define INCLUDED_VFS_PATH
|
||||
|
||||
// VFS paths are of the form: "(dir/)*file?"
|
||||
// in English: '/' as path separator; trailing '/' required for dir names;
|
||||
// no leading '/', since "" is the root dir.
|
||||
|
||||
// there is no restriction on path length; when dimensioning character
|
||||
// arrays, prefer PATH_MAX.
|
||||
|
||||
// pathnames are case-insensitive.
|
||||
// implementation:
|
||||
// when mounting, we get the exact filenames as reported by the OS;
|
||||
// we allow open requests with mixed case to match those,
|
||||
// but still use the correct case when passing to other libraries
|
||||
// (e.g. the actual open() syscall).
|
||||
// rationale:
|
||||
// necessary, because some exporters output .EXT uppercase extensions
|
||||
// and it's unreasonable to expect that users will always get it right.
|
||||
|
||||
|
||||
class VfsFile;
|
||||
class VfsDirectory;
|
||||
|
||||
/**
|
||||
* Find a file in the VFS by traversing pathname components.
|
||||
*
|
||||
* @param vfsPath must not end with a slash character.
|
||||
**/
|
||||
extern VfsFile* LookupFile(const char* vfsPathname, const VfsDirectory* startDirectory);
|
||||
|
||||
/**
|
||||
* Find a directory in the VFS by traversing path components.
|
||||
*
|
||||
* @param vfsPath must end with a slash character.
|
||||
**/
|
||||
extern VfsDirectory* LookupDirectory(const char* vfsPath, const VfsDirectory* startDirectory);
|
||||
|
||||
/**
|
||||
* Traverse a path, creating subdirectories that didn't yet exist.
|
||||
*
|
||||
* @param lastDirectory receives the last directory that was encountered
|
||||
* @param file references the VFS file represented by vfsPath, or 0 if
|
||||
* it doesn't include a filename.
|
||||
**/
|
||||
extern void TraverseAndCreate(const char* vfsPath, VfsDirectory* startDirectory, VfsDirectory*& lastDirectory, VfsFile*& file);
|
||||
|
||||
#endif // #ifndef INCLUDED_VFS_PATH
|
274
source/lib/file/vfs/vfs_tree.cpp
Normal file
274
source/lib/file/vfs/vfs_tree.cpp
Normal file
@ -0,0 +1,274 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_tree.cpp
|
||||
* Project : 0 A.D.
|
||||
* Description : the actual 'filesystem' and its tree of directories.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "vfs_tree.h"
|
||||
|
||||
#include "lib/file/posix/fs_posix.h"
|
||||
#include "lib/file/archive/archive.h"
|
||||
|
||||
static Filesystem_Posix fsPosix;
|
||||
|
||||
static const char* svnName = path_Pool()->UniqueCopy(".svn");
|
||||
|
||||
|
||||
LibError VfsFile::Load(u8* buf) const
|
||||
{
|
||||
if(m_archiveEntry)
|
||||
RETURN_ERR(m_archiveEntry->archiveReader->LoadFile(*m_archiveEntry, buf));
|
||||
else
|
||||
{
|
||||
const char* pathname = path_append2(m_path, Name());
|
||||
File_Posix file;
|
||||
RETURN_ERR(file.Open(pathname, 'r'));
|
||||
RETURN_ERR(io_Read(file, 0, buf, Size()));
|
||||
}
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
VfsDirectory::VfsDirectory(const char* vfsPath, const char* path)
|
||||
: m_vfsPath(vfsPath), m_path(0)
|
||||
{
|
||||
if(path)
|
||||
AssociateWithRealDirectory(path);
|
||||
}
|
||||
|
||||
|
||||
VfsDirectory::~VfsDirectory()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
static bool ShouldReplace(VfsFile& vf1, VfsFile& vf2)
|
||||
{
|
||||
// 1) keep old if new priority is lower.
|
||||
if(vf2.Priority() < vf1.Priority())
|
||||
return false;
|
||||
|
||||
// assume they're the same if size and last-modified time match.
|
||||
// note: FAT timestamp only has 2 second resolution
|
||||
const double diff = difftime(vf1.MTime(), vf2.MTime());
|
||||
const bool identical = (vf1.Size() == vf2.Size()) && fabs(diff) <= 2.0;
|
||||
|
||||
// 3) go with more efficient source (if files are identical)
|
||||
//
|
||||
// since priority is not less, we really ought to always go with m_new.
|
||||
// however, there is one special case we handle for performance reasons:
|
||||
// if the file contents are the same, prefer the more efficient source.
|
||||
// note that priority doesn't automatically take care of this,
|
||||
// especially if set incorrectly.
|
||||
if(identical && vf1.Precedence() > vf2.Precedence())
|
||||
return false;
|
||||
|
||||
// 4) don't replace "old" file if modified more recently than "new".
|
||||
// (still provide for 2 sec. FAT tolerance - see above)
|
||||
if(diff > 2.0)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void VfsDirectory::AddFile(const FileInfo& fileInfo)
|
||||
{
|
||||
std::pair<const char*, VfsFile> value = std::make_pair(fileInfo.Name(), VfsFile(fileInfo));
|
||||
std::pair<Files::iterator, bool> ret = m_files.insert(value);
|
||||
if(!ret.second) // already existed
|
||||
{
|
||||
VfsFile& previousFile = ret.first.second;
|
||||
if(!ShouldReplace(previousFile, value.second))
|
||||
return INFO::ALREADY_EXISTS;
|
||||
replace_in_map
|
||||
}
|
||||
|
||||
stats_vfs_file_add(size);
|
||||
}
|
||||
|
||||
|
||||
void VfsDirectory::AddSubdirectory(const char* name)
|
||||
{
|
||||
const char* vfsPath = path_append2(m_vfsPath, name);
|
||||
std::pair<const char*, VfsDirectory> value = std::make_pair(name, VfsDirectory(vfsPath));
|
||||
(void)m_subdirectories.insert(value);
|
||||
}
|
||||
|
||||
|
||||
VfsFile* VfsDirectory::GetFile(const char* name) const
|
||||
{
|
||||
Files::const_iterator it = m_files.find(name);
|
||||
if(it == m_files.end())
|
||||
return 0;
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
|
||||
VfsDirectory* VfsDirectory::GetSubdirectory(const char* name) const
|
||||
{
|
||||
Subdirectories::const_iterator it = m_subdirectories.find(name);
|
||||
if(it == m_subdirectories.end())
|
||||
return 0;
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
|
||||
void VfsDirectory::GetEntries(std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
|
||||
{
|
||||
if(files)
|
||||
{
|
||||
files->reserve(m_files.size());
|
||||
for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it)
|
||||
files->push_back(it->second.m_fileInfo);
|
||||
}
|
||||
|
||||
if(subdirectories)
|
||||
{
|
||||
subdirectories->reserve(m_subdirectories.size());
|
||||
for(Subdirectories::const_iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
|
||||
subdirectories->push_back(it->second.m_name);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void VfsDirectory::DisplayR(unsigned depth) const
|
||||
{
|
||||
const char indent[] = " ";
|
||||
|
||||
// build format string (set width of name field so that everything
|
||||
// lines up correctly)
|
||||
char fmt[25];
|
||||
const int maxNameChars = 80 - depth*(sizeof(indent)-1);
|
||||
sprintf(fmt, "%%-%d.%ds (%%c; %%6d; %%s)\n", maxNameChars, maxNameChars);
|
||||
|
||||
for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it)
|
||||
{
|
||||
VfsFile& file = it->second;
|
||||
|
||||
char timestamp[25];
|
||||
const time_t mtime = file.MTime();
|
||||
strftime(timestamp, ARRAY_SIZE(timestamp), "%a %b %d %H:%M:%S %Y", localtime(&mtime));
|
||||
|
||||
for(int i = 0; i < depth; i++)
|
||||
printf(indent);
|
||||
printf(fmt, file.Name(), file.LocationCode(), file.Size(), timestamp);
|
||||
}
|
||||
|
||||
for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
|
||||
{
|
||||
const char* name = it->first;
|
||||
const VfsDirectory& directory = it->second;
|
||||
|
||||
for(int i = 0; i < depth+1; i++)
|
||||
printf(indent);
|
||||
printf("[%s/]\n", name);
|
||||
directory.DisplayR(depth+1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void VfsDirectory::ClearR()
|
||||
{
|
||||
for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
|
||||
it->second.ClearR();
|
||||
|
||||
m_files.Clear();
|
||||
m_subdirectories.Clear();
|
||||
m_watch.Unregister();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static const char* AMBIGUOUS = (const char*)(intptr_t)-1;
|
||||
|
||||
|
||||
void VfsDirectory::AssociateWithRealDirectory(const char* path)
|
||||
{
|
||||
// mounting the same directory twice is probably a bug
|
||||
debug_assert(m_path != path);
|
||||
|
||||
if(!m_path)
|
||||
{
|
||||
m_path = path;
|
||||
m_watch.Register(path);
|
||||
}
|
||||
else
|
||||
m_path = AMBIGUOUS;
|
||||
}
|
||||
|
||||
|
||||
void VfsDirectory::Populate()
|
||||
{
|
||||
std::vector<FileInfo> files; files.reserve(100);
|
||||
std::vector<const char*> subdirectories; subdirectories.reserve(20);
|
||||
fsPosix.GetDirectoryEntries(m_path, &files, &subdirectories);
|
||||
|
||||
for(size_t i = 0; i < files.size(); i++)
|
||||
{
|
||||
AddFile(files[i]);
|
||||
|
||||
// note: check if archivable to exclude stuff like screenshots
|
||||
// from counting towards the threshold.
|
||||
if(m->IsArchivable())
|
||||
{
|
||||
// notify archive builder that this file could be archived but
|
||||
// currently isn't; if there are too many of these, archive will
|
||||
// be rebuilt.
|
||||
const char* pathname = path_append2(m_path, files[i].Name());
|
||||
vfs_opt_notify_loose_file(pathname);
|
||||
}
|
||||
}
|
||||
|
||||
for(size_t i = 0; i < subdirectories.size(); i++)
|
||||
{
|
||||
// skip version control directories - this avoids cluttering the
|
||||
// VFS with hundreds of irrelevant files.
|
||||
if(subdirectories[i] == svnName)
|
||||
continue;
|
||||
|
||||
const char* path = path_append2(m_path, subdirectories[i]);
|
||||
AddSubdirectory(subdirectories[i], path);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
LibError VfsDirectory::CreateFile(const char* name, const u8* buf, size_t size, uint flags = 0)
|
||||
{
|
||||
debug_assert(m_path != 0 && m_path != AMBIGUOUS);
|
||||
|
||||
const char* pathname = path_append2(pathname, m_path, name);
|
||||
File_Posix file;
|
||||
RETURN_ERR(file.Open(pathname, 'w'));
|
||||
RETURN_ERR(io_Write(file, 0, buf, size));
|
||||
|
||||
AddFile(FileInfo(name, size, time()));
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
|
||||
VfsTree::VfsTree()
|
||||
: m_rootDirectory("")
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void VfsTree::Display() const
|
||||
{
|
||||
m_rootDirectory.DisplayR();
|
||||
}
|
||||
|
||||
|
||||
void VfsTree::Clear()
|
||||
{
|
||||
m_rootDirectory.ClearR();
|
||||
}
|
198
source/lib/file/vfs/vfs_tree.h
Normal file
198
source/lib/file/vfs/vfs_tree.h
Normal file
@ -0,0 +1,198 @@
|
||||
/**
|
||||
* =========================================================================
|
||||
* File : vfs_tree.h
|
||||
* Project : 0 A.D.
|
||||
* Description : the actual 'filesystem' and its tree of directories.
|
||||
* =========================================================================
|
||||
*/
|
||||
|
||||
// license: GPL; see lib/license.txt
|
||||
|
||||
#ifndef INCLUDED_VFS_TREE
|
||||
#define INCLUDED_VFS_TREE
|
||||
|
||||
#include "lib/file/filesystem.h" // FileInfo
|
||||
|
||||
// we add/cancel directory watches from the VFS mount code for convenience -
|
||||
// it iterates through all subdirectories anyway (*) and provides storage for
|
||||
// a key to identify the watch (obviates separate TDir -> watch mapping).
|
||||
//
|
||||
// define this to strip out that code - removes .watch from struct TDir,
|
||||
// and calls to res_watch_dir / res_cancel_watch.
|
||||
//
|
||||
// *: the add_watch code would need to iterate through subdirs and watch
|
||||
// each one, because the monitor API (e.g. FAM) may only be able to
|
||||
// watch single directories, instead of a whole subdirectory tree.
|
||||
#define NO_DIR_WATCH
|
||||
|
||||
class DirectoryWatch
|
||||
{
|
||||
public:
|
||||
DirectoryWatch()
|
||||
#ifndef NO_DIR_WATCH
|
||||
: m_watch(0)
|
||||
#endif
|
||||
{
|
||||
}
|
||||
|
||||
// 'watch' this directory for changes to support hotloading.
|
||||
void Register(const char* path)
|
||||
{
|
||||
#ifndef NO_DIR_WATCH
|
||||
char osPath[PATH_MAX];
|
||||
(void)path_MakeAbsolute(path, osPath);
|
||||
(void)dir_add_watch(osPath, &m_watch);
|
||||
#else
|
||||
UNUSED2(path);
|
||||
#endif
|
||||
}
|
||||
|
||||
void Unregister()
|
||||
{
|
||||
#ifndef NO_DIR_WATCH
|
||||
if(m_watch) // avoid dir_cancel_watch complaining
|
||||
{
|
||||
dir_cancel_watch(m_watch);
|
||||
m_watch = 0;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef NO_DIR_WATCH
|
||||
private:
|
||||
intptr_t m_watch;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
struct ArchiveEntry;
|
||||
|
||||
class VfsFile
|
||||
{
|
||||
friend class VfsDirectory;
|
||||
|
||||
public:
|
||||
VfsFile(const FileInfo& fileInfo, unsigned priority)
|
||||
: m_fileInfo(fileInfo), m_priority(priority), m_path(0), m_archiveEntry(0)
|
||||
{
|
||||
}
|
||||
|
||||
void GetFileInfo(FileInfo& fileInfo) const
|
||||
{
|
||||
fileInfo = m_fileInfo;
|
||||
}
|
||||
|
||||
const char* Name() const
|
||||
{
|
||||
return m_fileInfo.Name();
|
||||
}
|
||||
|
||||
off_t Size() const
|
||||
{
|
||||
return m_fileInfo.Size();
|
||||
}
|
||||
|
||||
time_t MTime() const
|
||||
{
|
||||
return m_fileInfo.MTime();
|
||||
}
|
||||
|
||||
LibError Load(u8* buf) const;
|
||||
|
||||
unsigned Priority() const
|
||||
{
|
||||
return m_priority;
|
||||
}
|
||||
|
||||
unsigned Precedence() const
|
||||
{
|
||||
return m_archiveEntry? 1u : 2u;
|
||||
}
|
||||
|
||||
char LocationCode() const
|
||||
{
|
||||
return m_archiveEntry? 'A' : 'F';
|
||||
}
|
||||
|
||||
private:
|
||||
FileInfo m_fileInfo;
|
||||
|
||||
unsigned m_priority;
|
||||
|
||||
// location (directory or archive) of the file. this allows opening
|
||||
// the file in O(1) even when there are multiple mounted directories.
|
||||
const char* m_path;
|
||||
ArchiveEntry* m_archiveEntry;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class VfsDirectory
|
||||
{
|
||||
public:
|
||||
VfsDirectory(const char* vfsPath);
|
||||
~VfsDirectory();
|
||||
|
||||
void AddFile(const FileInfo& fileInfo);
|
||||
void AddSubdirectory(const char* name);
|
||||
|
||||
VfsFile* GetFile(const char* name) const;
|
||||
VfsDirectory* GetSubdirectory(const char* name) const;
|
||||
|
||||
// (don't split this into 2 functions because POSIX can't implement
|
||||
// that efficiently)
|
||||
void GetEntries(std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
|
||||
|
||||
void DisplayR() const;
|
||||
void ClearR();
|
||||
|
||||
void AssociateWithRealDirectory(const char* path);
|
||||
void Populate();
|
||||
|
||||
void CreateFile(const char* name, const u8* buf, size_t size, uint flags = 0);
|
||||
|
||||
private:
|
||||
typedef std::map<const char*, VfsFile> Files;
|
||||
Files m_files;
|
||||
|
||||
typedef std::map<const char*, VfsDirectory> Subdirectories;
|
||||
Subdirectories m_subdirectories;
|
||||
|
||||
const char* m_vfsPath;
|
||||
|
||||
// if exactly one real directory is mounted into this virtual dir,
|
||||
// this points to its location. used to add files to VFS when writing.
|
||||
const char* m_path;
|
||||
|
||||
DirectoryWatch m_watch;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class VfsTree
|
||||
{
|
||||
public:
|
||||
VfsTree();
|
||||
|
||||
VfsDirectory& Root()
|
||||
{
|
||||
return m_rootDirectory;
|
||||
}
|
||||
|
||||
const VfsDirectory& Root() const
|
||||
{
|
||||
return m_rootDirectory;
|
||||
}
|
||||
|
||||
void Display() const;
|
||||
void Clear();
|
||||
|
||||
private:
|
||||
VfsDirectory m_rootDirectory;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_VFS_TREE
|
Loading…
Reference in New Issue
Block a user