Dunedan/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Wiki Activity

WIP file code, getting pretty close to functional

Browse Source 
archive: ArchiveEntry now a mememto and not exposed directly
typedefs for boost::shared_ptr
fixed up dir_util to new interface
add "FileProvider" that does Load/Store and indicates Precedence

vfs: structure has changed. one RealDirectory per OS dir; vfs_tree
stores it; populate uses it to Add() stuff to vfs_tree; lookup does path
traversal and calls populate

This was SVN commit r5499.
This commit is contained in:
janwas 2007-12-01 18:32:43 +00:00
parent ea96aa6b89
commit 75ab906315
39 changed files with 1351 additions and 1579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
source/lib/file/archive/archive.h
View File

@ -11,6 +11,8 @@
#ifndef INCLUDED_ARCHIVE
#define INCLUDED_ARCHIVE
#include "lib/file/filesystem.h"
// rationale: this module doesn't build a directory tree of the entries
// within an archive. that task is left to the VFS; here, we are only
// concerned with enumerating all archive entries.
@ -25,7 +27,7 @@ namespace ERR
// the IArchiveReader implementation.
struct ArchiveEntry;
struct IArchiveReader
struct IArchiveReader : public IFileProvider
{
virtual ~IArchiveReader();
@ -33,12 +35,12 @@ struct IArchiveReader
* called for each archive entry.
* @param pathname full pathname of entry (unique pointer)
**/
typedef void (*ArchiveEntryCallback)(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData);
typedef void (*ArchiveEntryCallback)(const char* pathname, const FileInfo& fileInfo, const ArchiveEntry* archiveEntry, uintptr_t cbData);
virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) = 0;
virtual LibError LoadFile(ArchiveEntry& archiveEntry, u8* fileContents) const = 0;
};
typedef boost::shared_ptr<IArchiveReader> PIArchiveReader;
// note: when creating an archive, any existing file with the given pathname
// will be overwritten.
@ -69,40 +71,6 @@ struct IArchiveWriter
virtual LibError AddFile(const char* pathname) = 0;
};
/**
* describes an archive entry (either file or directory).
**/
struct ArchiveEntry
{
ArchiveEntry()
{
}
ArchiveEntry(off_t ofs, off_t usize, off_t csize, time_t mtime, u32 checksum, uint method, uint flags)
{
this->ofs = ofs;
this->usize = usize;
this->csize = csize;
this->mtime = mtime;
this->checksum = checksum;
this->method = method;
this->flags = flags;
}
// this is needed by the VFS and stored here instead of in VfsFile to
// yield a nice 32-byte size.
IArchiveReader* archiveReader;
off_t ofs;
off_t usize;
off_t csize;
time_t mtime;
// the archive implementation establishes the meaning of the following:
u32 checksum;
uint method;
uint flags;
};
typedef boost::shared_ptr<IArchiveWriter> PIArchiveWriter;
#endif // #ifndef INCLUDED_ARCHIVE

54
source/lib/file/archive/archive_builder.cpp
View File

@ -10,7 +10,7 @@
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "vfs_optimizer.h"
//#include "vfs_optimizer.h"
#include <set>
#include <map>
@ -1017,7 +1017,7 @@ void trace_get(Trace* t)
// whether this IO would be satisfied by the file cache.
bool trace_entry_causes_io(FileCache& simulatedCache, const TraceEntry* ent)
{
IoBuf buf;
FileCacheData buf;
const size_t size = ent->size;
const char* vfsPathname = ent->vfsPathname;
switch(ent->op)
@ -1027,7 +1027,7 @@ bool trace_entry_causes_io(FileCache& simulatedCache, const TraceEntry* ent)
case TO_LOAD:
// cache miss
if(!simulatedCache.Find(vfsPathname, size))
if(!simulatedCache.Retrieve(vfsPathname, size))
{
// TODO: simulatedCache never evicts anything..
simulatedCache.Reserve();
@ -1045,4 +1045,50 @@ bool trace_entry_causes_io(FileCache& simulatedCache, const TraceEntry* ent)
}
return false;
}
}
// one run per file
// enabled by default. by the time we can decide whether a trace needs to
// be generated (see should_rebuild_main_archive), file accesses will
// already have occurred; hence default enabled and disable if not needed.
{
// carry out this entry's operation
switch(ent->op)
{
// do not 'run' writes - we'd destroy the existing data.
case TO_STORE:
break;
case TO_LOAD:
{
IoBuf buf; size_t size;
(void)vfs_load(ent->vfsPathname, buf, size, ent->flags);
break;
}
case TO_FREE:
fileCache.Release(ent->vfsPathname);
break;
default:
debug_warn("unknown TraceOp");
}
}
LibError trace_run(const char* osPathname)
{
Trace trace;
RETURN_ERR(trace.Load(osPathname));
for(uint i = 0; i < trace.NumEntries(); i++)
trace.Entries()[i]->Run();
return INFO::OK;
}

156
source/lib/file/archive/archive_zip.cpp
View File

@ -17,9 +17,9 @@
#include "lib/bits.h"
#include "lib/byte_order.h"
#include "lib/fat_time.h"
#include "lib/path_util.h"
#include "lib/allocators/pool.h"
#include "lib/sysdep/cpu.h" // cpu_memcpy
#include "lib/file/path.h"
#include "archive.h"
#include "codec_zlib.h"
#include "stream.h"
@ -29,9 +29,19 @@
#include "lib/file/io/io_manager.h"
enum ArchiveZipFlags
//-----------------------------------------------------------------------------
// Zip file data structures and signatures
//-----------------------------------------------------------------------------
enum ZipMethod
{
// indicates ArchiveEntry.ofs points to a "local file header" instead of
ZIP_METHOD_NONE = 0,
ZIP_METHOD_DEFLATE = 8
};
enum ZipFlags
{
// indicates ZipEntry.ofs points to a "local file header" instead of
// the file data. a fixup routine is called when reading the file;
// it skips past the LFH and clears this flag.
// this is somewhat of a hack, but vital to archive open performance.
@ -44,16 +54,31 @@ enum ArchiveZipFlags
NEEDS_LFH_FIXUP = 1
};
//-----------------------------------------------------------------------------
// Zip file data structures and signatures
//-----------------------------------------------------------------------------
enum ZipMethod
// all relevant LFH/CDFH fields not covered by FileInfo
struct ZipEntry
{
ZIP_METHOD_NONE = 0,
ZIP_METHOD_DEFLATE = 8
ZipEntry()
{
}
ZipEntry(off_t ofs, off_t csize, u32 checksum, u16 method, u16 flags)
{
this->ofs = ofs;
this->csize = csize;
this->checksum = checksum;
this->method = method;
this->flags = flags;
}
off_t ofs;
off_t csize;
u32 checksum;
u16 method;
u16 flags;
};
static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2');
static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4');
static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6');
@ -63,16 +88,16 @@ static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6');
class LFH
{
public:
void Init(const ArchiveEntry& archiveEntry, const char* pathname, size_t pathnameLength)
void Init(const FileInfo& fileInfo, const ZipEntry& zipEntry, const char* pathname, size_t pathnameLength)
{
m_magic = lfh_magic;
m_x1 = to_le16(0);
m_flags = to_le16(0);
m_method = to_le16(u16_from_larger(archiveEntry.method));
m_fat_mtime = to_le32(FAT_from_time_t(archiveEntry.mtime));
m_crc = to_le32(archiveEntry.checksum);
m_csize = to_le32(u32_from_larger(archiveEntry.csize));
m_usize = to_le32(u32_from_larger(archiveEntry.usize));
m_method = to_le16(u16_from_larger(zipEntry.method));
m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime()));
m_crc = to_le32(zipEntry.checksum);
m_csize = to_le32(u32_from_larger(zipEntry.csize));
m_usize = to_le32(u32_from_larger(fileInfo.Size()));
m_fn_len = to_le16(u16_from_larger((uint)pathnameLength));
m_e_len = to_le16(0);
@ -108,27 +133,27 @@ cassert(sizeof(LFH) == 30);
class CDFH
{
public:
void Init(const ArchiveEntry& archiveEntry, const char* pathname, size_t pathnameLength, size_t slack)
void Init(const FileInfo& fileInfo, const ZipEntry& zipEntry, const char* pathname, size_t pathnameLength, size_t slack)
{
m_magic = cdfh_magic;
m_x1 = to_le32(0);
m_flags = to_le16(0);
m_method = to_le16(u16_from_larger(archiveEntry.method));
m_fat_mtime = to_le32(FAT_from_time_t(archiveEntry.mtime));
m_crc = to_le32(archiveEntry.checksum);
m_csize = to_le32(u32_from_larger(archiveEntry.csize));
m_usize = to_le32(u32_from_larger(archiveEntry.usize));
m_method = to_le16(u16_from_larger(zipEntry.method));
m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime()));
m_crc = to_le32(zipEntry.checksum);
m_csize = to_le32(u32_from_larger(zipEntry.csize));
m_usize = to_le32(u32_from_larger(fileInfo.Size()));
m_fn_len = to_le16(u16_from_larger((uint)pathnameLength));
m_e_len = to_le16(0);
m_c_len = to_le16(u16_from_larger((uint)slack));
m_x2 = to_le32(0);
m_x3 = to_le32(0);
m_lfh_ofs = to_le32(archiveEntry.ofs);
m_lfh_ofs = to_le32(zipEntry.ofs);
cpu_memcpy((char*)this + sizeof(CDFH), pathname, pathnameLength);
}
void Decompose(ArchiveEntry& archiveEntry, const char*& pathname, size_t& cdfhSize) const
void Decompose(FileInfo& fileInfo, ZipEntry& zipEntry, const char*& pathname, size_t& cdfhSize) const
{
const u16 zipMethod = read_le16(&m_method);
const u32 fat_mtime = read_le32(&m_fat_mtime);
@ -140,21 +165,16 @@ public:
const u16 c_len = read_le16(&m_c_len);
const u32 lfh_ofs = read_le32(&m_lfh_ofs);
archiveEntry.ofs = (off_t)lfh_ofs;
archiveEntry.usize = (off_t)usize;
archiveEntry.csize = (off_t)csize;
archiveEntry.mtime = time_t_from_FAT(fat_mtime);
archiveEntry.checksum = crc;
archiveEntry.method = (uint)zipMethod;
archiveEntry.flags = NEEDS_LFH_FIXUP;
// return 0-terminated copy of filename
// get 0-terminated copy of pathname
const char* fn = (const char*)this + sizeof(CDFH); // not 0-terminated!
char buf[PATH_MAX]; // path_Pool()->UniqueCopy requires a 0-terminated string
cpu_memcpy(buf, fn, fn_len*sizeof(char));
buf[fn_len] = '\0';
pathname = path_Pool()->UniqueCopy(buf);
fileInfo = FileInfo(path_name_only(pathname), (off_t)usize, time_t_from_FAT(fat_mtime));
zipEntry = ZipEntry((off_t)lfh_ofs, (off_t)csize, crc, zipMethod, NEEDS_LFH_FIXUP);
cdfhSize = sizeof(CDFH) + fn_len + e_len + c_len;
}
@ -337,13 +357,14 @@ public:
if(!cdfh)
WARN_RETURN(ERR::CORRUPTED);
ArchiveEntry archiveEntry;
FileInfo fileInfo;
ZipEntry zipEntry;
const char* pathname;
size_t cdfhSize;
cdfh->Decompose(archiveEntry, pathname, cdfhSize);
cdfh->Decompose(fileInfo, zipEntry, pathname, cdfhSize);
m_entries.push_back(archiveEntry);
cb(pathname, m_entries.back(), cbData);
m_entries.push_back(zipEntry);
cb(pathname, fileInfo, (ArchiveEntry*)&m_entries.back(), cbData);
pos += cdfhSize;
}
@ -351,16 +372,28 @@ public:
return INFO::OK;
}
virtual LibError LoadFile(ArchiveEntry& archiveEntry, u8* fileContents) const
virtual unsigned Precedence() const
{
if((archiveEntry.flags & NEEDS_LFH_FIXUP) != 0)
return 2u;
}
virtual char LocationCode() const
{
return 'A';
}
virtual LibError Load(const char* UNUSED(name), const void* location, u8* fileContents, size_t size) const
{
ZipEntry* zipEntry = (ZipEntry*)location;
if((zipEntry->flags & NEEDS_LFH_FIXUP) != 0)
{
FixupEntry(archiveEntry);
archiveEntry.flags &= ~NEEDS_LFH_FIXUP;
FixupEntry(*zipEntry);
zipEntry->flags &= ~NEEDS_LFH_FIXUP;
}
boost::shared_ptr<ICodec> codec;
switch(archiveEntry.method)
switch(zipEntry->method)
{
case ZIP_METHOD_NONE:
codec = CreateCodec_ZLibNone();
@ -373,14 +406,19 @@ public:
}
Stream stream(codec);
stream.SetOutputBuffer(fileContents, archiveEntry.usize);
RETURN_ERR(io_Read(m_file, archiveEntry.ofs, IO_BUF_TEMP, archiveEntry.csize, FeedStream, (uintptr_t)&stream));
stream.SetOutputBuffer(fileContents, size);
RETURN_ERR(io_Read(m_file, zipEntry->ofs, IO_BUF_TEMP, zipEntry->csize, FeedStream, (uintptr_t)&stream));
RETURN_ERR(stream.Finish());
debug_assert(archiveEntry.checksum == stream.Checksum());
debug_assert(zipEntry->checksum == stream.Checksum());
return INFO::OK;
}
virtual LibError Store(const char* UNUSED(name), const void* UNUSED(location), const u8* UNUSED(fileContents), size_t UNUSED(size)) const
{
return ERR::NOT_SUPPORTED;
}
private:
struct LFH_Copier
{
@ -409,9 +447,9 @@ private:
}
/**
* fix up archiveEntry's offset within the archive.
* fix up zipEntry's offset within the archive.
* called by archive_LoadFile iff NEEDS_LFH_FIXUP is set.
* side effects: adjusts archiveEntry.ofs.
* side effects: adjusts zipEntry.ofs.
*
* ensures <ent.ofs> points to the actual file contents; it is initially
* the offset of the LFH. we cannot use CDFH filename and extra field
@ -421,27 +459,27 @@ private:
* reduce seeks: since reading the file will typically follow, the
* block cache entirely absorbs the IO cost.
**/
void FixupEntry(ArchiveEntry& archiveEntry) const
void FixupEntry(ZipEntry& zipEntry) const
{
// performance note: this ends up reading one file block, which is
// only in the block cache if the file starts in the same block as a
// previously read file (i.e. both are small).
LFH lfh;
LFH_Copier params = { (u8*)&lfh, sizeof(LFH) };
THROW_ERR(io_Read(m_file, archiveEntry.ofs, IO_BUF_TEMP, sizeof(LFH), lfh_copier_cb, (uintptr_t)&params));
THROW_ERR(io_Read(m_file, zipEntry.ofs, IO_BUF_TEMP, sizeof(LFH), lfh_copier_cb, (uintptr_t)&params));
archiveEntry.ofs += (off_t)lfh.Size();
zipEntry.ofs += (off_t)lfh.Size();
}
File_Posix m_file;
off_t m_fileSize;
std::vector<ArchiveEntry> m_entries;
std::vector<ZipEntry> m_entries;
};
boost::shared_ptr<IArchiveReader> CreateArchiveReader_Zip(const char* archivePathname)
{
return boost::shared_ptr<IArchiveReader>(new ArchiveReader_Zip(archivePathname));
return PIArchiveReader(new ArchiveReader_Zip(archivePathname));
}
@ -517,7 +555,7 @@ public:
// choose method and the corresponding codec
ZipMethod zipMethod;
boost::shared_ptr<ICodec> codec;
PICodec codec;
if(IsFileTypeIncompressible(pathname))
{
zipMethod = ZIP_METHOD_NONE;
@ -546,12 +584,12 @@ public:
checksum = stream.Checksum();
}
ArchiveEntry archiveEntry(m_fileSize, usize, (off_t)csize, fileInfo.MTime(), checksum, zipMethod, 0);
ZipEntry zipEntry(m_fileSize, (off_t)csize, checksum, zipMethod, 0);
// build LFH
{
LFH* lfh = (LFH*)buf.get();
lfh->Init(archiveEntry, pathname, pathnameLength);
lfh->Init(fileInfo, zipEntry, pathname, pathnameLength);
}
// write to LFH, pathname and cdata to file
@ -559,7 +597,7 @@ public:
RETURN_ERR(io_Write(m_file, m_fileSize, buf, packageSize));
m_fileSize += (off_t)packageSize;
// append a CDFH to the central dir (in memory)
// append a CDFH to the central directory (in memory)
// .. note: pool_alloc may round size up for padding purposes.
const size_t prev_pos = m_cdfhPool.da.pos;
const size_t cdfhSize = sizeof(CDFH) + pathnameLength;
@ -567,7 +605,7 @@ public:
if(!cdfh)
WARN_RETURN(ERR::NO_MEM);
const size_t slack = m_cdfhPool.da.pos - prev_pos - cdfhSize;
cdfh->Init(archiveEntry, pathname, pathnameLength, slack);
cdfh->Init(fileInfo, zipEntry, pathname, pathnameLength, slack);
m_numEntries++;
@ -582,7 +620,7 @@ private:
uint m_numEntries;
};
boost::shared_ptr<IArchiveWriter> CreateArchiveWriter_Zip(const char* archivePathname)
PIArchiveWriter CreateArchiveWriter_Zip(const char* archivePathname)
{
return boost::shared_ptr<IArchiveWriter>(new ArchiveWriter_Zip(archivePathname));
return PIArchiveWriter(new ArchiveWriter_Zip(archivePathname));
}

7
source/lib/file/archive/archive_zip.h
View File

@ -11,10 +11,9 @@
#ifndef INCLUDED_ARCHIVE_ZIP
#define INCLUDED_ARCHIVE_ZIP
struct IArchiveReader;
boost::shared_ptr<IArchiveReader> CreateArchiveReader_Zip(const char* archivePathname);
#include "archive.h"
struct IArchiveWriter;
extern boost::shared_ptr<IArchiveWriter> CreateArchiveWriter_Zip(const char* archivePathname);
PIArchiveReader CreateArchiveReader_Zip(const char* archivePathname);
PIArchiveWriter CreateArchiveWriter_Zip(const char* archivePathname);
#endif // #ifndef INCLUDED_ARCHIVE_ZIP

2
source/lib/file/archive/codec.h
View File

@ -67,4 +67,6 @@ public:
virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const = 0;
};
typedef boost::shared_ptr<ICodec> PICodec;
#endif // #ifndef INCLUDED_CODEC

12
source/lib/file/archive/codec_zlib.cpp
View File

@ -257,17 +257,17 @@ public:
//-----------------------------------------------------------------------------
boost::shared_ptr<ICodec> CreateCodec_ZLibNone()
PICodec CreateCodec_ZLibNone()
{
return boost::shared_ptr<ICodec>(new Codec_ZLibNone);
return PICodec(new Codec_ZLibNone);
}
boost::shared_ptr<ICodec> CreateCompressor_ZLibDeflate()
PICodec CreateCompressor_ZLibDeflate()
{
return boost::shared_ptr<ICodec>(new Compressor_ZLib);
return PICodec(new Compressor_ZLib);
}
boost::shared_ptr<ICodec> CreateDecompressor_ZLibDeflate()
PICodec CreateDecompressor_ZLibDeflate()
{
return boost::shared_ptr<ICodec>(new Decompressor_ZLib);
return PICodec (new Decompressor_ZLib);
}

6
source/lib/file/archive/codec_zlib.h
View File

@ -1,5 +1,5 @@
#include "codec.h"
extern boost::shared_ptr<ICodec> CreateCodec_ZLibNone();
extern boost::shared_ptr<ICodec> CreateCompressor_ZLibDeflate();
extern boost::shared_ptr<ICodec> CreateDecompressor_ZLibDeflate();
extern PICodec CreateCodec_ZLibNone();
extern PICodec CreateCompressor_ZLibDeflate();
extern PICodec CreateDecompressor_ZLibDeflate();

2
source/lib/file/archive/stream.cpp
View File

@ -78,7 +78,7 @@ bool OutputBufferManager::IsAllowableBuffer(u8* buffer, size_t size)
//-----------------------------------------------------------------------------
Stream::Stream(boost::shared_ptr<ICodec> codec)
Stream::Stream(PICodec codec)
: m_codec(codec)
, m_inConsumed(0), m_outProduced(0)
{

6
source/lib/file/archive/stream.h
View File

@ -11,7 +11,7 @@
#ifndef INCLUDED_STREAM
#define INCLUDED_STREAM
struct ICodec;
#include "codec.h"
// note: this is similar in function to std::vector, but we don't need
// iterators etc. and would prefer to avoid initializing each byte.
@ -62,7 +62,7 @@ private:
class Stream
{
public:
Stream(boost::shared_ptr<ICodec> codec);
Stream(PICodec codec);
void SetOutputBuffer(u8* out, size_t outSize);
@ -90,7 +90,7 @@ public:
}
private:
boost::shared_ptr<ICodec> m_codec;
PICodec m_codec;
OutputBufferManager m_outputBufferManager;
size_t m_inConsumed;

235
source/lib/file/archive/trace.cpp
View File

@ -1,235 +0,0 @@
/**
* =========================================================================
* File : trace.cpp
* Project : 0 A.D.
* Description : allows recording and 'playing back' a sequence of
* : I/Os - useful for benchmarking and archive builder.
* =========================================================================
*/
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "trace.h"
#include "lib/allocators.h"
#include "lib/timer.h"
#include "lib/sysdep/cpu.h"
// one run per file
/*virtual*/ ITrace::~ITrace()
{
}
class Trace_Dummy : public ITrace
{
public:
Trace_Dummy(size_t UNUSED(maxSize))
{
}
virtual void NotifyLoad(const char* UNUSED(pathname), size_t UNUSED(size))
{
}
virtual void NotifyStore(const char* UNUSED(pathname), size_t UNUSED(size))
{
}
virtual void NotifyFree(const char* UNUSED(pathname))
{
}
virtual LibError Load(const char* UNUSED(vfsPathname))
{
return INFO::OK;
}
virtual LibError Store(const char* UNUSED(vfsPathname)) const
{
return INFO::OK;
}
virtual const TraceEntry* Entries() const
{
return 0;
}
virtual size_t NumEntries() const
{
return 0;
}
};
class Trace : public ITrace
{
public:
Trace(size_t maxSize)
{
(void)pool_create(&m_pool, maxSize, sizeof(TraceEntry));
}
virtual ~Trace()
{
(void)pool_destroy(&m_pool);
}
virtual void NotifyLoad(const char* pathname, size_t size)
{
AddEntry(TO_LOAD, pathname, size);
}
virtual void NotifyStore(const char* pathname, size_t size)
{
AddEntry(TO_STORE, pathname, size);
}
virtual void NotifyFree(const char* pathname)
{
const size_t size = 0; // meaningless for TO_FREE
AddEntry(TO_FREE, pathname, size);
}
virtual LibError Load(const char* osPathname)
{
pool_free_all(&m_pool);
FILE* f = fopen(osPathname, "rt");
if(!f)
WARN_RETURN(ERR::FILE_ACCESS);
for(size_t i = 0; ; i++)
{
TraceEntry* t = (TraceEntry*)pool_alloc(&m_pool, 0);
t->Read(f);
}
fclose(f);
return INFO::OK;
}
virtual LibError Store(const char* osPathname) const
{
FILE* f = fopen(osPathname, "at");
if(!f)
WARN_RETURN(ERR::FILE_ACCESS);
for(size_t i = 0; i < NumEntries(); i++)
Entries()[i]->Write(f);
(void)fclose(f);
return INFO::OK;
}
virtual const TraceEntry* Entries() const
{
return (const TraceEntry*)m_pool.da.base;
}
virtual size_t NumEntries() const
{
return m_pool.da.pos / sizeof(TraceEntry);
}
private:
void AddEntry(TraceOp op, const char* pathname, size_t size, double timestamp = 0.0)
{
TraceEntry* t = (TraceEntry*)pool_alloc(&m_pool, 0);
debug_assert(t);
t->timestamp = timestamp == 0.0? get_time() : timestamp;
t->vfsPathname = path_Pool()->UniqueCopy(pathname);
t->size = size;
t->op = op;
}
Pool m_pool;
};
// parse lines and stuff them in m_pool
// (as if they had been AddEntry-ed; replaces any existing data)
// .. bake PATH_MAX limit into string.
char fmt[30];
snprintf(fmt, ARRAY_SIZE(fmt), "%%lf: %%c \"%%%d[^\"]\" %%d %%04x\n", PATH_MAX);
double timestamp; char opcode; char P_path[PATH_MAX]; size_t size;
int fieldsRead = fscanf(f, fmt, &timestamp, &opcode, P_path, &size);
if(fieldsRead == EOF)
break;
debug_assert(fieldsRead == 4);
static char CharFromTraceOp(TraceOp op)
{
switch(op)
{
case TO_LOAD:
return 'L';
case TO_STORE:
return 'S';
case TO_FREE:
return 'F';
default:
DEBUG_WARN_ERR(ERR::CORRUPTED);
}
}
static TraceOp TraceOpFromChar(char c)
{
switch(c)
{
case 'L':
return TO_LOAD;
case 'S':
return TO_STORE;
case 'F':
return TO_FREE;
default:
DEBUG_WARN_ERR(ERR::CORRUPTED);
}
}
static void write_entry(FILE* f, const TraceEntry* ent)
{
fprintf(f, "%#010f: %c \"%s\" %d\n", ent->timestamp, CharFromTraceOp(ent->op), ent->vfsPathname, ent->size);
}
{
// carry out this entry's operation
switch(ent->op)
{
// do not 'run' writes - we'd destroy the existing data.
case TO_STORE:
break;
case TO_LOAD:
{
IoBuf buf; size_t size;
(void)vfs_load(ent->vfsPathname, buf, size, ent->flags);
break;
}
case TO_FREE:
fileCache.Release(ent->vfsPathname);
break;
default:
debug_warn("unknown TraceOp");
}
}
LibError trace_run(const char* osPathname)
{
Trace trace;
RETURN_ERR(trace.Load(osPathname));
for(uint i = 0; i < trace.NumEntries(); i++)
trace.Entries()[i]->Run();
return INFO::OK;
}

66
source/lib/file/archive/trace.h
View File

@ -1,66 +0,0 @@
/**
* =========================================================================
* File : trace.h
* Project : 0 A.D.
* Description : allows recording and 'playing back' a sequence of
* : I/Os - useful for benchmarking and archive builder.
* =========================================================================
*/
// license: GPL; see lib/license.txt
#ifndef INCLUDED_TRACE
#define INCLUDED_TRACE
// TraceEntry operation type.
// note: rather than only a list of accessed files, we also need to
// know the application's behavior WRT caching (e.g. when it releases
// cached buffers). this is necessary so that our simulation can
// yield the same behavior.
enum TraceOp
{
TO_LOAD,
TO_STORE,
TO_FREE,
};
// stores one event that is relevant for file IO / caching.
//
// size-optimized a bit since these are all kept in memory
// (to prevent trace file writes from affecting other IOs)
struct TraceEntry
{
// note: float instead of double for nice 16 byte struct size
float timestamp; // returned by get_time before operation starts
const char* vfsPathname;
// rationale: store size in the trace because other applications
// that use this trace format but not our IO code wouldn't know
// size (since they cannot retrieve the file info given vfsPathname).
size_t size; // of IO (usually the entire file)
TraceOp op;
};
struct ITrace
{
virtual ~ITrace();
virtual void NotifyLoad(const char* vfsPathname, size_t size);
virtual void NotifyStore(const char* vfsPathname, size_t size);
virtual void NotifyFree(const char* vfsPathname);
// note: file format is text-based to allow human inspection and because
// storing filenames would be a bit awkward
virtual LibError Load(const char* osPathname);
virtual LibError Store(const char* osPathname) const;
virtual const TraceEntry* Entries() const;
virtual size_t NumEntries() const;
};
// enabled by default. by the time we can decide whether a trace needs to
// be generated (see should_rebuild_main_archive), file accesses will
// already have occurred; hence default enabled and disable if not needed.
#endif // #ifndef INCLUDED_TRACE

217
source/lib/file/dir_util.cpp
View File

@ -2,7 +2,7 @@
* =========================================================================
* File : dir_util.cpp
* Project : 0 A.D.
* Description :
* Description : helper functions for directory access
* =========================================================================
*/
@ -12,9 +12,8 @@
#include "dir_util.h"
#include <queue>
#include <cstring>
#include "filesystem.h"
#include "path.h"
#include "lib/path_util.h"
#include "lib/regex.h"
@ -22,226 +21,116 @@
bool dir_FileExists(IFilesystem* fs, const char* pathname)
{
FileInfo fileInfo;
if(fs->GetEntry(pathname, fileInfo) < 0)
if(fs->GetFileInfo(pathname, fileInfo) < 0)
return false;
debug_assert(!fileInfo.IsDirectory());
return true;
}
bool dir_DirectoryExists(IFilesystem* fs, const char* dirPath)
{
FileInfo fileInfo;
if(fs->GetEntry(dirPath, fileInfo) < 0)
return false;
debug_assert(fileInfo.IsDirectory());
return true;
}
struct FsEntryNameLess : public std::binary_function<const FileInfo, const FileInfo, bool>
struct FileInfoNameLess : public std::binary_function<const FileInfo, const FileInfo, bool>
{
bool operator()(const FileInfo& fileInfo1, const FileInfo& fileInfo2) const
{
return strcmp(fileInfo1.name, fileInfo2.name) < 0;
return strcmp(fileInfo1.Name(), fileInfo2.Name()) < 0;
}
};
LibError dir_GatherSortedEntries(IFilesystem* fs, const char* dirPath, FilesystemEntries& fsEntries)
void dir_SortFiles(FileInfos& files)
{
DirectoryIterator di(fs, dirPath);
fsEntries.reserve(50); // preallocate for efficiency
FileInfo fileInfo;
for(;;)
{
LibError ret = di.NextEntry(fileInfo);
if(ret == ERR::DIR_END)
break;
RETURN_ERR(ret);
fsEntries.push_back(fileInfo);
}
std::sort(fsEntries.begin(), fsEntries.end(), FsEntryNameLess());
return INFO::OK;
std::sort(files.begin(), files.end(), FileInfoNameLess());
}
LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, const DirCallback cb, const uintptr_t cbData)
struct NameLess : public std::binary_function<const char*, const char*, bool>
{
PathPackage pp;
RETURN_ERR(path_package_set_dir(&pp, dirPath));
FilesystemEntries fsEntries;
RETURN_ERR(dir_GatherSortedEntries(fs, dirPath, fsEntries));
for(FilesystemEntries::const_iterator it = fsEntries.begin(); it != fsEntries.end(); ++it)
bool operator()(const char* name1, const char* name2) const
{
const FileInfo& fileInfo = *it;
path_package_append_file(&pp, fileInfo.name);
LibError ret = cb(pp.path, fileInfo, cbData);
if(ret != INFO::CB_CONTINUE)
return ret;
return strcmp(name1, name2) < 0;
}
};
return INFO::OK;
void dir_SortDirectories(Directories& directories)
{
std::sort(directories.begin(), directories.end(), NameLess());
}
LibError dir_filtered_next_ent(DirectoryIterator& di, FileInfo& fileInfo, const char* filter)
{
bool want_dir = true;
if(filter)
{
// directory
if(filter[0] == '/')
{
// .. and also files
if(filter[1] == '|')
filter += 2;
}
// file only
else
want_dir = false;
}
// loop until fileInfo matches what is requested, or end of directory.
for(;;)
{
RETURN_ERR(di.NextEntry(fileInfo));
if(fileInfo.IsDirectory())
{
if(want_dir)
break;
}
else
{
// (note: filter = 0 matches anything)
if(match_wildcard(fileInfo.name, filter))
break;
}
}
return INFO::OK;
}
// call <cb> for each fileInfo matching <user_filter> (see vfs_next_dirent) in
// directory <path>; if flags & VFS_DIR_RECURSIVE, entries in
// subdirectories are also returned.
//
// note: EnumDirEntsCB path and fileInfo are only valid during the callback.
LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint flags, const char* user_filter, DirCallback cb, uintptr_t cbData)
LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, DirCallback cb, uintptr_t cbData, const char* pattern, uint flags)
{
debug_assert((flags & ~(VFS_DIR_RECURSIVE)) == 0);
const bool recursive = (flags & VFS_DIR_RECURSIVE) != 0;
char filter_buf[PATH_MAX];
const char* filter = user_filter;
bool user_filter_wants_dirs = true;
if(user_filter)
{
if(user_filter[0] != '/')
user_filter_wants_dirs = false;
// we need subdirectories and the caller hasn't already requested them
if(recursive && !user_filter_wants_dirs)
{
snprintf(filter_buf, sizeof(filter_buf), "/|%s", user_filter);
filter = filter_buf;
}
}
// note: FIFO queue instead of recursion is much more efficient
// (less stack usage; avoids seeks by reading all entries in a
// directory consecutively)
// (declare here to avoid reallocations)
FileInfos files; Directories subdirectories;
// (a FIFO queue is more efficient than recursion because it uses less
// stack space and avoids seeks due to breadth-first traversal.)
std::queue<const char*> dir_queue;
dir_queue.push(path_Pool()->UniqueCopy(dirPath));
// for each directory:
do
while(!dir_queue.empty())
{
// get current directory path from queue
// note: can't refer to the queue contents - those are invalidated
// as soon as a directory is pushed onto it.
// get directory path
PathPackage pp;
(void)path_package_set_dir(&pp, dir_queue.front());
dir_queue.pop();
DirectoryIterator di(fs, pp.path);
RETURN_ERR(fs->GetDirectoryEntries(pp.path, &files, &subdirectories));
// for each fileInfo (file, subdir) in directory:
FileInfo fileInfo;
while(dir_filtered_next_ent(di, fileInfo, filter) == 0)
for(size_t i = 0; i < files.size(); i++)
{
// build complete path (FileInfo only stores fileInfo name)
(void)path_package_append_file(&pp, fileInfo.name);
const char* atom_path = path_Pool()->UniqueCopy(pp.path);
const FileInfo fileInfo = files[i];
if(!match_wildcard(fileInfo.Name(), pattern))
continue;
if(fileInfo.IsDirectory())
{
if(recursive)
dir_queue.push(atom_path);
// build complete path (FileInfo only stores name)
(void)path_package_append_file(&pp, fileInfo.Name());
cb(pp.path, fileInfo, cbData);
}
if(user_filter_wants_dirs)
cb(atom_path, fileInfo, cbData);
}
else
cb(atom_path, fileInfo, cbData);
if((flags & VFS_DIR_RECURSIVE) == 0)
break;
for(size_t i = 0; i < subdirectories.size(); i++)
{
(void)path_package_append_file(&pp, subdirectories[i]);
dir_queue.push(path_Pool()->UniqueCopy(pp.path));
}
}
while(!dir_queue.empty());
return INFO::OK;
}
// fill V_next_fn (which must be big enough for PATH_MAX chars) with
// the next numbered filename according to the pattern defined by V_fn_fmt.
// <state> must be initially zeroed (e.g. by defining as static) and passed
// each time.
//
// this function is useful when creating new files which are not to
// overwrite the previous ones, e.g. screenshots.
// example for V_fn_fmt: "screenshots/screenshot%04d.png".
void dir_NextNumberedFilename(IFilesystem* fs, const char* fn_fmt, NextNumberedFilenameState* state, char* next_fn)
void dir_NextNumberedFilename(IFilesystem* fs, const char* pathnameFmt, unsigned& nextNumber, char* nextPathname)
{
// (first call only:) scan directory and set next_num according to
// (first call only:) scan directory and set nextNumber according to
// highest matching filename found. this avoids filling "holes" in
// the number series due to deleted files, which could be confusing.
// example: add 1st and 2nd; [exit] delete 1st; [restart]
// add 3rd -> without this measure it would get number 1, not 3.
if(state->next_num == 0)
if(nextNumber == 0)
{
char dirPath[PATH_MAX];
path_dir_only(fn_fmt, dirPath);
const char* name_fmt = path_name_only(fn_fmt);
const char* path; const char* nameFmt;
path_split(pathnameFmt, &path, &nameFmt);
int max_num = -1; int num;
DirectoryIterator di(fs, dirPath);
FileInfo fileInfo;
while(di.NextEntry(fileInfo) == INFO::OK)
unsigned maxNumber = 0;
FileInfos files;
fs->GetDirectoryEntries(path, &files, 0);
for(size_t i = 0; i < files.size(); i++)
{
if(!fileInfo.IsDirectory() && sscanf(fileInfo.name, name_fmt, &num) == 1)
max_num = std::max(num, max_num);
unsigned number;
if(sscanf(files[i].Name(), nameFmt, &number) == 1)
maxNumber = std::max(number, maxNumber);
}
state->next_num = max_num+1;
nextNumber = maxNumber+1;
}
// now increment number until that file doesn't yet exist.
// this is fairly slow, but typically only happens once due
// to scan loop above. (we still need to provide for looping since
// someone may have added files in the meantime)
// binary search isn't expected to improve things.
// we don't bother with binary search - this isn't a bottleneck.
do
snprintf(next_fn, PATH_MAX, fn_fmt, state->next_num++);
while(dir_FileExists(fs, next_fn));
snprintf(nextPathname, PATH_MAX, pathnameFmt, nextNumber++);
while(dir_FileExists(fs, nextPathname));
}

146
source/lib/file/dir_util.h
View File

@ -2,7 +2,7 @@
* =========================================================================
* File : dir_util.h
* Project : 0 A.D.
* Description :
* Description : helper functions for directory access
* =========================================================================
*/
@ -14,118 +14,54 @@
#include "filesystem.h"
extern bool dir_FileExists(IFilesystem* fs, const char* pathname);
extern bool dir_DirectoryExists(IFilesystem* fs, const char* dirPath);
typedef std::vector<FileInfo> FilesystemEntries;
// enumerate all directory entries in <P_path>; add to container and
// then sort it by filename.
extern LibError dir_GatherSortedEntries(IFilesystem* fs, const char* dirPath, FilesystemEntries& fsEntries);
// called by dir_ForEachSortedEntry for each entry in the directory.
// return INFO::CB_CONTINUE to continue calling; anything else will cause
// dir_ForEachSortedEntry to abort and immediately return that value.
typedef LibError (*DirCallback)(const char* pathname, const FileInfo& fileInfo, const uintptr_t cbData);
// call <cb> for each file and subdirectory in <dir> (alphabetical order),
// passing the entry name (not full path!), stat info, and <user>.
//
// first builds a list of entries (sorted) and remembers if an error occurred.
// if <cb> returns non-zero, abort immediately and return that; otherwise,
// return first error encountered while listing files, or 0 on success.
//
// rationale:
// this makes dir_ForEachSortedEntry and zip_enum slightly incompatible, since zip_enum
// returns the full path. that's necessary because VFS zip_cb
// has no other way of determining what VFS dir a Zip file is in,
// since zip_enum enumerates all files in the archive (not only those
// in a given dir). no big deal though, since add_ent has to
// special-case Zip files anyway.
// the advantage here is simplicity, and sparing callbacks the trouble
// of converting from/to native path (we just give 'em the dirent name).
extern LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, DirCallback cb, uintptr_t cbData);
extern void dir_SortFiles(FileInfos& files);
extern void dir_SortDirectories(Directories& directories);
/**
* (mostly) insulating concrete class providing iterator access to
* directory entries.
* this is usable for posix, VFS, etc.; instances are created via IFilesystem.
* called for files in a directory.
*
* @param pathname full pathname (since FileInfo only gives the name).
* @param fileInfo file information
* @param cbData user-specified context
* @return INFO::CB_CONTINUE on success; any other value will immediately
* be returned to the caller (no more calls will be forthcoming).
*
* CAVEAT: pathname and fileInfo are only valid until the function
* returns!
**/
class DirectoryIterator
{
public:
DirectoryIterator(IFilesystem* fs, const char* dirPath)
: m_impl(fs->OpenDirectory(dirPath))
{
}
// return ERR::DIR_END if all entries have already been returned once,
// another negative error code, or INFO::OK on success, in which case <fileInfo>
// describes the next (order is unspecified) directory entry.
LibError NextEntry(FileInfo& fileInfo)
{
return m_impl.get()->NextEntry(fileInfo);
}
private:
boost::shared_ptr<IDirectoryIterator> m_impl;
};
// retrieve the next (order is unspecified) dir entry matching <filter>.
// return 0 on success, ERR::DIR_END if no matching entry was found,
// or a negative error code on failure.
// filter values:
// - 0: anything;
// - "/": any subdirectory;
// - "/|<pattern>": any subdirectory, or as below with <pattern>;
// - <pattern>: any file whose name matches; ? and * wildcards are allowed.
//
// note that the directory entries are only scanned once; after the
// end is reached (-> ERR::DIR_END returned), no further entries can
// be retrieved, even if filter changes (which shouldn't happen - see impl).
//
// rationale: we do not sort directory entries alphabetically here.
// most callers don't need it and the overhead is considerable
// (we'd have to store all entries in a vector). it is left up to
// other routines.
extern LibError dir_filtered_next_ent(DirectoryIterator& di, FileInfo& fileInfo, const char* filter);
enum DirEnumFlags
typedef LibError (*DirCallback)(const char* pathname, const FileInfo& fileInfo, const uintptr_t cbData);
enum DirFlags
{
/// include files in subdirectories.
VFS_DIR_RECURSIVE = 1
};
// call <cb> for each entry matching <user_filter> (see vfs_next_dirent) in
// directory <path>; if flags & VFS_DIR_RECURSIVE, entries in
// subdirectories are also returned.
extern LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint enum_flags, const char* filter, DirCallback cb, uintptr_t cbData);
/**
* call back for each file in a directory (tree)
*
* @param cb see DirCallback
* @param pattern that file names must match. '*' and '&' wildcards
* are allowed. 0 matches everything.
* @param flags see DirFlags
* @param LibError
**/
extern LibError dir_ForEachFile(IFilesystem* fs, const char* path, DirCallback cb, uintptr_t cbData, const char* pattern = 0, uint flags = 0);
struct NextNumberedFilenameState
{
int next_num;
};
/**
* determine the next available pathname with a given format.
* this is useful when creating new files without overwriting the previous
* ones (screenshots are a good example).
*
* @param pathnameFmt format string for the pathname; must contain one
* format specifier for an (unsigned) int.
* example: "screenshots/screenshot%04d.png"
* @param nextNumber in: the first number to try; out: the next number.
* if 0, numbers corresponding to existing files are skipped.
* @param receives the output; must hold at least PATH_MAX characters.
**/
extern void dir_NextNumberedFilename(IFilesystem* fs, const char* pathnameFmt, unsigned& nextNumber, char* nextPathname);
// fill V_next_fn (which must be big enough for PATH_MAX chars) with
// the next numbered filename according to the pattern defined by V_fn_fmt.
// <nnfs> must be initially zeroed (e.g. by defining as static) and passed
// each time.
// if <use_vfs> (default), the paths are treated as VFS paths; otherwise,
// file.cpp's functions are used. this is necessary because one of
// our callers needs a filename for VFS archive files.
//
// this function is useful when creating new files which are not to
// overwrite the previous ones, e.g. screenshots.
// example for V_fn_fmt: "screenshots/screenshot%04d.png".
extern void dir_NextNumberedFilename(IFilesystem* fs, const char* V_fn_fmt, NextNumberedFilenameState* nnfs, char* V_next_fn);
#endif // #ifndef INCLUDED_DIR_UTIL
#endif // #ifndef INCLUDED_DIR_UTIL

2
source/lib/file/file_stats.cpp
View File

@ -92,7 +92,7 @@ void stats_vfs_file_remove(size_t file_size)
vfs_size_total -= file_size;
}
// stats_vfs_init_* are currently unused
void stats_vfs_init_start()
{
timer_start();

14
source/lib/file/filesystem.cpp
View File

@ -1,16 +1,10 @@
#include "precompiled.h"
#include "filesystem.h"
#include "path.h"
ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES);
ERROR_ASSOCIATE(ERR::DIR_NOT_FOUND, "Directory not found", ENOENT);
ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO);
ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1);
// rationale for out-of-line dtor: see [Lakos]
IFilesystem::~IFilesystem()
{
}
IFileProvider::~IFileProvider()
{
}

22
source/lib/file/filesystem.h
View File

@ -56,6 +56,24 @@ private:
time_t m_mtime;
};
typedef std::vector<FileInfo> FileInfos;
typedef std::vector<const char*> Directories;
//
struct IFileProvider
{
virtual ~IFileProvider();
virtual unsigned Precedence() const = 0;
virtual char LocationCode() const = 0;
virtual LibError Load(const char* name, const void* location, u8* fileContents, size_t size) const = 0;
virtual LibError Store(const char* name, const void* location, const u8* fileContents, size_t size) const = 0;
};
typedef boost::shared_ptr<IFileProvider> PIFileProvider;
struct IFilesystem
{
@ -65,7 +83,9 @@ struct IFilesystem
// note: this interface avoids having to lock a directory while an
// iterator is extant.
virtual LibError GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const = 0;
// (don't split this into 2 functions because POSIX can't implement
// that efficiently)
virtual LibError GetDirectoryEntries(const char* path, FileInfos* files, Directories* subdirectories) const = 0;
};
#endif // #ifndef INCLUDED_FILESYSTEM

56
source/lib/file/io/io_manager.cpp
View File

@ -46,34 +46,6 @@
// (apparently since NTFS files are sector-padded anyway?)
// helper routine used by functions that call back to a IoCallback.
//
// bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE }
// note: don't abort if = 0: zip callback may not actually
// output anything if passed very little data.
static LibError InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed)
{
if(cb)
{
stats_cb_start();
LibError ret = cb(cbData, block, size, bytesProcessed);
stats_cb_finish();
// failed - reset byte count in case callback didn't
if(ret != INFO::OK && ret != INFO::CB_CONTINUE)
bytesProcessed = 0;
CHECK_ERR(ret); // user might not have raised a warning; make sure
return ret;
}
// no callback to process data: raw = actual
else
{
bytesProcessed = size;
return INFO::CB_CONTINUE;
}
}
//-----------------------------------------------------------------------------
@ -180,6 +152,34 @@ public:
private:
// helper routine used by functions that call back to a IoCallback.
//
// bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE }
// note: don't abort if = 0: zip callback may not actually
// output anything if passed very little data.
static LibError InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed)
{
if(cb)
{
stats_cb_start();
LibError ret = cb(cbData, block, size, bytesProcessed);
stats_cb_finish();
// failed - reset byte count in case callback didn't
if(ret != INFO::OK && ret != INFO::CB_CONTINUE)
bytesProcessed = 0;
CHECK_ERR(ret); // user might not have raised a warning; make sure
return ret;
}
// no callback to process data: raw = actual
else
{
bytesProcessed = size;
return INFO::CB_CONTINUE;
}
}
void wait(BlockIo& blockIo, u8*& block, size_t& blockSize)
{
LibError ret = blockIo.WaitUntilComplete(block, blockSize);

2
source/lib/file/io/io_manager.h
View File

@ -11,7 +11,7 @@
#ifndef INCLUDED_IO_MANAGER
#define INCLUDED_IO_MANAGER
class File_Posix;
#include "lib/file/posix/io_posix.h" // File_Posix, IoBuf
// called by file_io after a block IO has completed.
// bytesProcessed must be set; file_io will return the sum of these values.

48
source/lib/file/posix/fs_posix.cpp
View File

@ -38,11 +38,11 @@ static bool IsDummyDirectory(const char* name)
return (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'));
}
/*virtual*/ LibError Filesystem_Posix::GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories)
/*virtual*/ LibError Filesystem_Posix::GetDirectoryEntries(const char* path, FileInfos* files, Directories* subdirectories) const
{
// open directory
char osPath[PATH_MAX];
(void)path_MakeAbsolute(path, osPath);
path_MakeAbsolute(path, osPath);
errno = 0;
boost::shared_ptr<DIR> osDir(opendir(osPath), DirDeleter());
if(!osDir.get())
@ -93,7 +93,7 @@ static bool IsDummyDirectory(const char* name)
LibError Filesystem_Posix::GetFileInfo(const char* pathname, FileInfo& fileInfo) const
{
char osPathname[PATH_MAX];
RETURN_ERR(path_MakeAbsolute(pathname, osPathname));
path_MakeAbsolute(pathname, osPathname);
// if path ends in slash, remove it (required by stat)
char* last_char = osPathname+strlen(osPathname)-1;
@ -115,7 +115,7 @@ LibError Filesystem_Posix::GetFileInfo(const char* pathname, FileInfo& fileInfo)
LibError Filesystem_Posix::DeleteFile(const char* pathname)
{
char osPathname[PATH_MAX+1];
RETURN_ERR(path_MakeAbsolute(pathname, osPathname));
path_MakeAbsolute(pathname, osPathname);
errno = 0;
if(unlink(osPathname) != 0)
@ -128,7 +128,7 @@ LibError Filesystem_Posix::DeleteFile(const char* pathname)
LibError Filesystem_Posix::CreateDirectory(const char* path)
{
char osPath[PATH_MAX];
RETURN_ERR(path_MakeAbsolute(path, osPath));
path_MakeAbsolute(path, osPath);
errno = 0;
struct stat s;
@ -149,12 +149,11 @@ LibError Filesystem_Posix::DeleteDirectory(const char* path)
// be deleted (required by Windows and POSIX rmdir()).
char osPath[PATH_MAX];
RETURN_ERR(path_MakeAbsolute(path, osPath));
path_MakeAbsolute(path, osPath);
PathPackage pp;
RETURN_ERR(path_package_set_dir(&pp, osPath));
std::vector<FileInfo> files;
std::vector<const char*> subdirectories;
FileInfos files; Directories subdirectories;
RETURN_ERR(GetDirectoryEntries(path, &files, &subdirectories));
// delete files
@ -180,3 +179,36 @@ LibError Filesystem_Posix::DeleteDirectory(const char* path)
return INFO::OK;
}
//-----------------------------------------------------------------------------
/*virtual*/ unsigned FileProvider_Posix::Precedence() const
{
return 1u;
}
/*virtual*/ char FileProvider_Posix::LocationCode() const
{
return 'F';
}
/*virtual*/ LibError FileProvider_Posix::LoadFile(const char* name, const void* location, u8* fileContents, size_t size) const
{
const char* path = (const char*)location;
const char* pathname = path_append2(path, name);
File_Posix file;
RETURN_ERR(file.Open(pathname, 'r'));
RETURN_ERR(io_Read(file, 0, buf, size));
return INFO::OK;
}
/*virtual*/ LibError FileProvider_Posix::StoreFile(const char* name, const void* location, const u8* fileContents, size_t size) const
{
const char* path = (const char*)location;
const char* pathname = path_append2(path, name);
File_Posix file;
RETURN_ERR(file.Open(pathname, 'r'));
RETURN_ERR(io_Write(file, 0, buf, size));
return INFO::OK;
}

12
source/lib/file/posix/fs_posix.h
View File

@ -14,10 +14,20 @@
#include "lib/file/filesystem.h"
struct FileProvider_Posix : public IFileProvider
{
virtual unsigned Precedence() const;
virtual char LocationCode() const;
virtual LibError Load(const char* name, const void* location, u8* fileContents, size_t size) const;
virtual LibError Store(const char* name, const void* location, const u8* fileContents, size_t size) const;
};
struct Filesystem_Posix : public IFilesystem
{
virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const;
virtual LibError GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
virtual LibError GetDirectoryEntries(const char* path, FileInfos* files, Directories* subdirectories) const;
LibError DeleteFile(const char* pathname);
LibError CreateDirectory(const char* dirPath);

6
source/lib/file/posix/io_posix.cpp
View File

@ -17,6 +17,12 @@
#include "lib/posix/posix_aio.h"
ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES);
ERROR_ASSOCIATE(ERR::DIR_NOT_FOUND, "Directory not found", ENOENT);
ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO);
ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1);
//-----------------------------------------------------------------------------
File_Posix::File_Posix()

2
source/lib/file/posix/io_posix.h
View File

@ -11,7 +11,7 @@
#ifndef INCLUDED_IO_POSIX
#define INCLUDED_IO_POSIX
#include "../io/io_buf.h"
#include "lib/file/io/io_buf.h"
// rationale for using aio instead of mmap:
// - parallelism: instead of just waiting for the transfer to complete,

189
source/lib/file/trace.cpp Normal file
View File

@ -0,0 +1,189 @@
/**
* =========================================================================
* File : trace.cpp
* Project : 0 A.D.
* Description : IO event recording
* =========================================================================
*/
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "trace.h"
#include "lib/allocators/pool.h"
#include "lib/path_util.h" // path_Pool
#include "lib/timer.h" // get_time
#include "lib/nommgr.h" // placement new
/*virtual*/ ITrace::~ITrace()
{
}
//-----------------------------------------------------------------------------
TraceEntry::TraceEntry(EAction action, const char* pathname, size_t size)
: m_timestamp(get_time())
, m_action(action)
, m_pathname(path_Pool()->UniqueCopy(pathname))
, m_size(size)
{
}
#define STRINGIZE(number) #number
TraceEntry::TraceEntry(const char* text)
{
const char* fmt = "%f: %c \"" STRINGIZE(PATH_MAX) "[^\"]\" %d\n";
char pathname[PATH_MAX];
char action;
const int fieldsRead = sscanf_s(text, fmt, &m_timestamp, &m_action, pathname, &m_size);
debug_assert(fieldsRead == 4);
debug_assert(action == 'L' || action == 'S');
m_action = (EAction)action;
m_pathname = path_Pool()->UniqueCopy(pathname);
}
void TraceEntry::EncodeAsText(char* text, size_t maxTextChars) const
{
const char action = m_action;
sprintf_s(text, maxTextChars, "%#010f: %c \"%s\" %d\n", m_timestamp, action, m_pathname, m_size);
}
//-----------------------------------------------------------------------------
class Trace_Dummy : public ITrace
{
public:
Trace_Dummy(size_t UNUSED(maxSize))
{
}
virtual void NotifyLoad(const char* UNUSED(pathname), size_t UNUSED(size))
{
}
virtual void NotifyStore(const char* UNUSED(pathname), size_t UNUSED(size))
{
}
virtual LibError Load(const char* UNUSED(vfsPathname))
{
return INFO::OK;
}
virtual LibError Store(const char* UNUSED(vfsPathname)) const
{
return INFO::OK;
}
virtual const TraceEntry* Entries() const
{
return 0;
}
virtual size_t NumEntries() const
{
return 0;
}
};
//-----------------------------------------------------------------------------
class Trace : public ITrace
{
public:
Trace(size_t maxSize)
{
(void)pool_create(&m_pool, maxSize, sizeof(TraceEntry));
}
virtual ~Trace()
{
(void)pool_destroy(&m_pool);
}
virtual void NotifyLoad(const char* pathname, size_t size)
{
new(Allocate()) TraceEntry(TraceEntry::Load, pathname, size);
}
virtual void NotifyStore(const char* pathname, size_t size)
{
new(Allocate()) TraceEntry(TraceEntry::Store, pathname, size);
}
virtual LibError Load(const char* osPathname)
{
pool_free_all(&m_pool);
errno = 0;
FILE* file = fopen(osPathname, "rt");
if(!file)
return LibError_from_errno();
for(;;)
{
char text[500];
if(!fgets(text, ARRAY_SIZE(text)-1, file))
break;
new(Allocate()) TraceEntry(text);
}
fclose(file);
return INFO::OK;
}
virtual LibError Store(const char* osPathname) const
{
errno = 0;
FILE* file = fopen(osPathname, "at");
if(!file)
return LibError_from_errno();
for(size_t i = 0; i < NumEntries(); i++)
{
char text[500];
Entries()[i].EncodeAsText(text, ARRAY_SIZE(text));
fputs(text, file);
}
(void)fclose(file);
return INFO::OK;
}
virtual const TraceEntry* Entries() const
{
return (const TraceEntry*)m_pool.da.base;
}
virtual size_t NumEntries() const
{
return m_pool.da.pos / sizeof(TraceEntry);
}
private:
void* Allocate()
{
void* p = pool_alloc(&m_pool, 0);
debug_assert(p);
return p;
}
Pool m_pool;
};
PITrace CreateDummyTrace(size_t maxSize)
{
return PITrace(new Trace_Dummy(maxSize));
}
PITrace CreateTrace(size_t maxSize)
{
return PITrace(new Trace(maxSize));
}

110
source/lib/file/trace.h Normal file
View File

@ -0,0 +1,110 @@
/**
* =========================================================================
* File : trace.h
* Project : 0 A.D.
* Description : IO event recording
* =========================================================================
*/
// license: GPL; see lib/license.txt
// traces are useful for determining the optimal ordering of archived files
// and can also serve as a repeatable IO benchmark.
// note: since FileContents are smart pointers, the trace can't easily
// be notified when they are released (relevant for cache simulation).
// we have to assume that users process one file at a time -- as they
// should.
#ifndef INCLUDED_TRACE
#define INCLUDED_TRACE
// stores information about an IO event.
class TraceEntry
{
public:
enum EAction
{
Load = 'L',
Store = 'S',
};
TraceEntry(EAction action, const char* pathname, size_t size);
TraceEntry(const char* textualRepresentation);
EAction Action() const
{
return m_action;
}
const char* Pathname() const
{
return m_pathname;
}
size_t Size() const
{
return m_size;
}
void EncodeAsText(char* text, size_t maxTextChars) const;
private:
// note: keep an eye on the class size because all instances are kept
// in memory (see ITrace)
// time (as returned by get_time) after the operation completes.
// rationale: when loading, the VFS doesn't know file size until
// querying the cache or retrieving file information.
float m_timestamp;
EAction m_action;
const char* m_pathname;
// size of file.
// rationale: other applications using this trace format might not
// have access to the VFS and its file information.
size_t m_size;
};
// note: to avoid interfering with measurements, this trace container
// does not cause any IOs (except of course in Load/Store)
struct ITrace
{
virtual ~ITrace();
virtual void NotifyLoad(const char* pathname, size_t size);
virtual void NotifyStore(const char* pathname, size_t size);
/**
* store all entries into a file.
*
* @param osPathname native (absolute) pathname
*
* note: the file format is text-based to allow human inspection and
* because storing filename strings in a binary format would be a
* bit awkward.
**/
virtual LibError Store(const char* osPathname) const;
/**
* load entries from file.
*
* @param osPathname native (absolute) pathname
*
* replaces any existing entries.
**/
virtual LibError Load(const char* osPathname);
virtual const TraceEntry* Entries() const;
virtual size_t NumEntries() const;
};
typedef boost::shared_ptr<ITrace> PITrace;
extern PITrace CreateDummyTrace(size_t maxSize);
extern PITrace CreateTrace(size_t maxSize);
#endif // #ifndef INCLUDED_TRACE

26
source/lib/file/vfs/file_cache.cpp
View File

@ -11,9 +11,7 @@
#include "precompiled.h"
#include "file_cache.h"
#include "path.h"
#include "file_stats.h"
#include "archive/trace.h"
#include "lib/file/file_stats.h"
#include "lib/cache_adt.h" // Cache
#include "lib/bits.h" // round_up
#include "lib/allocators/allocators.h"
@ -52,8 +50,8 @@ class Allocator;
class Deleter
{
public:
Deleter(size_t size, Allocator* allocator, const char* owner)
: m_size(size), m_allocator(allocator), m_owner(owner)
Deleter(size_t size, Allocator* allocator)
: m_size(size), m_allocator(allocator)
{
}
@ -63,7 +61,6 @@ public:
private:
size_t m_size;
Allocator* m_allocator;
const char* m_owner;
};
// >= sys_max_sector_size or else waio will have to realign.
@ -80,7 +77,7 @@ public:
{
}
FileCacheData Allocate(size_t size, const char* owner)
FileCacheData Allocate(size_t size)
{
const size_t alignedSize = round_up(size, alignment);
stats_buf_alloc(size, alignedSize);
@ -90,10 +87,10 @@ public:
m_checker.notify_alloc((void*)data, alignedSize);
#endif
return FileCacheData(data, Deleter(size, this, owner));
return FileCacheData(data, Deleter(size, this));
}
void Deallocate(const u8* data, size_t size, const char* owner)
void Deallocate(const u8* data, size_t size)
{
void* const p = (void*)data;
const size_t alignedSize = round_up(size, alignment);
@ -108,7 +105,6 @@ public:
m_allocator.Deallocate(p, alignedSize);
stats_buf_free();
trace_notify_free(owner);
}
private:
@ -122,7 +118,7 @@ private:
void Deleter::operator()(const u8* data) const
{
m_allocator->Deallocate(data, m_size, m_owner);
m_allocator->Deallocate(data, m_size);
}
@ -144,7 +140,7 @@ public:
{
}
FileCacheData Reserve(const char* vfsPathname, size_t size)
FileCacheData Reserve(size_t size)
{
// (this probably indicates a bug; caching 0-length files would
// have no benefit, anyway)
@ -154,7 +150,7 @@ public:
// of space in a full cache)
for(;;)
{
FileCacheData data = m_allocator.Allocate(size, vfsPathname);
FileCacheData data = m_allocator.Allocate(size);
if(data.get())
return data;
@ -212,9 +208,9 @@ FileCache::FileCache(size_t size)
{
}
FileCacheData FileCache::Reserve(const char* vfsPathname, size_t size)
FileCacheData FileCache::Reserve(size_t size)
{
return impl.get()->Reserve(vfsPathname, size);
return impl.get()->Reserve(size);
}
void FileCache::Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost)

5
source/lib/file/vfs/file_cache.h
View File

@ -41,15 +41,13 @@ public:
/**
* Reserve a chunk of the cache's memory region.
*
* @param vfsPathname pathname of the file that is to be read; this is
* the key that will be used to Retrieve the file contents.
* @param size required number of bytes (more may be allocated due to
* alignment and/or internal fragmentation)
* @return memory suitably aligned for IO; never fails.
*
* it is expected that this data will be Add()-ed once its IO completes.
**/
FileCacheData Reserve(const char* vfsPathname, size_t size);
FileCacheData Reserve(size_t size);
/**
* Add a file's contents to the cache.
@ -59,6 +57,7 @@ public:
* read-only. if need be and no references are currently attached to it,
* the memory can also be commandeered by Reserve().
*
* @param vfsPathname key that will be used to Retrieve file contents.
* @param cost is the expected cost of retrieving the file again and
* influences how/when it is evicted from the cache.
**/

109
source/lib/file/vfs/real_directory.cpp Normal file
View File

@ -0,0 +1,109 @@
// skip version control directories - this avoids cluttering the
// VFS with hundreds of irrelevant files.
static const char* const svnName = path_Pool()->UniqueCopy(".svn");
//-----------------------------------------------------------------------------
// ArchiveEnumerator
//-----------------------------------------------------------------------------
class ArchiveEnumerator
{
public:
ArchiveEnumerator(VfsDirectory* directory)
: m_startDirectory(directory), m_previousPath(0), m_previousDirectory(0)
{
}
private:
LibError Next(const char* pathname, const ArchiveEntry& archiveEntry)
{
vfs_opt_notify_archived_file(pathname);
const char* name = path_name_only(pathname);
const char* path = path_dir_only2(pathname);
// into which directory should the file be inserted?
VfsDirectory* directory = m_previousDirectory;
if(path != m_previousPath)
{
// (we have to create missing subdirectories because archivers
// don't always place directory entries before their files)
VfsFile* file;
TraverseAndCreate(path, m_startDirectory, directory, file);
debug_assert(file == 0); // should not be a file on the path
m_previousPath = path;
m_previousDirectory = directory;
}
const FileInfo fileInfo(name, archiveEntry.usize, archiveEntry.mtime);
directory->AddFile(fileInfo, pri, );
return INFO::CB_CONTINUE;
}
static LibError Callback(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData)
{
ArchiveEnumerator* archiveEnumerator = (ArchiveEnumerator*)cbData;
return ArchiveEnumerator->Next(pathname, archiveEntry);
}
VfsDirectory* m_startDirectory;
// optimization: looking up each full path is rather slow, so we
// cache the previous directory and use it if the path string
// addresses match.
const char* m_previousPath;
VfsDirectory* m_previousDirectory;
};
static LibError AddArchiveFiles(VfsDirectory* directory, PIArchiveReader archiveReader)
{
ArchiveEnumerator archiveEnumerator(directory);
RETURN_ERR(archiveReader->ReadEntries(&Callback, (uintptr_t)&archiveEnumerator));
}
/**
* search for archives and add their contents into VFS.
*
* @param directory already populated VFS directory into which to
* add the archives' contents.
*
* note: we only support archives in the mount point and not its
* subdirectories. this simplifies Populate() and avoids having to
* check the extension of every single VFS file.
**/
LibError AddArchives(VfsDirectory* directory)
{
std::vector<FileInfo> files;
directory->GetEntries(&files, 0);
for(size_t i = 0; i < files.size(); i++)
{
FileInfo& fileInfo = files[i];
const char* extension = path_extension(fileInfo.Name());
const char* pathname = path_append2(m_path, fileInfo.Name());
PIArchiveReader archiveReader;
if(strcasecmp(extension, "zip") == 0)
archiveReader = CreateArchiveReader_Zip(pathname);
else
continue; // not a (supported) archive file
AddArchiveFiles(archiveReader, directory);
m_archiveReaders.push_back(archiveReader);
}
return INFO::OK;
}
LibError RealDirectory::CreateFile(const char* name)
{
const char* pathname = path_append2(pathname, m_path, name);
File_Posix file;
RETURN_ERR(file.Open(pathname, 'w'));
RETURN_ERR(io_Write(file, 0, buf, size));
}
RETURN_ERR(AddArchives(directory, m_archiveReaders));

34
source/lib/file/vfs/real_directory.h Normal file
View File

@ -0,0 +1,34 @@
class RealDirectory
{
public:
RealDirectory(const char* path, unsigned priority, unsigned flags)
: m_path(path), m_priority(priority), m_flags(flags)
{
}
const char* Path() const
{
return m_path;
}
unsigned Priority() const
{
return m_priority;
}
unsigned Flags() const
{
return m_flags;
}
private:
// note: paths are relative to the root directory, so storing the
// entire path instead of just the portion relative to the mount point
// is not all too wasteful.
const char* m_path;
unsigned m_priority;
unsigned m_flags;
};

102
source/lib/file/vfs/vfs.cpp
View File

@ -13,48 +13,55 @@
#include "lib/path_util.h"
#include "lib/file/file_stats.h"
#include "file_cache.h"
#include "lib/file/trace.h"
#include "lib/file/archive/archive.h"
#include "lib/file/posix/fs_posix.h"
#include "vfs_tree.h"
#include "vfs_path.h"
#include "vfs_mount.h"
#include "vfs_lookup.h"
#include "vfs_populate.h"
#include "file_cache.h"
PIFileProvider provider;
class Filesystem_VFS::Impl : public IFilesystem
{
public:
Impl()
: m_fileCache(ChooseCacheSize())
: m_fileCache(ChooseCacheSize()), m_trace(CreateTrace(4*MiB))
{
}
~Impl()
{
trace_shutdown();
mount_shutdown();
}
LibError Mount(const char* vfsPath, const char* path, uint flags /* = 0 */, uint priority /* = 0 */)
{
// make sure caller didn't forget the required trailing '/'.
debug_assert(path_IsDirectory(vfsPath));
}
// note: we no longer need to check if mounting a subdirectory -
// the new RealDirectory scheme doesn't care.
void Unmount(const char* path)
{
// disallow "." because "./" isn't supported on Windows.
// "./" and "/." are caught by CHECK_PATH.
if(!strcmp(path, "."))
WARN_RETURN(ERR::PATH_NON_CANONICAL);
VfsDirectory* directory = vfs_LookupDirectory(vfsPath, &m_rootDirectory, VFS_LOOKUP_CREATE);
directory->Attach(RealDirectory(path, priority, flags));
return INFO::OK;
}
virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const
{
VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
VfsFile* file = vfs_LookupFile(vfsPathname, &m_rootDirectory);
if(!file)
return ERR::VFS_FILE_NOT_FOUND; // NOWARN
file->GetFileInfo(fileInfo);
return INFO::OK;
}
virtual LibError GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
virtual LibError GetDirectoryEntries(const char* vfsPath, FileInfos* files, Directories* subdirectories) const
{
VfsDirectory* directory = LookupDirectory(vfsPath, &m_tree.Root());
VfsDirectory* directory = vfs_LookupDirectory(vfsPath, &m_rootDirectory);
if(!directory)
WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
directory->GetEntries(files, subdirectories);
@ -65,15 +72,23 @@ public:
// coherency (need only invalidate when closing a FILE_WRITE file).
LibError CreateFile(const char* vfsPathname, const u8* buf, size_t size)
{
VfsDirectory* directory = LookupDirectory(vfsPathname, &m_tree.Root());
VfsDirectory* directory = vfs_LookupDirectory(vfsPathname, &m_rootDirectory);
if(!directory)
WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
const char* name = path_name_only(vfsPathname);
directory->CreateFile(name, buf, size);
const RealDirectory& realDirectory = directory->AttachedDirectories().back();
const char* location = realDirectory.Path();
const VfsFile file(FileInfo(name, (off_t)size, time(0)), realDirectory.Priority(), provider, location);
file.Store(buf, size);
directory->AddFile(file);
// wipe out any cached blocks. this is necessary to cover the (rare) case
// of file cache contents predating the file write.
m_fileCache.Remove(vfsPathname);
m_trace.get()->NotifyStore(vfsPathname, size);
return INFO::OK;
}
// read the entire file.
@ -93,45 +108,60 @@ public:
const bool isCacheHit = m_fileCache.Retrieve(vfsPathname, contents, size);
if(!isCacheHit)
{
VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
VfsFile* file = vfs_LookupFile(vfsPathname, &m_rootDirectory);
if(!file)
WARN_RETURN(ERR::VFS_FILE_NOT_FOUND);
contents = m_fileCache.Reserve(vfsPathname, file->Size());
contents = m_fileCache.Reserve(file->Size());
RETURN_ERR(file->Load((u8*)contents.get()));
m_fileCache.Add(vfsPathname, contents, size);
}
stats_io_user_request(size);
stats_cache(isCacheHit? CR_HIT : CR_MISS, size, vfsPathname);
trace_notify_io(vfsPathname, size);
m_trace.get()->NotifyLoad(vfsPathname, size);
return INFO::OK;
}
void RefreshFileInfo(const char* pathname)
{
//VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
//VfsFile* file = LookupFile(vfsPathname, &m_rootDirectory);
}
// "backs off of" all archives - closes their files and allows them to
// be rewritten or deleted (required by archive builder).
// must call mount_rebuild when done with the rewrite/deletes,
// because this call leaves the VFS in limbo!!
void ReleaseArchives()
{
}
// rebuild the VFS, i.e. re-mount everything. open files are not affected.
// necessary after loose files or directories change, so that the VFS
// "notices" the changes and updates file locations. res calls this after
// dir_watch reports changes; can also be called from the console after a
// rebuild command. there is no provision for updating single VFS dirs -
// it's not worth the trouble.
void Clear()
{
m_rootDirectory.ClearR();
}
void Display()
{
m_tree.Display();
m_rootDirectory.DisplayR(0);
}
private:
static size_t ChooseCacheSize()
{
return 96*MiB;
}
void Rebuild()
{
m_tree.Clear();
m_mounts.RedoAll();
}
VfsTree m_tree;
VfsDirectory m_rootDirectory;
FileCache m_fileCache;
PITrace m_trace;
};
//-----------------------------------------------------------------------------
@ -149,7 +179,7 @@ Filesystem_VFS::Filesystem_VFS(void* trace)
return impl.get()->GetFileInfo(vfsPathname, fileInfo);
}
/*virtual*/ LibError Filesystem_VFS::GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
/*virtual*/ LibError Filesystem_VFS::GetDirectoryEntries(const char* vfsPath, FileInfos* files, Directories* subdirectories) const
{
return impl.get()->GetDirectoryEntries(vfsPath, files, subdirectories);
}
@ -169,11 +199,6 @@ LibError Filesystem_VFS::Mount(const char* vfsPath, const char* path, uint flags
return impl.get()->Mount(vfsPath, path, flags, priority);
}
void Filesystem_VFS::Unmount(const char* path)
{
return impl.get()->Unmount(path);
}
void Filesystem_VFS::RefreshFileInfo(const char* pathname)
{
impl.get()->RefreshFileInfo(pathname);
@ -183,3 +208,8 @@ void Filesystem_VFS::Display() const
{
impl.get()->Display();
}
void Filesystem_VFS::Clear()
{
impl.get()->Clear();
}

29
source/lib/file/vfs/vfs.h
View File

@ -16,16 +16,32 @@
namespace ERR
{
const LibError VFS_DIR_NOT_FOUND = -110100;
const LibError VFS_FILE_NOT_FOUND = -110101;
const LibError VFS_DIR_NOT_FOUND = -110100;
const LibError VFS_FILE_NOT_FOUND = -110101;
const LibError VFS_ALREADY_MOUNTED = -110102;
}
// (recursive mounting and mounting archives are no longer optional since they don't hurt)
enum VfsMountFlags
{
// all real directories mounted during this operation will be watched
// for changes. this flag is provided to avoid watches in output-only
// directories, e.g. screenshots/ (only causes unnecessary overhead).
VFS_WATCH = 1,
// anything mounted from here should be added to archive when
// building via vfs_optimizer.
VFS_ARCHIVABLE = 2
};
typedef boost::shared_ptr<const u8> FileContents;
class Filesystem_VFS : public IFilesystem
{
public:
Filesystem_VFS(void* trace);
~Filesystem_VFS();
/**
* mount a directory into the VFS.
@ -41,15 +57,9 @@ public:
**/
LibError Mount(const char* vfsPath, const char* path, uint flags = 0, uint priority = 0);
/**
* unmount a previously mounted path and rebuild the VFS afterwards.
**/
void Unmount(const char* path);
// (see IFilesystem)
virtual ~Filesystem_VFS();
virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const;
virtual LibError GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
virtual LibError GetDirectoryEntries(const char* vfsPath, FileInfos* files, Directories* subdirectories) const;
// note: only allowing either reads or writes simplifies file cache
// coherency (need only invalidate when closing a FILE_WRITE file).
@ -69,6 +79,7 @@ public:
void RefreshFileInfo(const char* pathname);
void Display() const;
void Clear();
private:
class Impl;

115
source/lib/file/vfs/vfs_lookup.cpp Normal file
View File

@ -0,0 +1,115 @@
/**
* =========================================================================
* File : vfs_lookup.cpp
* Project : 0 A.D.
* Description :
* =========================================================================
*/
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "vfs_lookup.h"
#include "lib/path_util.h" // path_foreach_component
#include "vfs_tree.h"
#include "vfs_populate.h"
#include "vfs.h" // error codes
class PathResolver
{
public:
PathResolver(VfsDirectory* startDirectory, uint flags = 0)
: m_flags(flags), m_currentDirectory(startDirectory)
{
}
static LibError Callback(const char* component, bool isDirectory, uintptr_t cbData)
{
PathResolver* pathResolver = (PathResolver*)cbData;
return pathResolver->Next(component, isDirectory);
}
LibError Next(const char* component, bool isDirectory) const
{
if((m_flags & VFS_LOOKUP_NO_POPULATE ) == 0)
RETURN_ERR(vfs_Populate(m_currentDirectory));
// vfsLookup only sends us pathnames, so all components are directories
debug_assert(isDirectory);
if((m_flags & VFS_LOOKUP_CREATE) != 0)
m_currentDirectory->AddSubdirectory(component);
m_currentDirectory = m_currentDirectory->GetSubdirectory(component);
if(!m_currentDirectory)
WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
return INFO::CB_CONTINUE;
}
VfsDirectory* Directory() const
{
return m_currentDirectory;
}
private:
unsigned m_flags;
mutable VfsDirectory* m_currentDirectory;
};
LibError vfs_Lookup(const char* vfsPathname, VfsDirectory* startDirectory, VfsDirectory*& directory, VfsFile*& file, unsigned flags)
{
const char* vfsPath; const char* name;
path_split(vfsPathname, &vfsPath, &name);
// optimization: looking up each full path is rather slow, so we
// cache the previous directory and use it if the path string
// addresses match.
static const char* vfsPreviousPath;
static VfsDirectory* previousDirectory;
if(vfsPath == vfsPreviousPath)
directory = previousDirectory;
else
{
PathResolver pathResolver(startDirectory, flags);
RETURN_ERR(path_foreach_component(vfsPathname, PathResolver::Callback, (uintptr_t)&pathResolver));
directory = pathResolver.Directory();
}
previousDirectory = directory;
if(name[0] == '\0')
file = 0;
else
{
file = directory->GetFile(name);
if(!file)
WARN_RETURN(ERR::VFS_FILE_NOT_FOUND);
}
return INFO::OK;
}
VfsFile* vfs_LookupFile(const char* vfsPathname, VfsDirectory* startDirectory, unsigned flags)
{
debug_assert(!path_IsDirectory(vfsPathname));
VfsDirectory* directory; VfsFile* file;
if(vfs_Lookup(vfsPathname, startDirectory, directory, file, flags) != INFO::OK)
return 0;
return file;
}
VfsDirectory* vfs_LookupDirectory(const char* vfsPath, VfsDirectory* startDirectory, unsigned flags)
{
debug_assert(path_IsDirectory(vfsPath));
VfsDirectory* directory; VfsFile* file;
if(vfs_Lookup(vfsPath, startDirectory, directory, file, flags) != INFO::OK)
return 0;
return directory;
}

32
source/lib/file/vfs/vfs_path.h → source/lib/file/vfs/vfs_lookup.h
View File

@ -1,6 +1,6 @@
/**
* =========================================================================
* File : vfs_path.h
* File : vfs_lookup.h
* Project : 0 A.D.
* Description :
* =========================================================================
@ -8,8 +8,8 @@
// license: GPL; see lib/license.txt
#ifndef INCLUDED_VFS_PATH
#define INCLUDED_VFS_PATH
#ifndef INCLUDED_VFS_LOOKUP
#define INCLUDED_VFS_LOOKUP
// VFS paths are of the form: "(dir/)*file?"
// in English: '/' as path separator; trailing '/' required for dir names;
@ -32,27 +32,29 @@
class VfsFile;
class VfsDirectory;
enum VfsLookupFlags
{
// when encountering subdirectory components in the path(name) that
// don't (yet) exist, add them.
VFS_LOOKUP_CREATE = 1,
VFS_LOOKUP_NO_POPULATE = 2
};
extern LibError vfs_Lookup(const char* vfsPathname, VfsDirectory* startDirectory, VfsDirectory*& directory, VfsFile*& file, unsigned flags = 0);
/**
* Find a file in the VFS by traversing pathname components.
*
* @param vfsPath must not end with a slash character.
**/
extern VfsFile* LookupFile(const char* vfsPathname, const VfsDirectory* startDirectory);
extern VfsFile* vfs_LookupFile(const char* vfsPathname, const VfsDirectory* startDirectory, unsigned flags = 0);
/**
* Find a directory in the VFS by traversing path components.
*
* @param vfsPath must end with a slash character.
**/
extern VfsDirectory* LookupDirectory(const char* vfsPath, const VfsDirectory* startDirectory);
extern VfsDirectory* vfs_LookupDirectory(const char* vfsPath, const VfsDirectory* startDirectory, unsigned flags = 0);
/**
* Traverse a path, creating subdirectories that didn't yet exist.
*
* @param lastDirectory receives the last directory that was encountered
* @param file references the VFS file represented by vfsPath, or 0 if
* it doesn't include a filename.
**/
extern void TraverseAndCreate(const char* vfsPath, VfsDirectory* startDirectory, VfsDirectory*& lastDirectory, VfsFile*& file);
#endif // #ifndef INCLUDED_VFS_PATH
#endif // #ifndef INCLUDED_VFS_LOOKUP

293
source/lib/file/vfs/vfs_mount.cpp
View File

@ -1,293 +0,0 @@
/**
* =========================================================================
* File : vfs_mount.cpp
* Project : 0 A.D.
* Description : mounts files and archives into VFS
* =========================================================================
*/
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "vfs_mount.h"
#include "lib/file/archive/archive_zip.h"
//-----------------------------------------------------------------------------
// ArchiveEnumerator
//-----------------------------------------------------------------------------
class ArchiveEnumerator
{
public:
ArchiveEnumerator(VfsDirectory* directory)
: m_startDirectory(directory), m_previousPath(0), m_previousDirectory(0)
{
}
LibError Next(const char* pathname, const ArchiveEntry& archiveEntry)
{
vfs_opt_notify_archived_file(pathname);
const char* name = path_name_only(pathname);
const char* path = path_dir_only2(pathname);
// into which directory should the file be inserted?
VfsDirectory* directory = m_previousDirectory;
if(path != m_previousPath)
{
// we have to create them if missing, since we can't rely on the
// archiver placing directories before subdirs or files that
// reference them (WinZip doesn't always).
VfsFile* file;
TraverseAndCreate(path, m_startDirectory, directory, file);
debug_assert(file == 0); // should not be a file on the path
m_previousPath = path;
m_previousDirectory = directory;
}
FileInfo fileInfo;
fileInfo.name = name;
fileInfo.size = archiveEntry.usize;
fileInfo.mtime = archiveEntry.mtime;
directory->AddFile(name, fileInfo);
return INFO::CB_CONTINUE;
}
private:
VfsDirectory* m_startDirectory;
// optimization: looking up each full path is rather slow, so we
// cache the previous directory and use it if the path string
// addresses match.
const char* m_previousPath;
VfsDirectory* m_previousDirectory;
};
static LibError ArchiveEnumeratorCallback(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData)
{
ArchiveEnumerator* archiveEnumerator = (ArchiveEnumerator*)cbData;
return ArchiveEnumerator->Next(pathname, archiveEntry);
}
//-----------------------------------------------------------------------------
// Mount
//-----------------------------------------------------------------------------
// not many instances => don't worry about efficiency.
// note: only check for archives in the root directory of the mounting
// because doing so for every single file would entail serious overhead
class Mount
{
Mount(const char* vfsPath, const char* path, uint flags, uint priority)
: m_vfsPath(vfsPath), m_path(path), m_flags(flags), m_priority(priority)
{
}
const char* VfsPath() const
{
return m_vfsPath;
}
const char* Path() const
{
return m_path;
}
// actually mount the specified entry. split out of vfs_mount,
// because when invalidating (reloading) the VFS, we need to
// be able to mount without changing the mount list.
LibError Mount(VfsDirectory* rootDirectory)
{
VfsDirectory* directory; VfsFile* file;
TraverseAndCreate(m_vfsPath, rootDirectory, directory, file);
debug_assert(file == 0); // should not be a file on the path
directory->AssociateWithRealDirectory(m_path);
// add archives
directory->Populate();
std::vector<FileInfo> files;
directory->GetEntries(&files, 0);
for(size_t i = 0; i < files.size(); i++)
{
FileInfo& fileInfo = files[i];
const char* pathname = path_append2(m_path, fileInfo.Name());
const char* extension = path_extension(fileInfo.Name());
boost::shared_ptr<IArchiveReader> archiveReader;
if(strcasecmp(extension, "zip") == 0)
archiveReader = CreateArchiveReader_Zip(pathname);
else
continue; // not a (supported) archive file
ArchiveEnumerator archiveEnumerator(directory);
RETURN_ERR(archiveReader->ReadEntries(ArchiveEnumeratorCallback, (uintptr_t)&archiveEnumerator));
m_archiveReaders.push_back(archiveReader);
}
}
// "backs off of" all archives - closes their files and allows them to
// be rewritten or deleted (required by archive builder).
// must call mount_rebuild when done with the rewrite/deletes,
// because this call leaves the VFS in limbo!!
void ReleaseArchives()
{
m_archiveReaders.clear();
}
private:
bool IsArchivable() const
{
return (m_flags & MOUNT_ARCHIVABLE) != 0;
}
// mounting into this VFS directory;
// must end in '/' (unless if root td, i.e. "")
const char* m_vfsPath;
const char* m_path;
std::vector<boost::shared_ptr<IArchiveReader> > m_archiveReaders;
uint m_flags; // MountFlags
uint m_priority;
};
//-----------------------------------------------------------------------------
// MountManager
//-----------------------------------------------------------------------------
class MountManager
{
public:
// mount <P_real_path> into the VFS at <V_mount_point>,
// which is created if it does not yet exist.
// files in that directory override the previous VFS contents if
// <pri>(ority) is not lower.
// all archives in <P_real_path> are also mounted, in alphabetical order.
//
// flags determines extra actions to perform; see VfsMountFlags.
//
// P_real_path = "." or "./" isn't allowed - see implementation for rationale.
LibError Add(const char* vfsPath, const char* path, uint flags, uint priority)
{
// make sure caller didn't forget the required trailing '/'.
debug_assert(VFS_PATH_IS_DIR(vfsPath));
// make sure it's not already mounted, i.e. in mounts.
// also prevents mounting a parent directory of a previously mounted
// directory, or vice versa. example: mount $install/data and then
// $install/data/mods/official - mods/official would also be accessible
// from the first mount point - bad.
char existingVfsPath[PATH_MAX];
if(GetVfsPath(path, existingVfsPath))
WARN_RETURN(ERR::ALREADY_MOUNTED);
// disallow "." because "./" isn't supported on Windows.
// "./" and "/." are caught by CHECK_PATH.
if(!strcmp(path, "."))
WARN_RETURN(ERR::PATH_NON_CANONICAL);
// (count this as "init" to obviate a separate timer)
stats_vfs_init_start();
mounts[path] = Mount(vfsPath, path, flags, priority);
LibError ret = remount(m);
stats_vfs_init_finish();
return ret;
}
void Remove(const char* path)
{
const size_t numRemoved = mounts.erase(path);
debug_assert(numRemoved == 1);
}
// rebuild the VFS, i.e. re-mount everything. open files are not affected.
// necessary after loose files or directories change, so that the VFS
// "notices" the changes and updates file locations. res calls this after
// dir_watch reports changes; can also be called from the console after a
// rebuild command. there is no provision for updating single VFS dirs -
// it's not worth the trouble.
void RedoAll()
{
for(MountIt it = mounts.begin(); it != mounts.end(); ++it)
{
const Mount& mount = it->second;
mount.Apply();
}
}
// if <path> or its ancestors are mounted,
// return a VFS path that accesses it.
// used when receiving paths from external code.
bool GetVfsPath(const char* path, char* vfsPath) const
{
for(MountIt it = mounts.begin(); it != mounts.end(); ++it)
{
const Mount& mount = it->second;
const char* remove = mount.Path();
const char* replace = mount.VfsPath();
if(path_replace(vfsPath, path, remove, replace) == INFO::OK)
return true;
}
return false;
}
void ReleaseArchives()
{
// "backs off of" all archives - closes their files and allows them to
// be rewritten or deleted (required by archive builder).
// must call mount_rebuild when done with the rewrite/deletes,
// because this call leaves the VFS in limbo!!
}
private:
typedef std::map<const char*, Mount> Mounts;
typedef Mounts::iterator MountIt;
Mounts mounts;
};
// 2006-05-09 JW note: we are wanting to move XMB files into a separate
// folder tree (no longer interspersed with XML), so that deleting them is
// easier and dirs are less cluttered.
//
// if several mods are active, VFS would have several OS directories mounted
// into a VfsDirectory and could no longer automatically determine the write target.
//
// one solution would be to use a set_write_target API to choose the
// correct dir; however, XMB files may be generated whilst editing
// (which also requires a write_target to write files that are actually
// currently in archives), so we'd need to save/restore write_target.
// this would't be thread-safe => disaster.
//
// a vfs_store_to(filename, flags, N_actual_path) API would work, but it'd
// impose significant burden on users (finding the actual native dir),
// and be prone to abuse. additionally, it would be difficult to
// propagate N_actual_path to VFile_reload where it is needed;
// this would end up messy.
//
// instead, we'll write XMB files into VFS path "mods/$MODNAME/..",
// into which the realdir of the same name (located in some writable folder)
// is mounted; VFS therefore can write without problems.
//
// however, other code (e.g. archive builder) doesn't know about this
// trick - it only sees the flat VFS namespace, which doesn't
// include mods/$MODNAME (that is hidden). to solve this, we also mount
// any active mod's XMB dir into VFS root for read access.
//
// write_targets are no longer necessary since files and directories are
// directly associated with a real directory; when writing files, we can
// do so directly.

37
source/lib/file/vfs/vfs_mount.h
View File

@ -1,37 +0,0 @@
/**
* =========================================================================
* File : vfs_mount.h
* Project : 0 A.D.
* Description : mounts files and archives into VFS; provides x_* API
* : that dispatches to file or archive implementation.
* =========================================================================
*/
// license: GPL; see lib/license.txt
#ifndef INCLUDED_VFS_MOUNT
#define INCLUDED_VFS_MOUNT
namespace ERR
{
const LibError ALREADY_MOUNTED = -110700;
const LibError NOT_MOUNTED = -110701;
const LibError MOUNT_INVALID_TYPE = -110702;
}
// (recursive mounting and mounting archives are no longer optional since they don't hurt)
enum MountFlags
{
// all real directories mounted during this operation will be watched
// for changes. this flag is provided to avoid watches in output-only
// directories, e.g. screenshots/ (only causes unnecessary overhead).
MOUNT_WATCH = 4,
// anything mounted from here should be added to archive when
// building via vfs_optimizer.
MOUNT_ARCHIVABLE = 8
};
#endif // #ifndef INCLUDED_VFS_MOUNT

122
source/lib/file/vfs/vfs_path.cpp
View File

@ -1,122 +0,0 @@
/**
* =========================================================================
* File : vfs_path.cpp
* Project : 0 A.D.
* Description :
* =========================================================================
*/
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "vfs_path.h"
#include "lib/path_util.h" // path_foreach_component
#include "vfs_tree.h"
static bool IsVfsPath(const char* path)
{
if(path[0] == '\0') // root dir
return true;
if(path[strlen(path)-1] == '/') // ends in slash => directory path
return true;
return false;
}
class PathResolver
{
public:
enum Flags
{
// when encountering subdirectory components in the path(name) that
// don't (yet) exist, add them.
CreateMissingDirectories = 1,
};
PathResolver(VfsDirectory* startDirectory, uint flags = 0)
: m_currentDirectory(startDirectory), m_file(0)
, m_createMissingDirectories((flags & CreateMissingDirectories) != 0)
{
}
LibError Next(const char* component, bool isDir) const
{
m_currentDirectory->Populate();
if(isDir)
{
if(m_createMissingDirectories)
m_currentDirectory->AddSubdirectory(component);
m_currentDirectory = m_currentDirectory->GetSubdirectory(component);
if(!m_currentDirectory)
WARN_RETURN(ERR::FAIL);
}
else
{
debug_assert(m_file == 0); // can't have encountered any files yet
m_file = m_currentDirectory->GetFile(component);
if(!m_file)
WARN_RETURN(ERR::FAIL);
}
return INFO::CB_CONTINUE;
}
VfsDirectory* Directory() const
{
return m_currentDirectory;
}
VfsFile* File() const
{
return m_file;
}
private:
mutable VfsDirectory* m_currentDirectory;
mutable VfsFile* m_file;
bool m_createMissingDirectories;
};
static LibError PathResolverCallback(const char* component, bool isDir, uintptr_t cbData)
{
PathResolver* pathResolver = (PathResolver*)cbData;
return pathResolver->Next(component, isDir);
}
VfsFile* LookupFile(const char* vfsPathname, VfsDirectory* startDirectory)
{
debug_assert(!IsVfsPath(vfsPathname));
PathResolver pathResolver(startDirectory);
if(path_foreach_component(vfsPathname, PathResolverCallback, (uintptr_t)&pathResolver) != INFO::OK)
return 0;
return pathResolver.File();
}
VfsDirectory* LookupDirectory(const char* vfsPath, VfsDirectory* startDirectory)
{
debug_assert(IsVfsPath(vfsPath));
PathResolver pathResolver(startDirectory);
if(path_foreach_component(vfsPath, PathResolverCallback, (uintptr_t)&pathResolver) != INFO::OK)
return 0;
return pathResolver.Directory();
}
void TraverseAndCreate(const char* vfsPath, VfsDirectory* startDirectory, VfsDirectory*& lastDirectory, VfsFile*& file)
{
PathResolver pathResolver(startDirectory, PathResolver::CreateMissingDirectories);
LibError ret = path_foreach_component(vfsPath, PathResolverCallback, (uintptr_t)&pathResolver);
debug_assert(ret == INFO::OK); // should never fail
lastDirectory = pathResolver.Directory();
file = pathResolver.File();
}

167
source/lib/file/vfs/vfs_populate.cpp Normal file
View File

@ -0,0 +1,167 @@
/**
* =========================================================================
* File : vfs_populate.cpp
* Project : 0 A.D.
* Description :
* =========================================================================
*/
// license: GPL; see lib/license.txt
#include "precompiled.h"
#include "vfs_populate.h"
#include "lib/path_util.h"
#include "lib/file/path.h"
#include "lib/file/archive/archive_zip.h"
#include "lib/file/posix/fs_posix.h"
#include "vfs_tree.h"
#include "vfs_lookup.h"
#include "vfs.h" // mount flags
typedef void* PIWatch;
static Filesystem_Posix s_fsPosix;
// since directories are never removed except when rebuilding the VFS,
// we can store the IWatch references here.
static std::vector<PIWatch> s_watches;
static std::vector<PIArchiveReader> s_archiveReaders;
static std::vector<const VfsFile*> s_looseFiles;
static size_t s_numArchivedFiles;
//-----------------------------------------------------------------------------
// helper class that allows breaking up the logic into sub-functions without
// always having to pass directory/realDirectory as parameters.
class PopulateHelper : boost::noncopyable
{
public:
PopulateHelper(VfsDirectory* directory, const RealDirectory& realDirectory)
: m_directory(directory)
, m_path(realDirectory.Path()), m_priority(realDirectory.Priority()), m_flags(realDirectory.Flags())
{
}
LibError AddEntries() const
{
FileInfos files; files.reserve(100);
Directories subdirectories; subdirectories.reserve(20);
RETURN_ERR(s_fsPosix.GetDirectoryEntries(m_path, &files, &subdirectories));
RETURN_ERR(AddFiles(files));
AddSubdirectories(subdirectories);
return INFO::OK;
}
private:
void AddFile(const FileInfo& fileInfo) const
{
static PIFileProvider provider(new FileProvider_Posix);
const VfsFile* file = m_directory->AddFile(VfsFile(fileInfo, m_priority, provider, m_path));
// notify archive builder that this file could be archived but
// currently isn't; if there are too many of these, archive will
// be rebuilt.
// note: check if archivable to exclude stuff like screenshots
// from counting towards the threshold.
if(m_flags & VFS_ARCHIVABLE)
s_looseFiles.push_back(file);
}
static void AddArchiveFile(const char* pathname, const FileInfo& fileInfo, const ArchiveEntry* archiveEntry, uintptr_t cbData)
{
PopulateHelper* this_ = (PopulateHelper*)cbData;
// (we have to create missing subdirectories because archivers
// don't always place directory entries before their files)
const unsigned flags = VFS_LOOKUP_CREATE|VFS_LOOKUP_NO_POPULATE;
VfsDirectory* directory = vfs_LookupDirectory(pathname, this_->m_directory, flags);
debug_assert(directory);
directory->AddFile(VfsFile(fileInfo, this_->m_priority, this_->m_archiveReader, archiveEntry));
s_numArchivedFiles++;
}
LibError AddFiles(const FileInfos& files) const
{
PathPackage pp;
path_package_set_dir(&pp, m_path);
for(size_t i = 0; i < files.size(); i++)
{
const char* name = files[i].Name();
path_package_append_file(&pp, name);
PIArchiveReader archiveReader;
const char* extension = path_extension(name);
if(strcasecmp(extension, "zip") == 0)
archiveReader = CreateArchiveReader_Zip(pp.path);
else // not a (supported) archive file
AddFile(files[i]);
RETURN_ERR(archiveReader->ReadEntries(AddArchiveFile, (uintptr_t)this));
s_archiveReaders.push_back(archiveReader);
}
return INFO::OK;
}
void AddSubdirectories(const Directories& subdirectories) const
{
for(size_t i = 0; i < subdirectories.size(); i++)
{
const char* name = subdirectories[i];
// skip version control directories - this avoids cluttering the
// VFS with hundreds of irrelevant files.
static const char* const svnName = path_Pool()->UniqueCopy(".svn");
if(name == svnName)
continue;
VfsDirectory* subdirectory = m_directory->AddSubdirectory(name);
const char* subdirectoryPath = path_append2(m_path, name);
const RealDirectory realSubdirectory(subdirectoryPath, m_priority, m_flags);
subdirectory->Attach(realSubdirectory);
}
}
VfsDirectory* const m_directory;
const char* m_path;
const unsigned m_priority;
const unsigned m_flags;
// used when populating archives
PIArchiveReader m_archiveReader;
};
//-----------------------------------------------------------------------------
LibError vfs_Populate(VfsDirectory* directory)
{
const std::vector<RealDirectory>& realDirectories = directory->AttachedDirectories();
for(size_t i = 0; i < realDirectories.size(); i++)
{
const RealDirectory& realDirectory = realDirectories[i];
// note: we need to do this in each directory because some watch APIs
// (e.g. FAM) cannot register entire directory trees with one call.
if(realDirectory.Flags() & VFS_WATCH)
{
char osPath[PATH_MAX];
(void)path_MakeAbsolute(realDirectory.Path(), osPath);
// s_watches.push_back(CreateWatch(osPath));
}
PopulateHelper helper(directory, realDirectory);
RETURN_ERR(helper.AddEntries());
}
return INFO::OK;
}

3
source/lib/file/vfs/vfs_populate.h Normal file
View File

@ -0,0 +1,3 @@
class VfsDirectory;
extern LibError vfs_Populate(VfsDirectory* directory);

263
source/lib/file/vfs/vfs_tree.cpp
View File

@ -2,7 +2,7 @@
* =========================================================================
* File : vfs_tree.cpp
* Project : 0 A.D.
* Description : the actual 'filesystem' and its tree of directories.
* Description : 'tree' of VFS directories and files
* =========================================================================
*/
@ -11,129 +11,138 @@
#include "precompiled.h"
#include "vfs_tree.h"
#include "lib/file/posix/fs_posix.h"
#include "lib/file/archive/archive.h"
static Filesystem_Posix fsPosix;
static const char* svnName = path_Pool()->UniqueCopy(".svn");
#include "lib/file/file_stats.h"
LibError VfsFile::Load(u8* buf) const
{
if(m_archiveEntry)
RETURN_ERR(m_archiveEntry->archiveReader->LoadFile(*m_archiveEntry, buf));
else
{
const char* pathname = path_append2(m_path, Name());
File_Posix file;
RETURN_ERR(file.Open(pathname, 'r'));
RETURN_ERR(io_Read(file, 0, buf, Size()));
}
//-----------------------------------------------------------------------------
return INFO::OK;
}
VfsDirectory::VfsDirectory(const char* vfsPath, const char* path)
: m_vfsPath(vfsPath), m_path(0)
{
if(path)
AssociateWithRealDirectory(path);
}
VfsDirectory::~VfsDirectory()
VfsFile::VfsFile(const FileInfo& fileInfo, unsigned priority, PIFileProvider provider, const void* location)
: m_fileInfo(fileInfo), m_priority(priority)
, m_provider(provider), m_location(location)
{
}
static bool ShouldReplace(VfsFile& vf1, VfsFile& vf2)
bool VfsFile::IsSupersededBy(const VfsFile& file) const
{
// 1) keep old if new priority is lower.
if(vf2.Priority() < vf1.Priority())
// 1) priority is lower => no.
if(file.m_priority < m_priority)
return false;
// assume they're the same if size and last-modified time match.
// note: FAT timestamp only has 2 second resolution
const double diff = difftime(vf1.MTime(), vf2.MTime());
const bool identical = (vf1.Size() == vf2.Size()) && fabs(diff) <= 2.0;
// 3) go with more efficient source (if files are identical)
//
// since priority is not less, we really ought to always go with m_new.
// however, there is one special case we handle for performance reasons:
// if the file contents are the same, prefer the more efficient source.
// note that priority doesn't automatically take care of this,
// especially if set incorrectly.
if(identical && vf1.Precedence() > vf2.Precedence())
// 2) timestamp is older => no.
// (note: we need to account for FAT's 2 sec. resolution)
if(difftime(file.m_fileInfo.MTime(), m_fileInfo.MTime()) < -2.0)
return false;
// 4) don't replace "old" file if modified more recently than "new".
// (still provide for 2 sec. FAT tolerance - see above)
if(diff > 2.0)
// 3) provider is less efficient => no.
if(file.m_provider.get()->Precedence() < m_provider.get()->Precedence())
return false;
return true;
}
void VfsDirectory::AddFile(const FileInfo& fileInfo)
void VfsFile::GenerateDescription(char* text, size_t maxChars) const
{
std::pair<const char*, VfsFile> value = std::make_pair(fileInfo.Name(), VfsFile(fileInfo));
char timestamp[25];
const time_t mtime = m_fileInfo.MTime();
strftime(timestamp, ARRAY_SIZE(timestamp), "%a %b %d %H:%M:%S %Y", localtime(&mtime));
// build format string (set width of name field so that everything
// lines up correctly)
const char* fmt = "(%c; %6d; %s)\n";
sprintf_s(text, maxChars, fmt, m_provider.get()->LocationCode(), m_fileInfo.Size(), timestamp);
}
LibError VfsFile::Store(const u8* fileContents, size_t size) const
{
RETURN_ERR(m_provider.get()->Store(m_fileInfo.Name(), m_location, fileContents, size));
// update size and mtime
m_fileInfo = FileInfo(m_fileInfo.Name(), (off_t)size, time(0));
return INFO::OK;
}
LibError VfsFile::Load(u8* fileContents) const
{
return m_provider.get()->Load(m_fileInfo.Name(), m_location, fileContents, m_fileInfo.Size());
}
//-----------------------------------------------------------------------------
VfsDirectory::VfsDirectory()
{
}
VfsFile* VfsDirectory::AddFile(const VfsFile& file)
{
std::pair<const char*, VfsFile> value = std::make_pair(file.Name(), file);
std::pair<Files::iterator, bool> ret = m_files.insert(value);
if(!ret.second) // already existed
{
VfsFile& previousFile = ret.first.second;
if(!ShouldReplace(previousFile, value.second))
return INFO::ALREADY_EXISTS;
replace_in_map
VfsFile& previousFile = ret.first->second;
const VfsFile& newFile = value.second;
if(previousFile.IsSupersededBy(newFile))
previousFile = newFile;
}
else
stats_vfs_file_add(file.Size());
stats_vfs_file_add(size);
return &(*ret.first).second;
}
void VfsDirectory::AddSubdirectory(const char* name)
// rationale: passing in a pre-constructed VfsDirectory and copying that into
// our map would be less efficient than this approach.
VfsDirectory* VfsDirectory::AddSubdirectory(const char* name)
{
const char* vfsPath = path_append2(m_vfsPath, name);
std::pair<const char*, VfsDirectory> value = std::make_pair(name, VfsDirectory(vfsPath));
(void)m_subdirectories.insert(value);
std::pair<const char*, VfsDirectory> value = std::make_pair(name, VfsDirectory());
std::pair<Subdirectories::iterator, bool> ret = m_subdirectories.insert(value);
return &(*ret.first).second;
}
VfsFile* VfsDirectory::GetFile(const char* name) const
VfsFile* VfsDirectory::GetFile(const char* name)
{
Files::const_iterator it = m_files.find(name);
Files::iterator it = m_files.find(name);
if(it == m_files.end())
return 0;
return &it->second;
}
VfsDirectory* VfsDirectory::GetSubdirectory(const char* name) const
VfsDirectory* VfsDirectory::GetSubdirectory(const char* name)
{
Subdirectories::const_iterator it = m_subdirectories.find(name);
Subdirectories::iterator it = m_subdirectories.find(name);
if(it == m_subdirectories.end())
return 0;
return &it->second;
}
void VfsDirectory::GetEntries(std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
void VfsDirectory::GetEntries(FileInfos* files, Directories* subdirectories) const
{
if(files)
{
files->reserve(m_files.size());
// (note: VfsFile doesn't return a pointer to FileInfo; instead,
// we have it write directly into the files container)
files->resize(m_files.size());
size_t i = 0;
for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it)
files->push_back(it->second.m_fileInfo);
it->second.GetFileInfo((*files)[i++]);
}
if(subdirectories)
{
subdirectories->clear();
subdirectories->reserve(m_subdirectories.size());
for(Subdirectories::const_iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
subdirectories->push_back(it->second.m_name);
subdirectories->push_back(it->first);
}
}
@ -142,33 +151,32 @@ void VfsDirectory::DisplayR(unsigned depth) const
{
const char indent[] = " ";
// build format string (set width of name field so that everything
// lines up correctly)
char fmt[25];
const int maxNameChars = 80 - depth*(sizeof(indent)-1);
sprintf(fmt, "%%-%d.%ds (%%c; %%6d; %%s)\n", maxNameChars, maxNameChars);
char fmt[20];
sprintf_s(fmt, ARRAY_SIZE(fmt), "%%-%d.%ds %s", maxNameChars, maxNameChars);
for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it)
{
VfsFile& file = it->second;
const char* name = it->first;
const VfsFile& file = it->second;
char timestamp[25];
const time_t mtime = file.MTime();
strftime(timestamp, ARRAY_SIZE(timestamp), "%a %b %d %H:%M:%S %Y", localtime(&mtime));
char description[100];
file.GenerateDescription(description, ARRAY_SIZE(description));
for(int i = 0; i < depth; i++)
for(unsigned i = 0; i < depth; i++)
printf(indent);
printf(fmt, file.Name(), file.LocationCode(), file.Size(), timestamp);
printf(fmt, name, description);
}
for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
for(Subdirectories::const_iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
{
const char* name = it->first;
const VfsDirectory& directory = it->second;
for(int i = 0; i < depth+1; i++)
for(unsigned i = 0; i < depth+1; i++)
printf(indent);
printf("[%s/]\n", name);
directory.DisplayR(depth+1);
}
}
@ -179,96 +187,13 @@ void VfsDirectory::ClearR()
for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
it->second.ClearR();
m_files.Clear();
m_subdirectories.Clear();
m_watch.Unregister();
m_files.clear();
m_subdirectories.clear();
m_attachedDirectories.clear();
}
static const char* AMBIGUOUS = (const char*)(intptr_t)-1;
void VfsDirectory::AssociateWithRealDirectory(const char* path)
void VfsDirectory::Attach(const RealDirectory& realDirectory)
{
// mounting the same directory twice is probably a bug
debug_assert(m_path != path);
if(!m_path)
{
m_path = path;
m_watch.Register(path);
}
else
m_path = AMBIGUOUS;
}
void VfsDirectory::Populate()
{
std::vector<FileInfo> files; files.reserve(100);
std::vector<const char*> subdirectories; subdirectories.reserve(20);
fsPosix.GetDirectoryEntries(m_path, &files, &subdirectories);
for(size_t i = 0; i < files.size(); i++)
{
AddFile(files[i]);
// note: check if archivable to exclude stuff like screenshots
// from counting towards the threshold.
if(m->IsArchivable())
{
// notify archive builder that this file could be archived but
// currently isn't; if there are too many of these, archive will
// be rebuilt.
const char* pathname = path_append2(m_path, files[i].Name());
vfs_opt_notify_loose_file(pathname);
}
}
for(size_t i = 0; i < subdirectories.size(); i++)
{
// skip version control directories - this avoids cluttering the
// VFS with hundreds of irrelevant files.
if(subdirectories[i] == svnName)
continue;
const char* path = path_append2(m_path, subdirectories[i]);
AddSubdirectory(subdirectories[i], path);
}
}
LibError VfsDirectory::CreateFile(const char* name, const u8* buf, size_t size, uint flags = 0)
{
debug_assert(m_path != 0 && m_path != AMBIGUOUS);
const char* pathname = path_append2(pathname, m_path, name);
File_Posix file;
RETURN_ERR(file.Open(pathname, 'w'));
RETURN_ERR(io_Write(file, 0, buf, size));
AddFile(FileInfo(name, size, time()));
return INFO::OK;
}
//-----------------------------------------------------------------------------
VfsTree::VfsTree()
: m_rootDirectory("")
{
}
void VfsTree::Display() const
{
m_rootDirectory.DisplayR();
}
void VfsTree::Clear()
{
m_rootDirectory.ClearR();
m_attachedDirectories.push_back(realDirectory);
}

175
source/lib/file/vfs/vfs_tree.h
View File

@ -2,7 +2,7 @@
* =========================================================================
* File : vfs_tree.h
* Project : 0 A.D.
* Description : the actual 'filesystem' and its tree of directories.
* Description : 'tree' of VFS directories and files
* =========================================================================
*/
@ -11,148 +11,74 @@
#ifndef INCLUDED_VFS_TREE
#define INCLUDED_VFS_TREE
#include "lib/file/filesystem.h" // FileInfo
#include "lib/file/filesystem.h" // FileInfo, PIFileProvider
#include "real_directory.h"
// we add/cancel directory watches from the VFS mount code for convenience -
// it iterates through all subdirectories anyway (*) and provides storage for
// a key to identify the watch (obviates separate TDir -> watch mapping).
//
// define this to strip out that code - removes .watch from struct TDir,
// and calls to res_watch_dir / res_cancel_watch.
//
// *: the add_watch code would need to iterate through subdirs and watch
// each one, because the monitor API (e.g. FAM) may only be able to
// watch single directories, instead of a whole subdirectory tree.
#define NO_DIR_WATCH
class DirectoryWatch
{
public:
DirectoryWatch()
#ifndef NO_DIR_WATCH
: m_watch(0)
#endif
{
}
// 'watch' this directory for changes to support hotloading.
void Register(const char* path)
{
#ifndef NO_DIR_WATCH
char osPath[PATH_MAX];
(void)path_MakeAbsolute(path, osPath);
(void)dir_add_watch(osPath, &m_watch);
#else
UNUSED2(path);
#endif
}
void Unregister()
{
#ifndef NO_DIR_WATCH
if(m_watch) // avoid dir_cancel_watch complaining
{
dir_cancel_watch(m_watch);
m_watch = 0;
}
#endif
}
#ifndef NO_DIR_WATCH
private:
intptr_t m_watch;
#endif
};
//-----------------------------------------------------------------------------
struct ArchiveEntry;
class VfsFile
{
friend class VfsDirectory;
public:
VfsFile(const FileInfo& fileInfo, unsigned priority)
: m_fileInfo(fileInfo), m_priority(priority), m_path(0), m_archiveEntry(0)
{
}
void GetFileInfo(FileInfo& fileInfo) const
{
fileInfo = m_fileInfo;
}
VfsFile(const FileInfo& fileInfo, unsigned priority, PIFileProvider provider, const void* location);
const char* Name() const
{
return m_fileInfo.Name();
}
off_t Size() const
{
return m_fileInfo.Size();
}
time_t MTime() const
void GetFileInfo(FileInfo& fileInfo) const
{
return m_fileInfo.MTime();
fileInfo = m_fileInfo;
}
LibError Load(u8* buf) const;
bool IsSupersededBy(const VfsFile& file) const;
void GenerateDescription(char* text, size_t maxChars) const;
unsigned Priority() const
{
return m_priority;
}
unsigned Precedence() const
{
return m_archiveEntry? 1u : 2u;
}
char LocationCode() const
{
return m_archiveEntry? 'A' : 'F';
}
LibError Store(const u8* fileContents, size_t size) const;
LibError Load(u8* fileContents) const;
private:
FileInfo m_fileInfo;
mutable FileInfo m_fileInfo;
unsigned m_priority;
// location (directory or archive) of the file. this allows opening
// the file in O(1) even when there are multiple mounted directories.
const char* m_path;
ArchiveEntry* m_archiveEntry;
PIFileProvider m_provider;
const void* m_location;
};
//-----------------------------------------------------------------------------
class VfsDirectory
{
public:
VfsDirectory(const char* vfsPath);
~VfsDirectory();
VfsDirectory();
void AddFile(const FileInfo& fileInfo);
void AddSubdirectory(const char* name);
/**
* @return address of existing or newly inserted file; remains
* valid until ClearR is called (i.e. VFS is rebuilt).
**/
VfsFile* AddFile(const VfsFile& file);
VfsFile* GetFile(const char* name) const;
VfsDirectory* GetSubdirectory(const char* name) const;
/**
* @return address of existing or newly inserted subdirectory; remains
* valid until ClearR is called (i.e. VFS is rebuilt).
**/
VfsDirectory* AddSubdirectory(const char* name);
// (don't split this into 2 functions because POSIX can't implement
// that efficiently)
void GetEntries(std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
VfsFile* GetFile(const char* name);
VfsDirectory* GetSubdirectory(const char* name);
void DisplayR() const;
void GetEntries(FileInfos* files, Directories* subdirectories) const;
void DisplayR(unsigned depth) const;
void ClearR();
void AssociateWithRealDirectory(const char* path);
void Populate();
void CreateFile(const char* name, const u8* buf, size_t size, uint flags = 0);
void Attach(const RealDirectory& realDirectory);
const std::vector<RealDirectory>& AttachedDirectories() const
{
return m_attachedDirectories;
}
private:
typedef std::map<const char*, VfsFile> Files;
@ -161,38 +87,7 @@ private:
typedef std::map<const char*, VfsDirectory> Subdirectories;
Subdirectories m_subdirectories;
const char* m_vfsPath;
// if exactly one real directory is mounted into this virtual dir,
// this points to its location. used to add files to VFS when writing.
const char* m_path;
DirectoryWatch m_watch;
};
//-----------------------------------------------------------------------------
class VfsTree
{
public:
VfsTree();
VfsDirectory& Root()
{
return m_rootDirectory;
}
const VfsDirectory& Root() const
{
return m_rootDirectory;
}
void Display() const;
void Clear();
private:
VfsDirectory m_rootDirectory;
std::vector<RealDirectory> m_attachedDirectories;
};
#endif // #ifndef INCLUDED_VFS_TREE