WIP of new file code

archive: pretty much done except for trace and archive_builder
IO: io_manager still needs to be fixed/revised (especially WRT buffer
and alignment management)
posix: done
vfs: need to complete vfs_mount and tie it in to vfs.cpp
dir_util: needs further revisions (VFS interface changed since then)

This was SVN commit r5475.

source/lib/file/archive/archive.cpp

@@ -0,0 +1,23 @@
+/**
+ * =========================================================================
+ * File        : archive.cpp
+ * Project     : 0 A.D.
+ * Description : interface for reading from and creating archives.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "archive.h"
+
+ERROR_ASSOCIATE(ERR::ARCHIVE_UNKNOWN_FORMAT, "Unknown archive format", -1);
+ERROR_ASSOCIATE(ERR::ARCHIVE_UNKNOWN_METHOD, "Unknown compression method", -1);
+
+IArchiveReader::~IArchiveReader()
+{
+}
+
+IArchiveWriter::~IArchiveWriter()
+{
+}

source/lib/file/archive/archive.h

@@ -11,91 +11,98 @@
 #ifndef INCLUDED_ARCHIVE
 #define INCLUDED_ARCHIVE
 
-struct ICodec;
+// rationale: this module doesn't build a directory tree of the entries
+// within an archive. that task is left to the VFS; here, we are only
+// concerned with enumerating all archive entries.
 
 namespace ERR
 {
-	const LibError UNKNOWN_FORMAT = -110400;
+	const LibError ARCHIVE_UNKNOWN_FORMAT = -110400;
-	const LibError IS_COMPRESSED  = -110401;
+	const LibError ARCHIVE_UNKNOWN_METHOD = -110401;
 }
 
-enum ArchiveEntryFlags
+// opaque 'memento' of an archive entry. the instances are stored by
+// the IArchiveReader implementation.
+struct ArchiveEntry;
+
+struct IArchiveReader
 {
-	// (this avoids having to interpret each archive's method values)
+	virtual ~IArchiveReader();
-	AEF_COMPRESSED = 1,
-
-	// indicates ArchiveEntry.ofs points to a "local file header" instead of
-	// the file data. a fixup routine is called upon file open; it skips
-	// past LFH and clears this flag.
-	// this is somewhat of a hack, but vital to archive open performance.
-	// without it, we'd have to scan through the entire archive file,
-	// which can take *seconds*.
-	// (we cannot use the information in CDFH, because its 'extra' field
-	// has been observed to differ from that of the LFH)
-	// by reading LFH when a file in archive is opened, the block cache
-	// absorbs the IO cost because the file will likely be read anyway.
-	AEF_NEEDS_FIXUP = 2
-};
-
-// holds all per-file information extracted from the header.
-// this is intended to work for all archive types.
-struct ArchiveEntry
-{
-	off_t ofs;
-	size_t usize;
-	size_t csize;
-	time_t mtime;
-	u32 checksum;
-	uint method;
-	uint flags;	// ArchiveEntryFlags
-
-	// note that size == usize isn't foolproof, and adding a flag to
-	// ofs or size is ugly and error-prone.
-	bool IsCompressed() const
-	{
-		return (flags & AEF_COMPRESSED) != 0;
-	}
-};
-
-
-// successively called for each archive entry.
-typedef LibError (*ArchiveCB)(const char* pathname, const ArchiveEntry& ae, uintptr_t cbData);
-
-
-struct IArchive
-{
-	IArchive(const char* pathname);
-	virtual ~IArchive();
-
-	virtual boost::shared_ptr<ICodec> CreateDecompressor() const = 0;
 
 	/**
-	 * call back for each file entry in the archive.
+	 * called for each archive entry.
+	 * @param pathname full pathname of entry (unique pointer)
 	 **/
-	virtual LibError ForEachEntry(ArchiveCB cb, uintptr_t cbData) const = 0;
+	typedef void (*ArchiveEntryCallback)(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData);
+	virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) = 0;
 
-	virtual LibError LoadFile(ArchiveEntry& archiveEntry) const = 0;
+	virtual LibError LoadFile(ArchiveEntry& archiveEntry, u8* fileContents) const = 0;
 };
 
-
 // note: when creating an archive, any existing file with the given pathname
 // will be overwritten.
-struct IArchiveBuilder
+
+// rationale: don't support partial adding, i.e. updating archive with
+// only one file. this would require overwriting parts of the Zip archive,
+// which is annoying and slow. also, archives are usually built in
+// seek-optimal order, which would break if we start inserting files.
+// while testing, loose files can be used, so there's no loss.
+
+struct IArchiveWriter
 {
 	/**
 	 * write out the archive to disk; only hereafter is it valid.
 	 **/
-	virtual ~IArchiveBuilder();
+	virtual ~IArchiveWriter();
-
-	virtual boost::shared_ptr<ICodec> CreateCompressor() const = 0;
 
 	/**
 	 * add a file to the archive.
 	 *
-	 * @param fileContents the file data; its compression method is defined by
+	 * rationale: passing in a filename instead of the compressed file
-	 * ae.method and can be CM_NONE.
+	 * contents makes for better encapsulation because callers don't need
+	 * to know about the codec. one disadvantage is that loading the file
+	 * contents can no longer take advantage of the VFS cache nor previously
+	 * archived versions. however, the archive builder usually adds files
+	 * precisely because they aren't in archives, and the cache would
+	 * thrash anyway, so this is deemed acceptable.
 	 **/
-	virtual LibError AddFile(const ArchiveEntry& ae, const u8* fileContents) = 0;
+	virtual LibError AddFile(const char* pathname) = 0;
+};
+
+/**
+ * describes an archive entry (either file or directory).
+ **/
+struct ArchiveEntry
+{
+	ArchiveEntry()
+	{
+	}
+
+	ArchiveEntry(off_t ofs, off_t usize, off_t csize, time_t mtime, u32 checksum, uint method, uint flags)
+	{
+		this->ofs = ofs;
+		this->usize = usize;
+		this->csize = csize;
+		this->mtime = mtime;
+		this->checksum = checksum;
+		this->method = method;
+		this->flags = flags;
+	}
+
+	// this is needed by the VFS and stored here instead of in VfsFile to
+	// yield a nice 32-byte size.
+	IArchiveReader* archiveReader;
+
+	off_t ofs;
+	off_t usize;
+	off_t csize;
+	time_t mtime;
+
+	// the archive implementation establishes the meaning of the following:
+
+	u32 checksum;
+	uint method;
+	uint flags;
 };
 
 #endif	// #ifndef INCLUDED_ARCHIVE

source/lib/file/archive/archive_builder.h

@@ -0,0 +1,26 @@
+/**
+ * =========================================================================
+ * File        : vfs_optimizer.h
+ * Project     : 0 A.D.
+ * Description : automatically bundles files into archives in order of
+ *             : access to optimize I/O.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_VFS_OPTIMIZER
+#define INCLUDED_VFS_OPTIMIZER
+
+
+extern LibError vfs_opt_rebuild_main_archive(const char* trace_filename, const char* archive_fn_fmt);
+
+extern void vfs_opt_auto_build_cancel();
+
+extern int vfs_opt_auto_build(const char* trace_filename,
+	const char* archive_fn_fmt, const char* mini_archive_fn_fmt, bool force_build = false);
+
+extern void vfs_opt_notify_loose_file(const char* atom_fn);
+extern void vfs_opt_notify_archived_file(const char* atom_fn);
+
+#endif	// #ifndef INCLUDED_VFS_OPTIMIZER

source/lib/file/archive/archive_zip.cpp

@@ -0,0 +1,588 @@
+/**
+ * =========================================================================
+ * File        : archive_zip.cpp
+ * Project     : 0 A.D.
+ * Description : archive backend for Zip files.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "archive_zip.h"
+
+#include <time.h>
+#include <limits>
+
+#include "lib/bits.h"
+#include "lib/byte_order.h"
+#include "lib/fat_time.h"
+#include "lib/allocators/pool.h"
+#include "lib/sysdep/cpu.h"		// cpu_memcpy
+#include "lib/file/path.h"
+#include "archive.h"
+#include "codec_zlib.h"
+#include "stream.h"
+#include "lib/file/filesystem.h"
+#include "lib/file/posix/fs_posix.h"
+#include "lib/file/posix/io_posix.h"
+#include "lib/file/io/io_manager.h"
+
+
+enum ArchiveZipFlags
+{
+	// indicates ArchiveEntry.ofs points to a "local file header" instead of
+	// the file data. a fixup routine is called when reading the file;
+	// it skips past the LFH and clears this flag.
+	// this is somewhat of a hack, but vital to archive open performance.
+	// without it, we'd have to scan through the entire archive file,
+	// which can take *seconds*.
+	// (we cannot use the information in CDFH, because its 'extra' field
+	// has been observed to differ from that of the LFH)
+	// since we read the LFH right before the rest of the file, the block
+	// cache will absorb the IO cost.
+	NEEDS_LFH_FIXUP = 1
+};
+
+//-----------------------------------------------------------------------------
+// Zip file data structures and signatures
+//-----------------------------------------------------------------------------
+
+enum ZipMethod
+{
+	ZIP_METHOD_NONE    = 0,
+	ZIP_METHOD_DEFLATE = 8
+};
+
+static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2');
+static const u32  lfh_magic = FOURCC_LE('P','K','\3','\4');
+static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6');
+
+#pragma pack(push, 1)
+
+class LFH
+{
+public:
+	void Init(const ArchiveEntry& archiveEntry, const char* pathname, size_t pathnameLength)
+	{
+		m_magic     = lfh_magic;
+		m_x1        = to_le16(0);
+		m_flags     = to_le16(0);
+		m_method    = to_le16(u16_from_larger(archiveEntry.method));
+		m_fat_mtime = to_le32(FAT_from_time_t(archiveEntry.mtime));
+		m_crc       = to_le32(archiveEntry.checksum);
+		m_csize     = to_le32(u32_from_larger(archiveEntry.csize));
+		m_usize     = to_le32(u32_from_larger(archiveEntry.usize));
+		m_fn_len    = to_le16(u16_from_larger((uint)pathnameLength));
+		m_e_len     = to_le16(0);
+
+		cpu_memcpy((char*)this + sizeof(LFH), pathname, pathnameLength);
+	}
+
+	size_t Size() const
+	{
+		debug_assert(m_magic == lfh_magic);
+		const size_t fn_len = read_le16(&m_fn_len);
+		const size_t  e_len = read_le16(&m_e_len);
+		// note: LFH doesn't have a comment field!
+
+		return sizeof(LFH) + fn_len + e_len;
+	}
+
+private:
+	u32 m_magic;
+	u16 m_x1;			// version needed
+	u16 m_flags;
+	u16 m_method;
+	u32 m_fat_mtime;	// last modified time (DOS FAT format)
+	u32 m_crc;
+	u32 m_csize;
+	u32 m_usize;
+	u16 m_fn_len;
+	u16 m_e_len;
+};
+
+cassert(sizeof(LFH) == 30);
+
+
+class CDFH
+{
+public:
+	void Init(const ArchiveEntry& archiveEntry, const char* pathname, size_t pathnameLength, size_t slack)
+	{
+		m_magic     = cdfh_magic;
+		m_x1        = to_le32(0);
+		m_flags     = to_le16(0);
+		m_method    = to_le16(u16_from_larger(archiveEntry.method));
+		m_fat_mtime = to_le32(FAT_from_time_t(archiveEntry.mtime));
+		m_crc       = to_le32(archiveEntry.checksum);
+		m_csize     = to_le32(u32_from_larger(archiveEntry.csize));
+		m_usize     = to_le32(u32_from_larger(archiveEntry.usize));
+		m_fn_len    = to_le16(u16_from_larger((uint)pathnameLength));
+		m_e_len     = to_le16(0);
+		m_c_len     = to_le16(u16_from_larger((uint)slack));
+		m_x2        = to_le32(0);
+		m_x3        = to_le32(0);
+		m_lfh_ofs   = to_le32(archiveEntry.ofs);
+
+		cpu_memcpy((char*)this + sizeof(CDFH), pathname, pathnameLength);
+	}
+
+	void Decompose(ArchiveEntry& archiveEntry, const char*& pathname, size_t& cdfhSize) const
+	{
+		const u16 zipMethod = read_le16(&m_method);
+		const u32 fat_mtime = read_le32(&m_fat_mtime);
+		const u32 crc       = read_le32(&m_crc);
+		const u32 csize     = read_le32(&m_csize);
+		const u32 usize     = read_le32(&m_usize);
+		const u16 fn_len    = read_le16(&m_fn_len);
+		const u16 e_len     = read_le16(&m_e_len);
+		const u16 c_len     = read_le16(&m_c_len);
+		const u32 lfh_ofs   = read_le32(&m_lfh_ofs);
+
+		archiveEntry.ofs = (off_t)lfh_ofs;
+		archiveEntry.usize = (off_t)usize;
+		archiveEntry.csize = (off_t)csize;
+		archiveEntry.mtime = time_t_from_FAT(fat_mtime);
+		archiveEntry.checksum = crc;
+		archiveEntry.method = (uint)zipMethod;
+		archiveEntry.flags = NEEDS_LFH_FIXUP;
+
+		// return 0-terminated copy of filename
+		const char* fn = (const char*)this + sizeof(CDFH); // not 0-terminated!
+		char buf[PATH_MAX];	// path_Pool()->UniqueCopy requires a 0-terminated string
+		cpu_memcpy(buf, fn, fn_len*sizeof(char));
+		buf[fn_len] = '\0';
+		pathname = path_Pool()->UniqueCopy(buf);
+
+		cdfhSize = sizeof(CDFH) + fn_len + e_len + c_len;
+	}
+
+private:
+	u32 m_magic;
+	u32 m_x1;			// versions
+	u16 m_flags;
+	u16 m_method;
+	u32 m_fat_mtime;	// last modified time (DOS FAT format)
+	u32 m_crc;
+	u32 m_csize;
+	u32 m_usize;
+	u16 m_fn_len;
+	u16 m_e_len;
+	u16 m_c_len;
+	u32 m_x2;			// spanning
+	u32 m_x3;			// attributes
+	u32 m_lfh_ofs;
+};
+
+cassert(sizeof(CDFH) == 46);
+
+
+class ECDR
+{
+public:
+	void Init(uint cd_numEntries, off_t cd_ofs, size_t cd_size)
+	{
+		m_magic         = ecdr_magic;
+		memset(m_x1, 0, sizeof(m_x1));
+		m_cd_numEntries = to_le16(u16_from_larger(cd_numEntries));
+		m_cd_size       = to_le32(u32_from_larger(cd_size));
+		m_cd_ofs        = to_le32(u32_from_larger(cd_ofs));
+		m_comment_len   = to_le16(0);
+	}
+
+	void Decompose(uint& cd_numEntries, off_t& cd_ofs, size_t& cd_size) const
+	{
+		cd_numEntries = (uint)read_le16(&m_cd_numEntries);
+		cd_ofs    = (off_t)read_le32(&m_cd_ofs);
+		cd_size  = (size_t)read_le32(&m_cd_size);
+	}
+
+private:
+	u32 m_magic;
+	u8 m_x1[6];	// multiple-disk support
+	u16 m_cd_numEntries;
+	u32 m_cd_size;
+	u32 m_cd_ofs;
+	u16 m_comment_len;
+};
+
+cassert(sizeof(ECDR) == 22);
+
+#pragma pack(pop)
+
+
+//-----------------------------------------------------------------------------
+
+/**
+ * scan buffer for a Zip file record.
+ *
+ * @param start position within buffer
+ * @param magic signature of record
+ * @param recordSize size of record (including signature)
+ * @return pointer to record within buffer or 0 if not found.
+ **/
+static const u8* zip_FindRecord(const u8* buf, size_t size, const u8* start, u32 magic, size_t recordSize)
+{
+	// (don't use <start> as the counter - otherwise we can't tell if
+	// scanning within the buffer was necessary.)
+	for(const u8* p = start; p <= buf+size-recordSize; p++)
+	{
+		// found it
+		if(*(u32*)p == magic)
+		{
+			debug_assert(p == start);	// otherwise, the archive is a bit broken
+			return p;
+		}
+	}
+
+	// passed EOF, didn't find it.
+	// note: do not warn - this happens in the initial ECDR search at
+	// EOF if the archive contains a comment field.
+	return 0;
+}
+
+
+// search for ECDR in the last <maxScanSize> bytes of the file.
+// if found, fill <dst_ecdr> with a copy of the (little-endian) ECDR and
+// return INFO::OK, otherwise IO error or ERR::CORRUPTED.
+static LibError zip_ReadECDR(const File_Posix& file, off_t fileSize, IoBuf& buf, size_t maxScanSize, uint& cd_numEntries, off_t& cd_ofs, size_t& cd_size)
+{
+	// don't scan more than the entire file
+	const size_t scanSize = std::min(maxScanSize, (size_t)fileSize);
+
+	// read desired chunk of file into memory
+	const off_t ofs = fileSize - (off_t)scanSize;
+	RETURN_ERR(io_Read(file, ofs, buf, scanSize));
+
+	// look for ECDR in buffer
+	const u8* start = (const u8*)buf.get();
+	const ECDR* ecdr = (const ECDR*)zip_FindRecord(start, scanSize, start, ecdr_magic, sizeof(ECDR));
+	if(!ecdr)
+		WARN_RETURN(INFO::CANNOT_HANDLE);
+
+	ecdr->Decompose(cd_numEntries, cd_ofs, cd_size);
+	return INFO::OK;
+}
+
+
+static LibError zip_LocateCentralDirectory(const File_Posix& file, off_t fileSize, off_t& cd_ofs, uint& cd_numEntries, size_t& cd_size)
+{
+	const size_t maxScanSize = 66000u;	// see below
+	IoBuf buf = io_buf_Allocate(maxScanSize);
+
+	// expected case: ECDR at EOF; no file comment
+	LibError ret = zip_ReadECDR(file, fileSize, buf, sizeof(ECDR), cd_numEntries, cd_ofs, cd_size);
+	if(ret == INFO::OK)
+		return INFO::OK;
+	// worst case: ECDR precedes 64 KiB of file comment
+	ret = zip_ReadECDR(file, fileSize, buf, maxScanSize, cd_numEntries, cd_ofs, cd_size);
+	if(ret == INFO::OK)
+		return INFO::OK;
+
+	// both ECDR scans failed - this is not a valid Zip file.
+	RETURN_ERR(io_Read(file, 0, buf, sizeof(LFH)));
+	// the Zip file has an LFH but lacks an ECDR. this can happen if
+	// the user hard-exits while an archive is being written.
+	// notes:
+	// - return ERR::CORRUPTED so VFS will not include this file.
+	// - we could work around this by scanning all LFHs, but won't bother
+	//   because it'd be slow.
+	// - do not warn - the corrupt archive will be deleted on next
+	//   successful archive builder run anyway.
+	if(zip_FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH)))
+		return ERR::CORRUPTED;	// NOWARN
+	// totally bogus
+	else
+		WARN_RETURN(ERR::ARCHIVE_UNKNOWN_FORMAT);
+}
+
+
+//-----------------------------------------------------------------------------
+// ArchiveReader_Zip
+//-----------------------------------------------------------------------------
+
+static Filesystem_Posix fsPosix;
+
+class ArchiveReader_Zip : public IArchiveReader
+{
+public:
+	ArchiveReader_Zip(const char* pathname)
+	{
+		m_file.Open(pathname, 'r');
+		
+		FileInfo fileInfo;
+		fsPosix.GetFileInfo(pathname, fileInfo);
+		m_fileSize = fileInfo.Size();
+		debug_assert(m_fileSize >= sizeof(LFH)+sizeof(CDFH)+sizeof(ECDR));
+	}
+
+	virtual LibError ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData)
+	{
+		// locate and read Central Directory
+		off_t cd_ofs; uint cd_numEntries; size_t cd_size;
+		RETURN_ERR(zip_LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size));
+		IoBuf buf = io_buf_Allocate(cd_size);
+		RETURN_ERR(io_Read(m_file, cd_ofs, buf, cd_size));
+		const u8* cd = (const u8*)buf.get();
+
+		m_entries.reserve(cd_numEntries);
+
+		// iterate over Central Directory
+		const u8* pos = cd;
+		for(uint i = 0; i < cd_numEntries; i++)
+		{
+			// scan for next CDFH
+			CDFH* cdfh = (CDFH*)zip_FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH));
+			if(!cdfh)
+				WARN_RETURN(ERR::CORRUPTED);
+
+			ArchiveEntry archiveEntry;
+			const char* pathname;
+			size_t cdfhSize;
+			cdfh->Decompose(archiveEntry, pathname, cdfhSize);
+
+			m_entries.push_back(archiveEntry);
+			cb(pathname, m_entries.back(), cbData);
+
+			pos += cdfhSize;
+		}
+
+		return INFO::OK;
+	}
+
+	virtual LibError LoadFile(ArchiveEntry& archiveEntry, u8* fileContents) const
+	{
+		if((archiveEntry.flags & NEEDS_LFH_FIXUP) != 0)
+		{
+			FixupEntry(archiveEntry);
+			archiveEntry.flags &= ~NEEDS_LFH_FIXUP;
+		}
+
+		boost::shared_ptr<ICodec> codec;
+		switch(archiveEntry.method)
+		{
+		case ZIP_METHOD_NONE:
+			codec = CreateCodec_ZLibNone();
+			break;
+		case ZIP_METHOD_DEFLATE:
+			codec = CreateDecompressor_ZLibDeflate();
+			break;
+		default:
+			WARN_RETURN(ERR::ARCHIVE_UNKNOWN_METHOD);
+		}
+
+		Stream stream(codec);
+		stream.SetOutputBuffer(fileContents, archiveEntry.usize);
+		RETURN_ERR(io_Read(m_file, archiveEntry.ofs, IO_BUF_TEMP, archiveEntry.csize, FeedStream, (uintptr_t)&stream));
+		RETURN_ERR(stream.Finish());
+		debug_assert(archiveEntry.checksum == stream.Checksum());
+
+		return INFO::OK;
+	}
+
+private:
+	struct LFH_Copier
+	{
+		u8* lfh_dst;
+		size_t lfh_bytes_remaining;
+	};
+
+	// this code grabs an LFH struct from file block(s) that are
+	// passed to the callback. usually, one call copies the whole thing,
+	// but the LFH may straddle a block boundary.
+	//
+	// rationale: this allows using temp buffers for zip_fixup_lfh,
+	// which avoids involving the file buffer manager and thus
+	// avoids cluttering the trace and cache contents.
+	static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size, size_t& bytesProcessed)
+	{
+		LFH_Copier* p = (LFH_Copier*)cbData;
+
+		debug_assert(size <= p->lfh_bytes_remaining);
+		cpu_memcpy(p->lfh_dst, block, size);
+		p->lfh_dst += size;
+		p->lfh_bytes_remaining -= size;
+
+		bytesProcessed = size;
+		return INFO::CB_CONTINUE;
+	}
+
+	/**
+	 * fix up archiveEntry's offset within the archive.
+	 * called by archive_LoadFile iff NEEDS_LFH_FIXUP is set.
+	 * side effects: adjusts archiveEntry.ofs.
+	 *
+	 * ensures <ent.ofs> points to the actual file contents; it is initially
+	 * the offset of the LFH. we cannot use CDFH filename and extra field
+	 * lengths to skip past LFH since that may not mirror CDFH (has happened).
+	 *
+	 * this is called at file-open time instead of while mounting to
+	 * reduce seeks: since reading the file will typically follow, the
+	 * block cache entirely absorbs the IO cost.
+	 **/
+	void FixupEntry(ArchiveEntry& archiveEntry) const
+	{
+		// performance note: this ends up reading one file block, which is
+		// only in the block cache if the file starts in the same block as a
+		// previously read file (i.e. both are small).
+		LFH lfh;
+		LFH_Copier params = { (u8*)&lfh, sizeof(LFH) };
+		THROW_ERR(io_Read(m_file, archiveEntry.ofs, IO_BUF_TEMP, sizeof(LFH), lfh_copier_cb, (uintptr_t)&params));
+
+		archiveEntry.ofs += (off_t)lfh.Size();
+	}
+
+	File_Posix m_file;
+	off_t m_fileSize;
+	std::vector<ArchiveEntry> m_entries;
+};
+
+
+boost::shared_ptr<IArchiveReader> CreateArchiveReader_Zip(const char* archivePathname)
+{
+	return boost::shared_ptr<IArchiveReader>(new ArchiveReader_Zip(archivePathname));
+}
+
+
+//-----------------------------------------------------------------------------
+// ArchiveWriterZip
+//-----------------------------------------------------------------------------
+
+static bool IsFileTypeIncompressible(const char* extension)
+{
+	// file extensions that we don't want to compress
+	static const char* incompressibleExtensions[] =
+	{
+		"zip", "rar",
+		"jpg", "jpeg", "png", 
+		"ogg", "mp3"
+	};
+
+	for(uint i = 0; i < ARRAY_SIZE(incompressibleExtensions); i++)
+	{
+		if(!strcasecmp(extension, incompressibleExtensions[i]))
+			return true;
+	}
+
+	return false;
+}
+
+
+class ArchiveWriter_Zip : public IArchiveWriter
+{
+public:
+	ArchiveWriter_Zip(const char* archivePathname)
+		: m_fileSize(0)
+		, m_numEntries(0)
+	{
+		THROW_ERR(m_file.Open(archivePathname, 'w'));
+		THROW_ERR(pool_create(&m_cdfhPool, 10*MiB, 0));
+	}
+
+	~ArchiveWriter_Zip()
+	{
+		const size_t cd_size = m_cdfhPool.da.pos;
+
+		// append an ECDR to the CDFH list (this allows us to
+		// write out both to the archive file in one burst)
+		ECDR* ecdr = (ECDR*)pool_alloc(&m_cdfhPool, sizeof(ECDR));
+		if(ecdr)
+		{
+			ecdr->Init(m_numEntries, m_fileSize, cd_size);
+			io_Write(m_file, m_fileSize, m_cdfhPool.da.base, cd_size+sizeof(ECDR));
+		}
+
+		(void)pool_destroy(&m_cdfhPool);
+	}
+
+	LibError AddFile(const char* pathname)
+	{
+		FileInfo fileInfo;
+		RETURN_ERR(fsPosix.GetFileInfo(pathname, fileInfo));
+		const off_t usize = fileInfo.Size();
+		// skip 0-length files.
+		// rationale: zip.cpp needs to determine whether a CDFH entry is
+		// a file or directory (the latter are written by some programs but
+		// not needed - they'd only pollute the file table).
+		// it looks like checking for usize=csize=0 is the safest way -
+		// relying on file attributes (which are system-dependent!) is
+		// even less safe.
+		// we thus skip 0-length files to avoid confusing them with directories.
+		if(!usize)
+			return INFO::SKIPPED;
+
+		File_Posix file;
+		RETURN_ERR(file.Open(pathname, 'r'));
+
+		// choose method and the corresponding codec
+		ZipMethod zipMethod;
+		boost::shared_ptr<ICodec> codec;
+		if(IsFileTypeIncompressible(pathname))
+		{
+			zipMethod = ZIP_METHOD_NONE;
+			codec = CreateCodec_ZLibNone();
+		}
+		else
+		{
+			zipMethod = ZIP_METHOD_DEFLATE;
+			codec = CreateCompressor_ZLibDeflate();
+		}
+
+		// allocate memory
+		const size_t pathnameLength = strlen(pathname);
+		const size_t csizeMax = codec.get()->MaxOutputSize(usize);
+		IoBuf buf = io_buf_Allocate(sizeof(LFH) + pathnameLength + csizeMax);
+
+		// read and compress file contents
+		size_t csize; u32 checksum;
+		{
+			u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength;
+			Stream stream(codec);
+			stream.SetOutputBuffer(cdata, csizeMax);
+			RETURN_ERR(io_Read(file, 0, IO_BUF_TEMP, usize, FeedStream, (uintptr_t)&stream));
+			RETURN_ERR(stream.Finish());
+			csize = stream.OutSize();
+			checksum = stream.Checksum();
+		}
+
+		ArchiveEntry archiveEntry(m_fileSize, usize, (off_t)csize, fileInfo.MTime(), checksum, zipMethod, 0);
+
+		// build LFH
+		{
+			LFH* lfh = (LFH*)buf.get();
+			lfh->Init(archiveEntry, pathname, pathnameLength);
+		}
+
+		// write to LFH, pathname and cdata to file
+		const size_t packageSize = sizeof(LFH) + pathnameLength + csize;
+		RETURN_ERR(io_Write(m_file, m_fileSize, buf, packageSize));
+		m_fileSize += (off_t)packageSize;
+
+		// append a CDFH to the central dir (in memory)
+		// .. note: pool_alloc may round size up for padding purposes.
+		const size_t prev_pos = m_cdfhPool.da.pos;
+		const size_t cdfhSize = sizeof(CDFH) + pathnameLength;
+		CDFH* cdfh = (CDFH*)pool_alloc(&m_cdfhPool, cdfhSize);
+		if(!cdfh)
+			WARN_RETURN(ERR::NO_MEM);
+		const size_t slack = m_cdfhPool.da.pos - prev_pos - cdfhSize;
+		cdfh->Init(archiveEntry, pathname, pathnameLength, slack);
+
+		m_numEntries++;
+
+		return INFO::OK;
+	}
+
+private:
+	File_Posix m_file;
+	off_t m_fileSize;
+
+	Pool m_cdfhPool;
+	uint m_numEntries;
+};
+
+boost::shared_ptr<IArchiveWriter> CreateArchiveWriter_Zip(const char* archivePathname)
+{
+	return boost::shared_ptr<IArchiveWriter>(new ArchiveWriter_Zip(archivePathname));
+}

source/lib/file/archive/archive_zip.h

@@ -0,0 +1,20 @@
+/**
+ * =========================================================================
+ * File        : archive_zip.h
+ * Project     : 0 A.D.
+ * Description : archive backend for Zip files.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_ARCHIVE_ZIP
+#define INCLUDED_ARCHIVE_ZIP
+
+struct IArchiveReader;
+boost::shared_ptr<IArchiveReader> CreateArchiveReader_Zip(const char* archivePathname);
+
+struct IArchiveWriter;
+extern boost::shared_ptr<IArchiveWriter> CreateArchiveWriter_Zip(const char* archivePathname);
+
+#endif	// #ifndef INCLUDED_ARCHIVE_ZIP

source/lib/file/archive/codec.cpp

@@ -0,0 +1,16 @@
+/**
+ * =========================================================================
+ * File        : codec.cpp
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "codec.h"
+
+ICodec::~ICodec()
+{
+}

source/lib/file/archive/codec.h

@@ -0,0 +1,70 @@
+/**
+ * =========================================================================
+ * File        : codec.h
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_CODEC
+#define INCLUDED_CODEC
+
+// rationale: this layer allows for other compression methods/libraries
+// besides ZLib. it also simplifies the interface for user code and
+// does error checking, etc.
+
+struct ICodec
+{
+public:
+	/**
+	 * note: the implementation should not check whether any data remains -
+	 * codecs are sometimes destroyed without completing a transfer.
+	 **/
+	virtual ~ICodec();
+
+	/**
+	 * @return an upper bound on the output size for the given amount of input.
+	 * this is used when allocating a single buffer for the whole operation.
+	 **/
+	virtual size_t MaxOutputSize(size_t inSize) const = 0;
+
+	/**
+	 * clear all previous state and prepare for reuse.
+	 *
+	 * this is as if the object were destroyed and re-created, but more
+	 * efficient since it avoids reallocating a considerable amount of
+	 * memory (about 200KB for LZ).
+	 **/
+	virtual LibError Reset() = 0;
+
+	/**
+	 * process (i.e. compress or decompress) data.
+	 *
+	 * @param outSize bytes remaining in the output buffer; shall not be zero.
+	 * @param inConsumed, outProduced how many bytes in the input and
+	 * output buffers were used. either or both of these can be zero if
+	 * the input size is small or there's not enough output space.
+	 **/
+	virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) = 0;
+
+	/**
+	 * flush buffers and make sure all output has been produced.
+	 *
+	 * @param checksum over all input data.
+	 * @return error status for the entire operation.
+	 **/
+	virtual LibError Finish(u32& checksum) = 0;
+
+	/**
+	 * update a checksum to reflect the contents of a buffer.
+	 *
+	 * @param checksum the initial value (must be 0 on first call)
+	 * @return the new checksum. note: after all data has been seen, this is
+	 * identical to the what Finish would return.
+	 **/
+	virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const = 0;
+};
+
+#endif	// #ifndef INCLUDED_CODEC

source/lib/file/archive/codec_zlib.cpp

@@ -0,0 +1,273 @@
+#include "precompiled.h"
+#include "codec_zlib.h"
+
+#include "codec.h"
+#include "lib/external_libraries/zlib.h"
+
+#include "lib/sysdep/cpu.h"
+
+
+class Codec_ZLib : public ICodec
+{
+public:
+	u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const
+	{
+		return (u32)crc32(checksum, in, (uInt)inSize);
+	}
+
+protected:
+	u32 InitializeChecksum()
+	{
+		return crc32(0, 0, 0);
+	}
+};
+
+
+//-----------------------------------------------------------------------------
+
+class Codec_ZLibNone : public Codec_ZLib
+{
+public:
+	Codec_ZLibNone()
+	{
+		Reset();
+	}
+
+	virtual ~Codec_ZLibNone()
+	{
+	}
+
+	virtual size_t MaxOutputSize(size_t inSize) const
+	{
+		return inSize;
+	}
+
+	virtual LibError Reset()
+	{
+		m_checksum = InitializeChecksum();
+		return INFO::OK;
+	}
+
+	virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed)
+	{
+		const size_t transferSize = std::min(inSize, outSize);
+		cpu_memcpy(out, in, transferSize);
+		inConsumed = outConsumed = transferSize;
+		m_checksum = UpdateChecksum(m_checksum, in, inSize);
+		return INFO::OK;
+	}
+
+	virtual LibError Finish(u32& checksum)
+	{
+		checksum = m_checksum;
+		return INFO::OK;
+	}
+
+private:
+	u32 m_checksum;
+};
+
+
+//-----------------------------------------------------------------------------
+
+class CodecZLibStream : public Codec_ZLib
+{
+protected:
+	CodecZLibStream()
+	{
+		memset(&m_zs, 0, sizeof(m_zs));
+		m_checksum = InitializeChecksum();
+	}
+
+	static LibError LibError_from_zlib(int zlib_ret)
+	{
+		switch(zlib_ret)
+		{
+		case Z_OK:
+			return INFO::OK;
+		case Z_STREAM_END:
+			WARN_RETURN(ERR::FAIL);
+		case Z_MEM_ERROR:
+			WARN_RETURN(ERR::NO_MEM);
+		case Z_DATA_ERROR:
+			WARN_RETURN(ERR::CORRUPTED);
+		case Z_STREAM_ERROR:
+			WARN_RETURN(ERR::INVALID_PARAM);
+		default:
+			WARN_RETURN(ERR::FAIL);
+		}
+	}
+
+	static void WarnIfZLibError(int zlib_ret)
+	{
+		(void)LibError_from_zlib(zlib_ret);
+	}
+
+	typedef int ZEXPORT (*ZLibFunc)(z_streamp strm, int flush);
+
+	LibError Process(ZLibFunc func, int flush, const u8* in, const size_t inSize, u8* out, const size_t outSize, size_t& inConsumed, size_t& outConsumed)
+	{
+		m_zs.next_in  = (Byte*)in;
+		m_zs.avail_in = (uInt)inSize;
+		m_zs.next_out  = (Byte*)out;
+		m_zs.avail_out = (uInt)outSize;
+
+		int ret = func(&m_zs, flush);
+		// sanity check: if ZLib reports end of stream, all input data
+		// must have been consumed.
+		if(ret == Z_STREAM_END)
+		{
+			debug_assert(m_zs.avail_in == 0);
+			ret = Z_OK;
+		}
+
+		debug_assert(inSize >= m_zs.avail_in && outSize >= m_zs.avail_out);
+		inConsumed  = inSize  - m_zs.avail_in;
+		outConsumed = outSize - m_zs.avail_out;
+
+		return LibError_from_zlib(ret);
+	}
+
+	mutable z_stream m_zs;
+
+	// note: z_stream does contain an 'adler' checksum field, but that's
+	// not updated in streams lacking a gzip header, so we'll have to
+	// calculate a checksum ourselves.
+	// adler32 is somewhat weaker than CRC32, but a more important argument
+	// is that we should use the latter for compatibility with Zip archives.
+	mutable u32 m_checksum;
+};
+
+
+//-----------------------------------------------------------------------------
+
+class Compressor_ZLib : public CodecZLibStream
+{
+public:
+	Compressor_ZLib()
+	{
+		// note: with Z_BEST_COMPRESSION, 78% percent of
+		// archive builder CPU time is spent in ZLib, even though
+		// that is interleaved with IO; everything else is negligible.
+		// we therefore enable this only in final builds; during
+		// development, 1.5% bigger archives are definitely worth much
+		// faster build time.
+#if CONFIG_FINAL
+		const int level      = Z_BEST_COMPRESSION;
+#else
+		const int level      = Z_BEST_SPEED;
+#endif
+		const int windowBits = -MAX_WBITS;	// max window size; omit ZLib header
+		const int memLevel   = 9;					// max speed; total mem ~= 384KiB
+		const int strategy   = Z_DEFAULT_STRATEGY;	// normal data - not RLE
+		const int ret = deflateInit2(&m_zs, level, Z_DEFLATED, windowBits, memLevel, strategy);
+		debug_assert(ret == Z_OK);
+	}
+
+	virtual ~Compressor_ZLib()
+	{
+		const int ret = deflateEnd(&m_zs);
+		WarnIfZLibError(ret);
+	}
+
+	virtual size_t MaxOutputSize(size_t inSize) const
+	{
+		return (size_t)deflateBound(&m_zs, (uLong)inSize);
+	}
+
+	virtual LibError Reset()
+	{
+		m_checksum = InitializeChecksum();
+		const int ret = deflateReset(&m_zs);
+		return LibError_from_zlib(ret);
+	}
+
+	virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed)
+	{
+		m_checksum = UpdateChecksum(m_checksum, in, inSize);
+		return CodecZLibStream::Process(deflate, 0, in, inSize, out, outSize, inConsumed, outConsumed);
+	}
+
+	virtual LibError Finish(u32& checksum)
+	{
+		// notify zlib that no more data is forthcoming and have it flush output.
+		// our output buffer has enough space due to use of deflateBound;
+		// therefore, deflate must return Z_STREAM_END.
+		const int ret = deflate(&m_zs, Z_FINISH);
+		debug_assert(ret == Z_STREAM_END);
+
+		checksum = m_checksum;
+		return INFO::OK;
+	}
+};
+
+
+//-----------------------------------------------------------------------------
+
+class Decompressor_ZLib : public CodecZLibStream
+{
+public:
+	Decompressor_ZLib()
+	{
+		const int windowBits = -MAX_WBITS;	// max window size; omit ZLib header
+		const int ret = inflateInit2(&m_zs, windowBits);
+		debug_assert(ret == Z_OK);
+	}
+
+	virtual ~Decompressor_ZLib()
+	{
+		const int ret = inflateEnd(&m_zs);
+		WarnIfZLibError(ret);
+	}
+
+	virtual size_t MaxOutputSize(size_t inSize) const
+	{
+		// relying on an upper bound for the output is a really bad idea for
+		// large files. archive formats store the uncompressed file sizes,
+		// so callers should use that when allocating the output buffer.
+		debug_assert(inSize < 1*MiB);
+
+		// http://www.zlib.org/zlib_tech.html
+		return inSize*1032;
+	}
+
+	virtual LibError Reset()
+	{
+		m_checksum = InitializeChecksum();
+		const int ret = inflateReset(&m_zs);
+		return LibError_from_zlib(ret);
+	}
+
+	virtual LibError Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outConsumed)
+	{
+		const LibError ret = CodecZLibStream::Process(inflate, Z_SYNC_FLUSH, in, inSize, out, outSize, inConsumed, outConsumed);
+		m_checksum = UpdateChecksum(m_checksum, in, inSize);
+		return ret;
+	}
+
+	virtual LibError Finish(u32& checksum)
+	{
+		// no action needed - decompression always flushes immediately.
+
+		checksum = m_checksum;
+		return INFO::OK;
+	}
+};
+
+
+//-----------------------------------------------------------------------------
+
+boost::shared_ptr<ICodec> CreateCodec_ZLibNone()
+{
+	return boost::shared_ptr<ICodec>(new Codec_ZLibNone);
+}
+
+boost::shared_ptr<ICodec> CreateCompressor_ZLibDeflate()
+{
+	return boost::shared_ptr<ICodec>(new Compressor_ZLib);
+}
+
+boost::shared_ptr<ICodec> CreateDecompressor_ZLibDeflate()
+{
+	return boost::shared_ptr<ICodec>(new Decompressor_ZLib);
+}

source/lib/file/archive/codec_zlib.h

@@ -0,0 +1,5 @@
+#include "codec.h"
+
+extern boost::shared_ptr<ICodec> CreateCodec_ZLibNone();
+extern boost::shared_ptr<ICodec> CreateCompressor_ZLibDeflate();
+extern boost::shared_ptr<ICodec> CreateDecompressor_ZLibDeflate();

source/lib/file/archive/disabled_tests/test_archive_builder.h

@@ -0,0 +1,144 @@
+#include "lib/self_test.h"
+
+#include "lib/base32.h"
+#include "lib/res/file/path.h"
+#include "lib/res/file/fp_posix.h"
+#include "lib/res/file/file_cache.h"
+#include "lib/res/file/vfs/vfs.h"
+#include "lib/res/file/archive/archive.h"
+#include "lib/res/file/archive/archive_builder.h"
+#include "lib/res/h_mgr.h"
+#include "lib/res/mem.h"
+#include "lib/rand.h"
+
+class TestArchiveBuilder : public CxxTest::TestSuite 
+{
+	const char* const archive_fn;
+	static const size_t NUM_FILES = 30;
+	static const size_t MAX_FILE_SIZE = 20000;
+
+	std::set<const char*> existing_names;
+	const char* gen_random_name()
+	{
+		// 10 chars is enough for (10-1)*5 bits = 45 bits > u32
+		char name_tmp[10];
+
+		for(;;)
+		{
+			u32 rand_num = rand(0, 100000);
+			base32(4, (const u8*)&rand_num, (u8*)name_tmp);
+
+			// store filename in atom pool
+			const char* atom_fn = file_make_unique_fn_copy(name_tmp);
+			// done if the filename is unique (not been generated yet)
+			if(existing_names.find(atom_fn) == existing_names.end())
+			{
+				existing_names.insert(atom_fn);
+				return atom_fn;
+			}
+		}
+	}
+
+	struct TestFile
+	{
+		off_t size;
+		u8* data;	// must be delete[]-ed after comparing
+	};
+	// (must be separate array and end with NULL entry (see Filenames))
+	const char* filenames[NUM_FILES+1];
+	TestFile files[NUM_FILES];
+
+	void generate_random_files()
+	{
+		for(size_t i = 0; i < NUM_FILES; i++)
+		{
+			const off_t size = rand(0, MAX_FILE_SIZE);
+			u8* data = new u8[size];
+
+			// random data won't compress at all, and we want to exercise
+			// the uncompressed codepath as well => make some of the files
+			// easily compressible (much less values).
+			const bool make_easily_compressible = (rand(0, 100) > 50);
+			if(make_easily_compressible)
+			{
+				for(off_t i = 0; i < size; i++)
+					data[i] = rand() & 0x0F;
+			}
+			else
+			{
+				for(off_t i = 0; i < size; i++)
+					data[i] = rand() & 0xFF;
+			}
+
+			filenames[i] = gen_random_name();
+			files[i].size = size;
+			files[i].data = data;
+
+			ssize_t bytes_written = vfs_store(filenames[i], data, size, FILE_NO_AIO);
+			TS_ASSERT_EQUALS(bytes_written, size);
+		}
+
+		// 0-terminate the list - see Filenames decl.
+		filenames[NUM_FILES] = NULL;
+	}
+
+public:
+	TestArchiveBuilder()
+		: archive_fn("test_archive_random_data.zip") {}
+
+	void setUp()
+	{
+		(void)path_SetRoot(0, ".");
+		vfs_init();
+	}
+
+	void tearDown()
+	{
+		vfs_shutdown();
+		path_ResetRootDir();
+	}
+
+	void test_create_archive_with_random_files()
+	{
+		if(!file_exists("archivetest")) // don't get stuck if this test fails and never deletes the directory it created
+			TS_ASSERT_OK(dir_create("archivetest"));
+
+		TS_ASSERT_OK(vfs_mount("", "archivetest"));
+
+		generate_random_files();
+		TS_ASSERT_OK(archive_build(archive_fn, filenames));
+
+		// wipe out file cache, otherwise we're just going to get back
+		// the file contents read during archive_build .
+		file_cache_reset();
+
+		// read in each file and compare file contents
+		Handle ha = archive_open(archive_fn);
+		TS_ASSERT(ha > 0);
+		for(size_t i = 0; i < NUM_FILES; i++)
+		{
+			File f;
+			TS_ASSERT_OK(afile_open(ha, filenames[i], 0, 0, &f));
+			FileIOBuf buf = FILE_BUF_ALLOC;
+			ssize_t bytes_read = afile_read(&f, 0, files[i].size, &buf);
+			TS_ASSERT_EQUALS(bytes_read, files[i].size);
+
+			TS_ASSERT_SAME_DATA(buf, files[i].data, files[i].size);
+
+			TS_ASSERT_OK(file_cache_free(buf));
+			TS_ASSERT_OK(afile_close(&f));
+			SAFE_ARRAY_DELETE(files[i].data);
+		}
+		TS_ASSERT_OK(archive_close(ha));
+
+		dir_delete("archivetest");
+		file_delete(archive_fn);
+	}
+
+	void test_multiple_init_shutdown()
+	{
+		// setUp has already vfs_init-ed it and tearDown will vfs_shutdown.
+		vfs_shutdown();
+		vfs_init();
+	}
+};

source/lib/file/archive/disabled_tests/test_codec_zlib.h

@@ -0,0 +1,59 @@
+#include "lib/self_test.h"
+
+#include "lib/self_test.h"
+#include "lib/res/file/archive/codec_zlib.h"
+
+class TestCodecZLib : public CxxTest::TestSuite 
+{
+public:
+	void test_compress_decompress_compare()
+	{
+		size_t inConsumed, outProduced;
+		u32 checksum;
+
+		// generate random input udata
+		// (limit values to 0..7 so that the udata will actually be compressible)
+		const size_t usize = 10000;
+		u8 udata[usize];
+		for(size_t i = 0; i < usize; i++)
+			udata[i] = rand() & 0x07;
+
+		// compress
+		u8* cdata; size_t csize;
+		{
+			boost::shared_ptr<ICodec> compressor_zlib = CreateCompressor_ZLib();
+			ICodec* c = compressor_zlib.get();
+			const size_t csizeMax = c->MaxOutputSize(usize);
+			cdata = new u8[csizeMax];
+			TS_ASSERT_OK(c->Process(udata, usize, cdata, csizeMax, inConsumed, outProduced));
+			TS_ASSERT_EQUALS(inConsumed, usize);
+			TS_ASSERT_LESS_THAN_EQUALS(outProduced, csizeMax);
+			u8* cdata2;
+			TS_ASSERT_OK(c->Finish(cdata2, csize, checksum));
+			TS_ASSERT_EQUALS(cdata, cdata2);
+			TS_ASSERT_EQUALS(csize, outProduced);
+		}
+
+		// make sure the data changed during compression
+		TS_ASSERT(csize != usize || memcmp(udata, cdata, std::min(usize, csize)) != 0);
+
+		// decompress
+		u8 ddata[usize];
+		{
+			boost::shared_ptr<ICodec> decompressor_zlib = CreateDecompressor_ZLib();
+			ICodec* d = decompressor_zlib.get();
+			TS_ASSERT_OK(decompressor_zlib->Process(cdata, csize, ddata, usize, inConsumed, outProduced));
+			TS_ASSERT_EQUALS(inConsumed, csize);	// ZLib always outputs as much data as possible
+			TS_ASSERT_EQUALS(outProduced, usize);	// .. so these figures are correct before Finish()
+			u8* ddata2; size_t dsize;
+			TS_ASSERT_OK(d->Finish(&ddata2, &dsize, &checksum));
+			TS_ASSERT_EQUALS(ddata, ddata2);
+			TS_ASSERT_EQUALS(dsize, outProduced);
+		}
+
+		// verify udata survived intact
+		TS_ASSERT_SAME_DATA(udata, ddata, usize);
+
+		delete[] cdata;
+	}
+};

source/lib/file/archive/disabled_tests/test_compression.h

@@ -0,0 +1,54 @@
+#include "lib/self_test.h"
+
+#include "lib/self_test.h"
+#include "lib/res/file/archive/compression.h"
+
+class TestCompression : public CxxTest::TestSuite 
+{
+public:
+	void test_compress_decompress_compare()
+	{
+		// generate random input data
+		// (limit values to 0..7 so that the data will actually be compressible)
+		const size_t data_size = 10000;
+		u8 data[data_size];
+		for(size_t i = 0; i < data_size; i++)
+			data[i] = rand() & 0x07;
+
+		u8* cdata; size_t csize;
+		u8 udata[data_size];
+
+		// compress
+		uintptr_t c = comp_alloc(CT_COMPRESSION, CM_DEFLATE);
+		{
+		TS_ASSERT(c != 0);
+		const size_t csizeBound = comp_max_output_size(c, data_size);
+		TS_ASSERT_OK(comp_alloc_output(c, csizeBound));
+		const ssize_t cdata_produced = comp_feed(c, data, data_size);
+		TS_ASSERT(cdata_produced >= 0);
+		u32 checksum;
+		TS_ASSERT_OK(comp_finish(c, &cdata, &csize, &checksum));
+		TS_ASSERT(cdata_produced <= (ssize_t)csize);	// can't have produced more than total
+		}
+
+		// decompress
+		uintptr_t d = comp_alloc(CT_DECOMPRESSION, CM_DEFLATE);
+		{
+		TS_ASSERT(d != 0);
+		comp_set_output(d, udata, data_size);
+		const ssize_t udata_produced = comp_feed(d, cdata, csize);
+		TS_ASSERT(udata_produced >= 0);
+		u8* udata_final; size_t usize_final; u32 checksum;
+		TS_ASSERT_OK(comp_finish(d, &udata_final, &usize_final, &checksum));
+		TS_ASSERT(udata_produced <= (ssize_t)usize_final);	// can't have produced more than total
+		TS_ASSERT_EQUALS(udata_final, udata);	// output buffer address is same
+		TS_ASSERT_EQUALS(usize_final, data_size);	// correct amount of output
+		}
+
+		comp_free(c);
+		comp_free(d);
+
+		// verify data survived intact
+		TS_ASSERT_SAME_DATA(data, udata, data_size);
+	}
+};

source/lib/file/archive/disabled_tests/test_fat_time.h

@@ -0,0 +1,25 @@
+#include "lib/self_test.h"
+
+#include <ctime>
+
+#include "lib/res/file/archive/fat_time.h"
+
+class TestFatTime: public CxxTest::TestSuite 
+{
+public:
+	void test_fat_timedate_conversion()
+	{
+		// note: FAT time stores second/2, which means converting may
+		// end up off by 1 second.
+
+		time_t t, converted_t;
+
+		t = time(0);
+		converted_t = time_t_from_FAT(FAT_from_time_t(t));
+		TS_ASSERT_DELTA(t, converted_t, 2);
+
+		t++;
+		converted_t = time_t_from_FAT(FAT_from_time_t(t));
+		TS_ASSERT_DELTA(t, converted_t, 2);
+	}
+};

source/lib/file/archive/disabled_tests/test_zip.h

@@ -0,0 +1,10 @@
+#include "lib/self_test.h"
+
+#include <time.h>
+
+#include "lib/res/file/archive/zip.h"
+
+class TestZip : public CxxTest::TestSuite 
+{
+public:
+};

source/lib/file/archive/stream.cpp

@@ -0,0 +1,127 @@
+/**
+ * =========================================================================
+ * File        : stream.cpp
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "stream.h"
+
+#include "lib/allocators/allocators.h"	// page_aligned_alloc
+#include "codec.h"
+
+
+OutputBufferManager::OutputBufferManager()
+{
+	Reset();
+}
+
+void OutputBufferManager::Reset()
+{
+	m_buffer = 0;
+	m_size = 0;
+	m_capacity = 0;
+}
+
+void OutputBufferManager::SetBuffer(u8* buffer, size_t size)
+{
+	debug_assert(IsAllowableBuffer(buffer, size));
+
+	m_buffer = buffer;
+	m_size = size;
+}
+
+void OutputBufferManager::AllocateBuffer(size_t size)
+{
+	// notes:
+	// - this implementation allows reusing previous buffers if they
+	//   are big enough, which reduces the number of allocations.
+	// - no further attempts to reduce allocations (e.g. by doubling
+	//   the current size) are made; this strategy is enough.
+	// - Pool etc. cannot be used because files may be huge (larger
+	//   than the address space of 32-bit systems).
+
+	// no buffer or the previous one wasn't big enough: reallocate
+	if(!m_mem.get() || m_capacity < size)
+	{
+		m_mem.reset((u8*)page_aligned_alloc(size), PageAlignedDeleter(size));
+		m_capacity = size;
+	}
+
+	SetBuffer(m_mem.get(), size);
+}
+
+bool OutputBufferManager::IsAllowableBuffer(u8* buffer, size_t size)
+{
+	// none yet established
+	if(m_buffer == 0 && m_size == 0)
+		return true;
+
+	// same as last time (happens with temp buffers)
+	if(m_buffer == buffer && m_size == size)
+		return true;
+
+	// located after the last buffer (note: not necessarily after
+	// the entire buffer; a lack of input can cause the output buffer
+	// to only partially be used before the next call.)
+	if((unsigned)(buffer - m_buffer) <= m_size)
+		return true;
+
+	return false;
+}
+
+
+//-----------------------------------------------------------------------------
+
+
+Stream::Stream(boost::shared_ptr<ICodec> codec)
+	: m_codec(codec)
+	, m_inConsumed(0), m_outProduced(0)
+{
+}
+
+
+void Stream::AllocateOutputBuffer(size_t outSizeMax)
+{
+	m_outputBufferManager.AllocateBuffer(outSizeMax);
+}
+
+
+void Stream::SetOutputBuffer(u8* out, size_t outSize)
+{
+	m_outputBufferManager.SetBuffer(out, outSize);
+}
+
+
+LibError Stream::Feed(const u8* in, size_t inSize, size_t& bytesProcessed)
+{
+	if(m_outProduced == m_outputBufferManager.Size())	// output buffer full; must not call Process
+		return INFO::OK;
+
+	size_t inConsumed, outProduced;
+	u8* const out = m_outputBufferManager.Buffer();
+	const size_t outSize = m_outputBufferManager.Size() - m_outProduced;
+	RETURN_ERR(m_codec.get()->Process(in, inSize, out, outSize, inConsumed, outProduced));
+
+	m_inConsumed += inConsumed;
+	m_outProduced += outProduced;
+	bytesProcessed = outProduced;
+	return INFO::CB_CONTINUE;
+}
+
+
+LibError Stream::Finish()
+{
+	return m_codec.get()->Finish(m_checksum);
+}
+
+
+LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize, size_t& bytesProcessed)
+{
+	Stream& stream = *(Stream*)cbData;
+	return stream.Feed(in, inSize, bytesProcessed);
+}

source/lib/file/archive/stream.h

@@ -0,0 +1,103 @@
+/**
+ * =========================================================================
+ * File        : stream.cpp
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_STREAM
+#define INCLUDED_STREAM
+
+struct ICodec;
+
+// note: this is similar in function to std::vector, but we don't need
+// iterators etc. and would prefer to avoid initializing each byte.
+class OutputBufferManager
+{
+public:
+	OutputBufferManager();
+
+	void Reset();
+	void SetBuffer(u8* buffer, size_t size);
+
+	/**
+	 * allocate a new output buffer.
+	 *
+	 * @param size [bytes] to allocate.
+	 *
+	 * notes:
+	 * - if a buffer had previously been allocated and is large enough,
+	 *   it is reused (this reduces the number of allocations).
+	 * - this class manages the lifetime of the buffer.
+	 **/
+	void AllocateBuffer(size_t size);
+
+	u8* Buffer() const
+	{
+		return m_buffer;
+	}
+
+	size_t Size() const
+	{
+		return m_size;
+	}
+
+private:
+	bool IsAllowableBuffer(u8* buffer, size_t size);
+
+	u8* m_buffer;
+	size_t m_size;
+
+	boost::shared_array<u8> m_mem;
+	// size of m_mem. allows reusing previously allocated buffers
+	// (user-specified buffers can't be reused because we have no control
+	// over their lifetime)
+	size_t m_capacity;
+};
+
+
+class Stream
+{
+public:
+	Stream(boost::shared_ptr<ICodec> codec);
+
+	void SetOutputBuffer(u8* out, size_t outSize);
+
+	void AllocateOutputBuffer(size_t outSizeMax);
+
+	/**
+	 * 'feed' the codec with a data block.
+	 *
+	 * @param bytesProcessed receives the number of output bytes produced.
+	 * it can legitimately be 0 - this happens if the input buffer is small
+	 * and the codec hasn't produced any output.
+	 **/
+	LibError Feed(const u8* in, size_t inSize, size_t& bytesProcessed);
+
+	LibError Finish();
+
+	size_t OutSize() const
+	{
+		return m_outProduced;
+	}
+
+	u32 Checksum() const
+	{
+		return m_checksum;
+	}
+
+private:
+	boost::shared_ptr<ICodec> m_codec;
+	OutputBufferManager m_outputBufferManager;
+
+	size_t m_inConsumed;
+	size_t m_outProduced;
+	u32 m_checksum;
+};
+
+extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize, size_t& bytesProcessed);
+
+#endif	// #ifndef INCLUDED_STREAM

source/lib/file/archive/trace.cpp

@@ -0,0 +1,235 @@
+/**
+ * =========================================================================
+ * File        : trace.cpp
+ * Project     : 0 A.D.
+ * Description : allows recording and 'playing back' a sequence of
+ *             : I/Os - useful for benchmarking and archive builder.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "trace.h"
+
+#include "lib/allocators.h"
+#include "lib/timer.h"
+#include "lib/sysdep/cpu.h"
+
+// one run per file
+
+
+/*virtual*/ ITrace::~ITrace()
+{
+}
+
+class Trace_Dummy : public ITrace
+{
+public:
+	Trace_Dummy(size_t UNUSED(maxSize))
+	{
+
+	}
+
+	virtual void NotifyLoad(const char* UNUSED(pathname), size_t UNUSED(size))
+	{
+	}
+
+	virtual void NotifyStore(const char* UNUSED(pathname), size_t UNUSED(size))
+	{
+	}
+
+	virtual void NotifyFree(const char* UNUSED(pathname))
+	{
+	}
+
+	virtual LibError Load(const char* UNUSED(vfsPathname))
+	{
+		return INFO::OK;
+	}
+
+	virtual LibError Store(const char* UNUSED(vfsPathname)) const
+	{
+		return INFO::OK;
+	}
+
+	virtual const TraceEntry* Entries() const
+	{
+		return 0;
+	}
+
+	virtual size_t NumEntries() const
+	{
+		return 0;
+	}
+};
+
+
+class Trace : public ITrace
+{
+public:
+	Trace(size_t maxSize)
+	{
+		(void)pool_create(&m_pool, maxSize, sizeof(TraceEntry));
+	}
+
+	virtual ~Trace()
+	{
+		(void)pool_destroy(&m_pool);
+	}
+
+	virtual void NotifyLoad(const char* pathname, size_t size)
+	{
+		AddEntry(TO_LOAD, pathname, size);
+	}
+
+	virtual void NotifyStore(const char* pathname, size_t size)
+	{
+		AddEntry(TO_STORE, pathname, size);
+	}
+
+	virtual void NotifyFree(const char* pathname)
+	{
+		const size_t size = 0;	// meaningless for TO_FREE
+		AddEntry(TO_FREE, pathname, size);
+	}
+
+	virtual LibError Load(const char* osPathname)
+	{
+		pool_free_all(&m_pool);
+
+		FILE* f = fopen(osPathname, "rt");
+		if(!f)
+			WARN_RETURN(ERR::FILE_ACCESS);
+		for(size_t i = 0; ; i++)
+		{
+			TraceEntry* t = (TraceEntry*)pool_alloc(&m_pool, 0);
+			t->Read(f);
+		}
+
+		fclose(f);
+
+		return INFO::OK;
+	}
+
+	virtual LibError Store(const char* osPathname) const
+	{
+		FILE* f = fopen(osPathname, "at");
+		if(!f)
+			WARN_RETURN(ERR::FILE_ACCESS);
+		for(size_t i = 0; i < NumEntries(); i++)
+			Entries()[i]->Write(f);
+		(void)fclose(f);
+		return INFO::OK;
+	}
+
+	virtual const TraceEntry* Entries() const
+	{
+		return (const TraceEntry*)m_pool.da.base;
+	}
+
+	virtual size_t NumEntries() const
+	{
+		return m_pool.da.pos / sizeof(TraceEntry);
+	}
+
+private:
+	void AddEntry(TraceOp op, const char* pathname, size_t size, double timestamp = 0.0)
+	{
+		TraceEntry* t = (TraceEntry*)pool_alloc(&m_pool, 0);
+		debug_assert(t);
+		t->timestamp = timestamp == 0.0? get_time() : timestamp;
+		t->vfsPathname = path_Pool()->UniqueCopy(pathname);
+		t->size = size;
+		t->op = op;
+	}
+
+	Pool m_pool;
+};
+
+
+
+
+
+// parse lines and stuff them in m_pool
+// (as if they had been AddEntry-ed; replaces any existing data)
+// .. bake PATH_MAX limit into string.
+char fmt[30];
+snprintf(fmt, ARRAY_SIZE(fmt), "%%lf: %%c \"%%%d[^\"]\" %%d %%04x\n", PATH_MAX);
+
+double timestamp; char opcode; char P_path[PATH_MAX]; size_t size;
+int fieldsRead = fscanf(f, fmt, &timestamp, &opcode, P_path, &size);
+if(fieldsRead == EOF)
+break;
+debug_assert(fieldsRead == 4);
+
+
+
+static char CharFromTraceOp(TraceOp op)
+{
+	switch(op)
+	{
+	case TO_LOAD:
+		return 'L';
+	case TO_STORE:
+		return 'S';
+	case TO_FREE:
+		return 'F';
+	default:
+		DEBUG_WARN_ERR(ERR::CORRUPTED);
+	}
+}
+
+static TraceOp TraceOpFromChar(char c)
+{
+	switch(c)
+	{
+	case 'L':
+		return TO_LOAD;
+	case 'S':
+		return TO_STORE;
+	case 'F':
+		return TO_FREE;
+	default:
+		DEBUG_WARN_ERR(ERR::CORRUPTED);
+	}
+}
+
+static void write_entry(FILE* f, const TraceEntry* ent)
+{
+	fprintf(f, "%#010f: %c \"%s\" %d\n", ent->timestamp, CharFromTraceOp(ent->op), ent->vfsPathname, ent->size);
+}
+
+{
+	// carry out this entry's operation
+	switch(ent->op)
+	{
+		// do not 'run' writes - we'd destroy the existing data.
+	case TO_STORE:
+		break;
+	case TO_LOAD:
+		{
+			IoBuf buf; size_t size;
+			(void)vfs_load(ent->vfsPathname, buf, size, ent->flags);
+			break;
+		}
+	case TO_FREE:
+		fileCache.Release(ent->vfsPathname);
+		break;
+	default:
+		debug_warn("unknown TraceOp");
+	}
+}
+
+
+
+
+
+LibError trace_run(const char* osPathname)
+{
+	Trace trace;
+	RETURN_ERR(trace.Load(osPathname));
+	for(uint i = 0; i < trace.NumEntries(); i++)
+		trace.Entries()[i]->Run();
+	return INFO::OK;
+}

source/lib/file/archive/trace.h

@@ -0,0 +1,66 @@
+/**
+ * =========================================================================
+ * File        : trace.h
+ * Project     : 0 A.D.
+ * Description : allows recording and 'playing back' a sequence of
+ *             : I/Os - useful for benchmarking and archive builder.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_TRACE
+#define INCLUDED_TRACE
+
+// TraceEntry operation type.
+// note: rather than only a list of accessed files, we also need to
+// know the application's behavior WRT caching (e.g. when it releases
+// cached buffers). this is necessary so that our simulation can
+// yield the same behavior.
+enum TraceOp
+{
+	TO_LOAD,
+	TO_STORE,
+	TO_FREE,
+};
+
+// stores one event that is relevant for file IO / caching.
+//
+// size-optimized a bit since these are all kept in memory
+// (to prevent trace file writes from affecting other IOs)
+struct TraceEntry
+{
+	// note: float instead of double for nice 16 byte struct size
+	float timestamp;		// returned by get_time before operation starts
+	const char* vfsPathname;
+	// rationale: store size in the trace because other applications
+	// that use this trace format but not our IO code wouldn't know
+	// size (since they cannot retrieve the file info given vfsPathname).
+	size_t size;			// of IO (usually the entire file)
+	TraceOp op;
+};
+
+struct ITrace
+{
+	virtual ~ITrace();
+
+	virtual void NotifyLoad(const char* vfsPathname, size_t size);
+	virtual void NotifyStore(const char* vfsPathname, size_t size);
+	virtual void NotifyFree(const char* vfsPathname);
+
+	// note: file format is text-based to allow human inspection and because
+	// storing filenames would be a bit awkward
+	virtual LibError Load(const char* osPathname);
+	virtual LibError Store(const char* osPathname) const;
+
+	virtual const TraceEntry* Entries() const;
+	virtual size_t NumEntries() const;
+};
+
+
+// enabled by default. by the time we can decide whether a trace needs to
+// be generated (see should_rebuild_main_archive), file accesses will
+// already have occurred; hence default enabled and disable if not needed.
+
+
+#endif	// #ifndef INCLUDED_TRACE

source/lib/file/dir_util.cpp

@@ -21,30 +21,30 @@
 
 bool dir_FileExists(IFilesystem* fs, const char* pathname)
 {
-	FilesystemEntry fsEntry;
+	FileInfo fileInfo;
-	if(fs->GetEntry(pathname, fsEntry) < 0)
+	if(fs->GetEntry(pathname, fileInfo) < 0)
 		return false;
-	debug_assert(!fsEntry.IsDirectory());
+	debug_assert(!fileInfo.IsDirectory());
 	return true;
 }
 
 
 bool dir_DirectoryExists(IFilesystem* fs, const char* dirPath)
 {
-	FilesystemEntry fsEntry;
+	FileInfo fileInfo;
-	if(fs->GetEntry(dirPath, fsEntry) < 0)
+	if(fs->GetEntry(dirPath, fileInfo) < 0)
 		return false;
-	debug_assert(fsEntry.IsDirectory());
+	debug_assert(fileInfo.IsDirectory());
 	return true;
 }
 
 
 
-struct FsEntryNameLess : public std::binary_function<const FilesystemEntry, const FilesystemEntry, bool>
+struct FsEntryNameLess : public std::binary_function<const FileInfo, const FileInfo, bool>
 {
-	bool operator()(const FilesystemEntry& fsEntry1, const FilesystemEntry& fsEntry2) const
+	bool operator()(const FileInfo& fileInfo1, const FileInfo& fileInfo2) const
 	{
-		return strcmp(fsEntry1.name, fsEntry2.name) < 0;
+		return strcmp(fileInfo1.name, fileInfo2.name) < 0;
 	}
 };
 
@@ -53,14 +53,14 @@ LibError dir_GatherSortedEntries(IFilesystem* fs, const char* dirPath, Filesyste
 	DirectoryIterator di(fs, dirPath);
 	fsEntries.reserve(50);	// preallocate for efficiency
 
-	FilesystemEntry fsEntry;
+	FileInfo fileInfo;
 	for(;;)
 	{
-		LibError ret = di.NextEntry(fsEntry);
+		LibError ret = di.NextEntry(fileInfo);
 		if(ret == ERR::DIR_END)
 			break;
 		RETURN_ERR(ret);
-		fsEntries.push_back(fsEntry);
+		fsEntries.push_back(fileInfo);
 	}
 
 	std::sort(fsEntries.begin(), fsEntries.end(), FsEntryNameLess());
@@ -79,10 +79,10 @@ LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, const DirC
 
 	for(FilesystemEntries::const_iterator it = fsEntries.begin(); it != fsEntries.end(); ++it)
 	{
-		const FilesystemEntry& fsEntry = *it;
+		const FileInfo& fileInfo = *it;
-		path_package_append_file(&pp, fsEntry.name);
+		path_package_append_file(&pp, fileInfo.name);
 
-		LibError ret = cb(pp.path, fsEntry, cbData);
+		LibError ret = cb(pp.path, fileInfo, cbData);
 		if(ret != INFO::CB_CONTINUE)
 			return ret;
 	}
@@ -91,7 +91,7 @@ LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, const DirC
 }
 
 
-LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry, const char* filter)
+LibError dir_filtered_next_ent(DirectoryIterator& di, FileInfo& fileInfo, const char* filter)
 {
 	bool want_dir = true;
 	if(filter)
@@ -108,12 +108,12 @@ LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry,
 			want_dir = false;
 	}
 
-	// loop until fsEntry matches what is requested, or end of directory.
+	// loop until fileInfo matches what is requested, or end of directory.
 	for(;;)
 	{
-		RETURN_ERR(di.NextEntry(fsEntry));
+		RETURN_ERR(di.NextEntry(fileInfo));
 
-		if(fsEntry.IsDirectory())
+		if(fileInfo.IsDirectory())
 		{
 			if(want_dir)
 				break;
@@ -121,7 +121,7 @@ LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry,
 		else
 		{
 			// (note: filter = 0 matches anything)
-			if(match_wildcard(fsEntry.name, filter))
+			if(match_wildcard(fileInfo.name, filter))
 				break;
 		}
 	}
@@ -130,11 +130,11 @@ LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry,
 }
 
 
-// call <cb> for each fsEntry matching <user_filter> (see vfs_next_dirent) in
+// call <cb> for each fileInfo matching <user_filter> (see vfs_next_dirent) in
 // directory <path>; if flags & VFS_DIR_RECURSIVE, entries in
 // subdirectories are also returned.
 //
-// note: EnumDirEntsCB path and fsEntry are only valid during the callback.
+// note: EnumDirEntsCB path and fileInfo are only valid during the callback.
 LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint flags, const char* user_filter, DirCallback cb, uintptr_t cbData)
 {
 	debug_assert((flags & ~(VFS_DIR_RECURSIVE)) == 0);
@@ -162,7 +162,7 @@ LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint fla
 	// directory consecutively)
 
 	std::queue<const char*> dir_queue;
-	dir_queue.push(path_UniqueCopy(dirPath));
+	dir_queue.push(path_Pool()->UniqueCopy(dirPath));
 
 	// for each directory:
 	do
@@ -176,24 +176,24 @@ LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint fla
 
 		DirectoryIterator di(fs, pp.path);
 
-		// for each fsEntry (file, subdir) in directory:
+		// for each fileInfo (file, subdir) in directory:
-		FilesystemEntry fsEntry;
+		FileInfo fileInfo;
-		while(dir_filtered_next_ent(di, fsEntry, filter) == 0)
+		while(dir_filtered_next_ent(di, fileInfo, filter) == 0)
 		{
-			// build complete path (FilesystemEntry only stores fsEntry name)
+			// build complete path (FileInfo only stores fileInfo name)
-			(void)path_package_append_file(&pp, fsEntry.name);
+			(void)path_package_append_file(&pp, fileInfo.name);
-			const char* atom_path = path_UniqueCopy(pp.path);
+			const char* atom_path = path_Pool()->UniqueCopy(pp.path);
 
-			if(fsEntry.IsDirectory())
+			if(fileInfo.IsDirectory())
 			{
 				if(recursive)
 					dir_queue.push(atom_path);
 
 				if(user_filter_wants_dirs)
-					cb(atom_path, fsEntry, cbData);
+					cb(atom_path, fileInfo, cbData);
 			}
 			else
-				cb(atom_path, fsEntry, cbData);
+				cb(atom_path, fileInfo, cbData);
 		}
 	}
 	while(!dir_queue.empty());
@@ -207,9 +207,6 @@ LibError dir_FilteredForEachEntry(IFilesystem* fs, const char* dirPath, uint fla
 // the next numbered filename according to the pattern defined by V_fn_fmt.
 // <state> must be initially zeroed (e.g. by defining as static) and passed
 // each time.
-// if <use_vfs> (default), the paths are treated as VFS paths; otherwise,
-// file.cpp's functions are used. this is necessary because one of
-// our callers needs a filename for VFS archive files.
 //
 // this function is useful when creating new files which are not to
 // overwrite the previous ones, e.g. screenshots.
@@ -229,10 +226,10 @@ void dir_NextNumberedFilename(IFilesystem* fs, const char* fn_fmt, NextNumberedF
 
 		int max_num = -1; int num;
 		DirectoryIterator di(fs, dirPath);
-		FilesystemEntry fsEntry;
+		FileInfo fileInfo;
-		while(di.NextEntry(fsEntry) == INFO::OK)
+		while(di.NextEntry(fileInfo) == INFO::OK)
 		{
-			if(!fsEntry.IsDirectory() && sscanf(fsEntry.name, name_fmt, &num) == 1)
+			if(!fileInfo.IsDirectory() && sscanf(fileInfo.name, name_fmt, &num) == 1)
 				max_num = std::max(num, max_num);
 		}
 

source/lib/file/dir_util.h

@@ -17,7 +17,7 @@ extern bool dir_FileExists(IFilesystem* fs, const char* pathname);
 extern bool dir_DirectoryExists(IFilesystem* fs, const char* dirPath);
 
 
-typedef std::vector<FilesystemEntry> FilesystemEntries;
+typedef std::vector<FileInfo> FilesystemEntries;
 
 // enumerate all directory entries in <P_path>; add to container and
 // then sort it by filename.
@@ -27,7 +27,7 @@ extern LibError dir_GatherSortedEntries(IFilesystem* fs, const char* dirPath, Fi
 // called by dir_ForEachSortedEntry for each entry in the directory.
 // return INFO::CB_CONTINUE to continue calling; anything else will cause
 // dir_ForEachSortedEntry to abort and immediately return that value.
-typedef LibError (*DirCallback)(const char* pathname, const FilesystemEntry& fsEntry, const uintptr_t cbData);
+typedef LibError (*DirCallback)(const char* pathname, const FileInfo& fileInfo, const uintptr_t cbData);
 
 
 // call <cb> for each file and subdirectory in <dir> (alphabetical order),
@@ -49,6 +49,33 @@ typedef LibError (*DirCallback)(const char* pathname, const FilesystemEntry& fsE
 extern LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, DirCallback cb, uintptr_t cbData);
 
 
+
+/**
+ * (mostly) insulating concrete class providing iterator access to
+ * directory entries.
+ * this is usable for posix, VFS, etc.; instances are created via IFilesystem.
+ **/
+class DirectoryIterator
+{
+public:
+	DirectoryIterator(IFilesystem* fs, const char* dirPath)
+		: m_impl(fs->OpenDirectory(dirPath))
+	{
+	}
+
+	// return ERR::DIR_END if all entries have already been returned once,
+	// another negative error code, or INFO::OK on success, in which case <fileInfo>
+	// describes the next (order is unspecified) directory entry.
+	LibError NextEntry(FileInfo& fileInfo)
+	{
+		return m_impl.get()->NextEntry(fileInfo);
+	}
+
+private:
+	boost::shared_ptr<IDirectoryIterator> m_impl;
+};
+
+
 // retrieve the next (order is unspecified) dir entry matching <filter>.
 // return 0 on success, ERR::DIR_END if no matching entry was found,
 // or a negative error code on failure.
@@ -65,8 +92,8 @@ extern LibError dir_ForEachSortedEntry(IFilesystem* fs, const char* dirPath, Dir
 // rationale: we do not sort directory entries alphabetically here.
 // most callers don't need it and the overhead is considerable
 // (we'd have to store all entries in a vector). it is left up to
-// higher-level code such as VfsUtil.
+// other routines.
-extern LibError dir_filtered_next_ent(DirectoryIterator& di, FilesystemEntry& fsEntry, const char* filter);
+extern LibError dir_filtered_next_ent(DirectoryIterator& di, FileInfo& fileInfo, const char* filter);
 
 
 

source/lib/file/disabled_tests/test_path.h

@@ -30,31 +30,27 @@ public:
 
 	// file_make_full_*_path is left untested (hard to do so)
 
-	void test_atom()
+	void test_pool()
 	{
-		// file_make_unique_fn_copy
-
 		// .. return same address for same string?
-		const char* atom1 = file_make_unique_fn_copy("a/bc/def");
+		const char* atom1 = path_Pool->UniqueCopy("a/bc/def");
-		const char* atom2 = file_make_unique_fn_copy("a/bc/def");
+		const char* atom2 = path_Pool->UniqueCopy("a/bc/def");
 		TS_ASSERT_EQUALS(atom1, atom2);
 
 		// .. early out (already in pool) check works?
-		const char* atom3 = file_make_unique_fn_copy(atom1);
+		const char* atom3 = path_Pool->UniqueCopy(atom1);
 		TS_ASSERT_EQUALS(atom3, atom1);
 
-
-		// path_is_atom_fn
 		// is it reported as in pool?
-		TS_ASSERT(path_is_atom_fn(atom1));
+		TS_ASSERT(path_Pool()->Contains(atom1));
 
-		// file_get_random_name
+		// path_Pool()->RandomString
 		// see if the atom added above eventually comes out when a
 		// random one is returned from the pool.
 		int tries_left;
 		for(tries_left = 1000; tries_left != 0; tries_left--)
 		{
-			const char* random_name = file_get_random_name();
+			const char* random_name = path_Pool->RandomString();
 			if(random_name == atom1)
 				break;
 		}

source/lib/file/file_cache.cpp

@@ -1,380 +0,0 @@
-/**
- * =========================================================================
- * File        : file_cache.cpp
- * Project     : 0 A.D.
- * Description : cache of file contents (supports zero-copy IO)
- * =========================================================================
- */
-
-// license: GPL; see lib/license.txt
-
-#include "precompiled.h"
-#include "file_cache.h"
-
-#include <map>
-
-#include "path.h"
-#include "file_stats.h"
-#include "archive/trace.h"
-#include "lib/cache_adt.h"				// Cache
-#include "lib/lockfree.h"
-#include "lib/bits.h"					// round_up
-#include "lib/allocators/allocators.h"
-#include "lib/allocators/headerless.h"
-#include "lib/allocators/mem_util.h"	// mem_PageSize
-
-
-// >= sys_max_sector_size or else waio will have to realign.
-// chosen as exactly 1 page: this allows write-protecting file buffers
-// without worrying about their (non-page-aligned) borders.
-// internal fragmentation is considerable but acceptable.
-static const size_t alignment = mem_PageSize();
-
-
-//-----------------------------------------------------------------------------
-// allocator
-
-/*
-the biggest worry of a file cache is external fragmentation. there are two
-basic ways to combat this:
-1) 'defragment' periodically - move blocks around to increase
-size of available 'holes'.
-2) prevent fragmentation from occurring at all via
-deliberate alloc/free policy.
-
-file contents are returned directly to the user (zero-copy IO), so only
-currently unreferenced blocks can be moved. it is believed that this would
-severely hamper defragmentation; we therefore go with the latter approach.
-
-the basic insight is: fragmentation occurs when a block is freed whose
-neighbors are not free (thus preventing coalescing). this can be prevented by
-allocating objects of similar lifetimes together. typical workloads
-(uniform access frequency) already show such behavior: the Landlord cache
-manager evicts files in an LRU manner, which matches the allocation policy.
-
-references:
-"The Memory Fragmentation Problem - Solved?" (Johnstone and Wilson)
-"Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson)
-*/
-
-class Allocator
-{
-public:
-	Allocator(size_t maxSize)
-		: m_allocator(maxSize)
-	{
-	}
-
-	IoBuf Allocate(size_t size)
-	{
-		const size_t alignedSize = round_up(size, alignment);
-		stats_buf_alloc(size, alignedSize);
-
-		void* p = m_allocator.Allocate(alignedSize);
-#ifndef NDEBUG
-		m_checker.notify_alloc(p, alignedSize);
-#endif
-
-		return (IoBuf)p;
-	}
-
-	void Deallocate(IoBuf buf, size_t size)
-	{
-		void* const p = (void*)buf;
-
-		// (re)allow writes. it would be nice to un-map the buffer, but this is
-		// not possible because HeaderlessAllocator needs to affix boundary tags.
-		(void)mprotect(p, size, PROT_READ|PROT_WRITE);
-
-		const size_t alignedSize = round_up(size, alignment);
-#ifndef NDEBUG
-		m_checker.notify_free(p, alignedSize);
-#endif
-		m_allocator.Deallocate(p, alignedSize);
-
-		stats_buf_free();
-	}
-
-private:
-	HeaderlessAllocator m_allocator;
-
-#ifndef NDEBUG
-	static AllocatorChecker m_checker;
-#endif
-};
-
-
-//-----------------------------------------------------------------------------
-
-typedef LF_RefCountedMemRange FileContents;
-
-/**
- * manages the active FileContents referenced by users.
- * ("active" means between Reserve() and the final Release())
- **/
-class ActiveList
-{
-public:
-	~ActiveList()
-	{
-		// display leaks
-		debug_printf("file_cache leaks:\n");
-		for(MapIt it = m_map.begin(); it != m_map.end(); ++it)
-		{
-			const char* atom_fn = it->first;
-			FileContents& fc = it->second;
-			debug_printf("  %s (0x%P 0x%08x)\n", atom_fn, fc.mem, fc.size);
-		}
-		debug_printf("--------\n");
-	}
-
-	void Add(const char* atom_fn, FileContents& fc)
-	{
-		const PairIB ret = m_map.insert(std::make_pair(atom_fn, fc));
-		debug_assert(ret.second);	// complain if already existed
-	}
-
-	void Remove(const char* atom_fn)
-	{
-		const size_t numRemoved = m_map.erase(atom_fn);
-		debug_assert(numRemoved == 1);
-	}
-
-	FileContents* Find(const char* atom_fn)
-	{
-		MapIt it = m_map.find(atom_fn);
-		if(it == m_map.end())
-			return 0;	// not an error
-		return &it->second;
-	}
-
-	// (called by FileCache::Impl::AllocateCacheSpace; we can't pass
-	// atom_fn because Cache only knows about buf and size.)
-	bool Contains(IoBuf buf) const
-	{
-		for(MapCIt it = m_map.begin(); it != m_map.end(); ++it)
-		{
-			const FileContents& fc = it->second;
-			if(fc.mem == buf)
-				return true;
-		}
-
-		return false;
-	}
-
-private:
-	typedef std::map<const char*, FileContents> Map;
-	typedef Map::iterator MapIt;
-	typedef Map::const_iterator MapCIt;
-	typedef std::pair<MapIt, bool> PairIB;
-	Map m_map;
-};
-
-
-//-----------------------------------------------------------------------------
-// FileCache::Impl
-//-----------------------------------------------------------------------------
-
-// the organization of this cache is somewhat counterintuitive. one might
-// expect a simple mapping of filename to FileContents. however, since users
-// are strongly encouraged to only load/process one file at a time, there
-// will only be a few active references. with the cache holding many more
-// entries, looking up files there is more expensive than consulting a
-// separate list of active FileContents.
-// this list (the "manager") and the cache contents are not necessarily
-// related; no inclusion relation need hold. the only requirement is that
-// each consult the other on ownership issues. if the cache decides a file
-// should be evicted while references to it are active, or users release a
-// reference to FileContents that the cache wants to keep, the memory must
-// not actually be freed. (it is then logically 'owned' by the other)
-
-class FileCache::Impl
-{
-public:
-	Impl(size_t size)
-		: m_allocator(size)
-	{
-	}
-
-	IoBuf Reserve(const char* atom_fn, size_t size)
-	{
-		// (this probably indicates a bug; caching 0-length files would
-		// have no benefit, anyway)
-		debug_assert(size != 0);
-
-		IoBuf buf = AllocateCacheSpace(size);
-
-		FileContents fc;
-		fc.refs.AcquireExclusiveAccess();
-		fc.mem = (void*)buf;
-		fc.size = size;
-		m_activeList.Add(atom_fn, fc);
-
-		return buf;
-	}
-
-	void MarkComplete(const char* atom_fn, uint cost)
-	{
-		FileContents* fc = m_activeList.Find(atom_fn);
-		debug_assert(fc);
-
-		fc->refs.RelinquishExclusiveAccess();
-
-		// zero-copy cache => all users share the contents => must not
-		// allow changes. this will be reverted when the buffer is freed.
-		(void)mprotect(fc->mem, fc->size, PROT_READ);
-
-		m_cache.add(atom_fn, (IoBuf)fc->mem, fc->size, cost);
-	}
-
-	IoBuf Retrieve(const char* atom_fn, size_t& size)
-	{
-		IoBuf buf;
-		if(!m_cache.retrieve(atom_fn, buf, &size))
-			return 0;
-
-		FileContents* pfc = m_activeList.Find(atom_fn);
-		// was already active; add a reference.
-		if(pfc)
-			pfc->refs.AddReference();
-		// in cache, but no active references; add to list.
-		else
-		{
-			FileContents fc;
-			fc.refs.AddReference();
-			fc.mem = (void*)buf;
-			fc.size = size;
-			m_activeList.Add(atom_fn, fc);
-		}
-
-		stats_buf_ref();
-		return buf;
-	}
-
-	void Release(const char* atom_fn)
-	{
-		FileContents* fc = m_activeList.Find(atom_fn);
-		debug_assert(fc);
-
-		fc->refs.Release();
-		if(fc->refs.ReferenceCount() == 0)
-		{
-			trace_notify_free(atom_fn);
-
-			if(!IsInCache(atom_fn))
-				m_allocator.Deallocate((IoBuf)fc->mem, fc->size);
-
-			m_activeList.Remove(atom_fn);
-		}
-	}
-
-	LibError Invalidate(const char* atom_fn)
-	{
-		// remove from cache
-		IoBuf cachedBuf; size_t cachedSize;
-		if(m_cache.peek(atom_fn, cachedBuf, &cachedSize))
-		{
-			m_cache.remove(atom_fn);
-			// note: we ensure cachedBuf is not active below.
-			m_allocator.Deallocate(cachedBuf, cachedSize);
-		}
-
-		// this could happen if a hotload notification comes while someone
-		// is holding a reference to the file contents. atom_fn has been
-		// removed from the cache, so subsequent Retrieve() calls will not
-		// return old data. however, (re)loading the file would fail because
-		// Reserve() ensures there's not already an extant buffer.
-		// the correct way to handle this is to delay or cancel the reload,
-		// so we notify our caller accordingly.
-		if(IsActive(cachedBuf))
-			WARN_RETURN(ERR::AGAIN);	// if this actually happens, remove the warning.
-
-		return INFO::OK;
-	}
-
-private:
-	bool IsActive(IoBuf buf) const
-	{
-		return m_activeList.Contains(buf);
-	}
-
-	bool IsInCache(const char* atom_fn) const
-	{
-		IoBuf cachedBuf; size_t cachedSize;	// unused
-		return m_cache.peek(atom_fn, cachedBuf, &cachedSize);
-	}
-
-	IoBuf AllocateCacheSpace(size_t size)
-	{
-		uint attempts = 0;
-		for(;;)
-		{
-			IoBuf buf = m_allocator.Allocate(size);
-			if(buf)
-				return buf;
-
-			// remove least valuable entry from cache
-			IoBuf discardedBuf; size_t discardedSize;
-			bool removed = m_cache.remove_least_valuable(&discardedBuf, &discardedSize);
-			// only false if cache is empty, which can't be the case because
-			// allocation failed.
-			debug_assert(removed);
-
-			// someone is holding a reference; we must not free the
-			// underlying memory, nor count this iteration.
-			if(IsActive(discardedBuf))
-				continue;
-
-			m_allocator.Deallocate(discardedBuf, discardedSize);
-
-			// note: this may seem hefty, but 300 is known to be reached.
-			// (after building an archive, the file cache will be full;
-			// attempting to allocate a few MB can take a while if only
-			// small scattered blocks are freed.)
-			debug_assert(++attempts < 500);	// otherwise: failed to make room in cache?!
-		}
-	}
-
-	ActiveList m_activeList;
-
-	// HACK: due to atom_fn, we are assured that strings are equal iff their
-	// addresses match. however, Cache's STL (hash_)map stupidly assumes that
-	// const char* keys are "strings". to avoid this behavior, we specify the
-	// key as const void*.
-	static Cache<const void*, IoBuf> m_cache;
-
-	Allocator m_allocator;
-};
-
-
-//-----------------------------------------------------------------------------
-
-FileCache::FileCache(size_t size)
-	: impl(new Impl(size))
-{
-}
-
-IoBuf FileCache::Reserve(const char* atom_fn, size_t size)
-{
-	return impl.get()->Reserve(atom_fn, size);
-}
-
-void FileCache::MarkComplete(const char* atom_fn, uint cost)
-{
-	impl.get()->MarkComplete(atom_fn, cost);
-}
-
-IoBuf FileCache::Retrieve(const char* atom_fn, size_t& size)
-{
-	return impl.get()->Retrieve(atom_fn, size);
-}
-
-void FileCache::Release(const char* atom_fn)
-{
-	impl.get()->Release(atom_fn);
-}
-
-LibError FileCache::Invalidate(const char* atom_fn)
-{
-	return impl.get()->Invalidate(atom_fn);
-}

source/lib/file/file_cache.h

@@ -1,111 +0,0 @@
-/**
- * =========================================================================
- * File        : file_cache.h
- * Project     : 0 A.D.
- * Description : cache of file contents (supports zero-copy IO)
- * =========================================================================
- */
-
-// license: GPL; see lib/license.txt
-
-#ifndef INCLUDED_FILE_CACHE
-#define INCLUDED_FILE_CACHE
-
-#include <boost/shared_ptr.hpp>
-#include "io_buf.h"
-
-
-/**
- * cache of file contents with support for zero-copy IO.
- * this works by reserving a region of the cache, using it as the IO buffer,
- * and returning the memory directly to users. optional write-protection
- * via MMU ensures that the shared contents aren't inadvertently changed.
- *
- * to ensure efficient operation and prevent fragmentation, only one
- * reference should be active at a time. in other words, read a file,
- * process it, and only then start reading the next file.
- *
- * rationale: this is very similar to BlockCache; however, the differences
- * (Reserve's size and MarkComplete's cost parameters and different eviction
- * policies) are enough to warrant separate implementations.
- **/
-class FileCache
-{
-public:
-	/**
-	 * @param size maximum amount [bytes] of memory to use for the cache.
-	 * (managed as a virtual memory region that's committed on-demand)
-	 **/
-	FileCache(size_t size);
-
-	/**
-	 * Allocate an IO buffer in the cache's memory region.
-	 *
-	 * @param atom_fn pathname of the file that is to be read; this is
-	 * the key that will be used to Retrieve the file contents.
-	 * @param size required number of bytes (more may be allocated due to
-	 * alignment and/or internal fragmentation)
-	 * @return suitably aligned memory; never fails.
-	 *
-	 * no further operations with the same atom_fn are allowed to succeed
-	 * until MarkComplete has been called.
-	 **/
-	IoBuf Reserve(const char* atom_fn, size_t size);
-
-	/**
-	 * Indicate that IO into the buffer has completed.
-	 *
-	 * this allows the cache to satisfy subsequent Retrieve() calls by
-	 * returning this buffer; if CONFIG_READ_ONLY_CACHE, the buffer is
-	 * made read-only. if need be and no references are currently attached
-	 * to it, the memory can also be commandeered by Reserve().
-	 *
-	 * @param cost is the expected cost of retrieving the file again and
-	 * influences how/when it is evicted from the cache.
-	 **/
-	void MarkComplete(const char* atom_fn, uint cost = 1);
-
-	/**
-	 * Attempt to retrieve a file's contents from the file cache.
-	 *
-	 * @return 0 if not in cache or its IO is still pending, otherwise a
-	 * pointer to its (read-only) contents.
-	 *
-	 * if successful, the size is passed back and a reference is added to
-	 * the file contents.
-	 *
-	 * note: does not call stats_cache because it does not know the file size
-	 * in case of a cache miss; doing so is left to the caller.
-	 **/
-	const u8* Retrieve(const char* atom_fn, size_t& size);
-
-	/**
-	 * Indicate the file contents are no longer needed.
-	 *
-	 * this decreases the reference count; the memory can only be reused
-	 * if it reaches 0. the contents remain in cache until they are evicted
-	 * by a subsequent Reserve() call.
-	 *
-	 * note: fails (raises a warning) if called for an file that is
-	 * currently between Reserve and MarkComplete operations.
-	 **/
-	void Release(const char* atom_fn);
-
-	/**
-	 * Invalidate the cached contents of a file.
-	 *
-	 * this ensures subsequent reads of the files see the current (presumably
-	 * recently changed) contents of the file. has no effect if the file is
-	 * not cached at the moment.
-	 *
-	 * this would typically be called in response to a notification that a
-	 * file has changed.
-	 **/
-	LibError Invalidate(const char* atom_fn);
-
-private:
-	class Impl;
-	boost::shared_ptr<Impl> impl;
-};
-
-#endif	// #ifndef INCLUDED_FILE_CACHE

source/lib/file/file_stats.h

@@ -29,6 +29,8 @@ extern void stats_vfs_init_start();
 extern void stats_vfs_init_finish();
 
 // file
+
+// currently not called because string_pool is now in lib/allocators
 extern void stats_unique_name(size_t name_len);
 extern void stats_open(const char* atom_fn, size_t file_size);
 extern void stats_close();

source/lib/file/filesystem.cpp

@@ -4,16 +4,12 @@
 #include "path.h"
 
 ERROR_ASSOCIATE(ERR::FILE_ACCESS, "Insufficient access rights to open file", EACCES);
-ERROR_ASSOCIATE(ERR::DIR_END, "End of directory reached (no more files)", -1);
+ERROR_ASSOCIATE(ERR::DIR_NOT_FOUND, "Directory not found", ENOENT);
 ERROR_ASSOCIATE(ERR::IO, "Error during IO", EIO);
 ERROR_ASSOCIATE(ERR::IO_EOF, "Reading beyond end of file", -1);
 
 
-// rationale for out-of-line dtors: see [Lakos]
+// rationale for out-of-line dtor: see [Lakos]
-
-IDirectoryIterator::~IDirectoryIterator()
-{
-}
 
 IFilesystem::~IFilesystem()
 {

source/lib/file/filesystem.h

@@ -11,104 +11,49 @@
 #ifndef INCLUDED_FILESYSTEM
 #define INCLUDED_FILESYSTEM
 
-#include <boost/shared_ptr.hpp>
-
-#include "io/io_manager.h"	// IoCallback
-
-namespace ERR
-{
-	const LibError FILE_ACCESS     = -110000;
-	const LibError DIR_NOT_FOUND   = -110002;
-	const LibError DIR_END         = -110003;
-}
-
 
 /**
- * information describing filesystem entries (i.e. files or directories)
+ * information describing file system entries (i.e. files or directories)
  *
  * note: don't be extravagant with memory - dir_ForEachSortedEntry allocates
  * one instance of this per directory entry.
  **/
-struct FilesystemEntry
+class FileInfo
 {
-	off_t size;
+public:
-	time_t mtime;
+	FileInfo()
+	{
+	}
 
+	FileInfo(const char* name, off_t size, time_t mtime)
+		: m_name(name), m_size(size), m_mtime(mtime)
+	{
+	}
+
+	const char* Name() const
+	{
+		return m_name;
+	}
+
+	off_t Size() const
+	{
+		return m_size;
+	}
+
+	time_t MTime() const
+	{
+		return m_mtime;
+	}
+
+private:
 	/**
 	 * name of the entry; does not include a path.
 	 * the underlying storage is guaranteed to remain valid and must not
 	 * be freed/modified.
 	 **/
-	const char* name;
+	const char* m_name;
-
+	off_t m_size;
-	// only defined for VFS files; points to their TMount.
+	time_t m_mtime;
-	const void* mount;
-
-	bool IsDirectory() const
-	{
-		return (size == -1);
-	}
-};
-
-
-// (note: this is defined in the public header to promote inlining of the
-// DirectoryIterator (concrete class) member functions.)
-// documentation: see DirectoryIterator
-struct IDirectoryIterator
-{
-	virtual ~IDirectoryIterator();
-	virtual LibError NextEntry(FilesystemEntry& fsEntry) = 0;
-};
-
-
-/**
- * flags controlling file IO and caching behavior.
- **/
-enum FileFlags
-{
-	// translate newlines: convert from/to native representation when
-	// reading/writing. this is useful if files we create need to be
-	// edited externally - e.g. Notepad requires \r\n.
-	// caveats:
-	// - FILE_NO_AIO must be set; translation is done by OS read()/write().
-	// - not supported by POSIX, so this currently only has meaning on Win32.
-	FILE_TEXT = 0x01,
-
-	// skip the aio path and use the OS-provided synchronous blocking
-	// read()/write() calls. this avoids the need for buffer alignment
-	// set out below, so it's useful for writing small text files.
-	// note: despite its more heavyweight operation, aio is still
-	// worthwhile for small files, so it is not automatically disabled.
-	FILE_NO_AIO = 0x02,
-
-	// do not add the (entire) contents of this file to the cache.
-	// this flag should be specified when the data is cached at a higher
-	// level (e.g. OpenGL textures) to avoid wasting precious cache space.
-	FILE_NO_CACHE = 0x04,
-
-	// enable caching individual blocks read from a file. the block cache
-	// is small, organized as LRU and incurs some copying overhead, so it
-	// should only be enabled when needed. this is the case for archives,
-	// where the cache absorbs overhead of block-aligning all IOs.
-	FILE_CACHE_BLOCK = 0x08,
-
-	// instruct file_open not to set FileCommon.atom_fn.
-	// this is a slight optimization used by VFS code: file_open
-	// would store the portable name, which is only used when calling
-	// the OS's open(); this would unnecessarily waste atom_fn memory.
-	//
-	// note: other file.cpp functions require atom_fn to be set,
-	// so this behavior is only triggered via flag (caller is
-	// promising they will set atom_fn).
-	FILE_DONT_SET_FN = 0x20,
-
-	// (only relevant for VFS) file will be written into the
-	// appropriate subdirectory of the mount point established by
-	// vfs_set_write_target. see documentation there.
-	FILE_WRITE_TO_TARGET = 0x40,
-
-	// sum of all flags above. used when validating flag parameters.
-	FILE_FLAG_ALL = 0x7F
 };
 
 
@@ -116,70 +61,11 @@ struct IFilesystem
 {
 	virtual ~IFilesystem();
 
-	/**
+	virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const = 0;
-	 * @return a single character identifying the filesystem.
-	 *
-	 * this is useful for VFS directory listings, where an indication is
-	 * made of where the file is actually stored.
-	 **/
-	virtual char IdentificationCode() const = 0;
 
-	/**
+	// note: this interface avoids having to lock a directory while an
-	 * @return a number that represents the precedence of this filesystem.
+	// iterator is extant.
-	 *
+	virtual LibError GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const = 0;
-	 * when mounting into the VFS, entries from a filesystem with higher
-	 * precedence override otherwise equivalent files.
-	 **/
-	virtual int Precedence() const = 0;
-
-	virtual LibError GetEntry(const char* pathname, FilesystemEntry& fsEntry) const = 0;
-
-	virtual LibError CreateDirectory(const char* dirPath) = 0;
-	virtual LibError DeleteDirectory(const char* dirPath) = 0;
-	virtual IDirectoryIterator* OpenDirectory(const char* dirPath) const = 0;
-
-	// note: only allowing either reads or writes simplifies file cache
-	// coherency (need only invalidate when closing a FILE_WRITE file).
-	virtual LibError CreateFile(const char* pathname, const u8* buf, size_t size, uint flags = 0) = 0;
-	virtual LibError DeleteFile(const char* pathname) = 0;
-
-	// read the entire file.
-	// return number of bytes transferred (see above), or a negative error code.
-	//
-	// if non-NULL, <cb> is called for each block transferred, passing <cbData>.
-	// it returns how much data was actually transferred, or a negative error
-	// code (in which case we abort the transfer and return that value).
-	// the callback mechanism is useful for user progress notification or
-	// processing data while waiting for the next I/O to complete
-	// (quasi-parallel, without the complexity of threads).
-	virtual LibError LoadFile(const char* pathname, const u8*& buf, size_t size, uint flags = 0, IoCallback cb = 0, uintptr_t cbData = 0) = 0;
-};
-
-
-
-/**
- * (mostly) insulating concrete class providing iterator access to
- * directory entries.
- * this is usable for posix, VFS, etc.; instances are created via IFilesystem.
- **/
-class DirectoryIterator
-{
-public:
-	DirectoryIterator(IFilesystem* fs, const char* dirPath)
-		: m_impl(fs->OpenDirectory(dirPath))
-	{
-	}
-
-	// return ERR::DIR_END if all entries have already been returned once,
-	// another negative error code, or INFO::OK on success, in which case <fsEntry>
-	// describes the next (order is unspecified) directory entry.
-	LibError NextEntry(FilesystemEntry& fsEntry)
-	{
-		return m_impl.get()->NextEntry(fsEntry);
-	}
-
-private:
-	boost::shared_ptr<IDirectoryIterator> m_impl;
 };
 
 #endif	// #ifndef INCLUDED_FILESYSTEM

source/lib/file/io/block_cache.cpp

@@ -25,9 +25,9 @@ BlockId::BlockId()
 
 BlockId::BlockId(const char* atom_fn, off_t ofs)
 {
-	debug_assert(ofs <= (u64)blockSize * 0xFFFFFFFF);	// ensure value fits in m_blockIndex
+	debug_assert(ofs <= (u64)BLOCK_SIZE * 0xFFFFFFFF);	// ensure value fits in m_blockIndex
 	m_atom_fn = atom_fn;	// unique (by definition)
-	m_blockIndex = (u32)(ofs / blockSize);
+	m_blockIndex = (u32)(ofs / BLOCK_SIZE);
 }
 
 bool BlockId::operator==(const BlockId& rhs) const
@@ -43,7 +43,16 @@ bool BlockId::operator!=(const BlockId& rhs) const
 
 //-----------------------------------------------------------------------------
 
-typedef LF_RefCountedMemRange Block;
+struct Block
+{
+	Block()
+	{
+		buf = io_buf_Allocate(BLOCK_SIZE);
+	}
+
+	IoBuf buf;
+	LF_ReferenceCounter refs;
+};
 
 class BlockManager
 {
@@ -51,14 +60,6 @@ public:
 	BlockManager(size_t numBlocks)
 		: m_ids(numBlocks), m_blocks(numBlocks), m_oldestIndex(0)
 	{
-		for(size_t i = 0; i < m_blocks.size(); i++)
-			m_blocks[i].mem = (void*)io_buf_Allocate(blockSize);
-	}
-
-	~BlockManager()
-	{
-		for(size_t i = 0; i < m_blocks.size(); i++)
-			io_buf_Deallocate((IoBuf)m_blocks[i].mem, blockSize);
 	}
 
 	// (linear search is ok since we only expect to manage a few blocks)
@@ -123,7 +124,7 @@ class BlockCache::Impl
 {
 public:
 	Impl(size_t cacheSize)
-		: m_blockManager(cacheSize / blockSize)
+		: m_blockManager(cacheSize / BLOCK_SIZE)
 	{
 	}
 
@@ -134,10 +135,10 @@ public:
 		Block* block = m_blockManager.AcquireOldestAvailableBlock(id);
 
 #if CONFIG_READ_ONLY_CACHE
-		mprotect(block->mem, block->size, PROT_WRITE|PROT_READ);
+		mprotect((void*)block->buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ);
 #endif
 
-		return (IoBuf)block->mem;
+		return block->buf;
 	}
 
 	void MarkComplete(BlockId id)
@@ -148,20 +149,21 @@ public:
 		block->refs.RelinquishExclusiveAccess();
 
 #if CONFIG_READ_ONLY_CACHE
-		mprotect(block.mem, block.size, PROT_READ);
+		mprotect((void*)block->buf.get(), BLOCK_SIZE, PROT_READ);
 #endif
 	}
 
-	IoBuf Retrieve(BlockId id)
+	bool Retrieve(BlockId id, IoBuf& buf)
 	{
 		Block* block = m_blockManager.Find(id);
 		if(!block)		// not found
-			return 0;
+			return false;
 
-		if(!block->refs.AddReference())	// contents are not yet valid
+		if(!block->refs.AddReference())	// contents are not yet valid (can happen due to multithreaded IOs)
-			return 0;					// (this can happen due to multithreaded IOs)
+			return false;
 
-		return (IoBuf)block->mem;
+		buf = block->buf;
+		return true;
 	}
 
 	void Release(BlockId id)
@@ -201,9 +203,9 @@ void BlockCache::MarkComplete(BlockId id)
 	impl.get()->Reserve(id);
 }
 
-IoBuf BlockCache::Retrieve(BlockId id)
+bool BlockCache::Retrieve(BlockId id, IoBuf& buf)
 {
-	return impl.get()->Retrieve(id);
+	return impl.get()->Retrieve(id, buf);
 }
 
 void BlockCache::Release(BlockId id)

source/lib/file/io/block_cache.h

@@ -11,7 +11,6 @@
 #ifndef INCLUDED_BLOCK_CACHE
 #define INCLUDED_BLOCK_CACHE
 
-#include <boost/shared_ptr.hpp>
 #include "io_buf.h"
 
 /**
@@ -82,12 +81,13 @@ public:
 	/**
 	 * Attempt to retrieve a block the file cache.
 	 *
-	 * @return 0 if not in cache or its IO is still pending, otherwise a
+	 * @return false if not in cache or its IO is still pending,
-	 * pointer to its (read-only) contents.
+	 * otherwise true.
 	 *
-	 * if successful, a reference is added to the block.
+	 * if successful, a reference is added to the block and its
+	 * buffer is returned.
 	 **/
-	IoBuf Retrieve(BlockId id);
+	bool Retrieve(BlockId id, IoBuf& buf);
 
 	/**
 	 * Indicate the block contents are no longer needed.
@@ -118,4 +118,4 @@ private:
 	boost::shared_ptr<Impl> impl;
 };
 
-#endif	 * #ifndef INCLUDED_BLOCK_CACHE
+#endif	// #ifndef INCLUDED_BLOCK_CACHE

source/lib/file/io/io_buf.cpp

@@ -2,27 +2,49 @@
 #include "io_buf.h"
 
 #include "lib/allocators/allocators.h"	// AllocatorChecker
+#include "lib/bits.h"	// round_up
+#include "block_cache.h"	// BLOCK_SIZE
+
 
 #ifndef NDEBUG
 static AllocatorChecker allocatorChecker;
 #endif
 
+
+class IoBufDeleter
+{
+public:
+	IoBufDeleter(size_t paddedSize)
+		: m_paddedSize(paddedSize)
+	{
+		debug_assert(m_paddedSize != 0);
+	}
+
+	void operator()(const u8* mem)
+	{
+		debug_assert(m_paddedSize != 0);
+#ifndef NDEBUG
+		allocatorChecker.notify_free((void*)mem, m_paddedSize);
+#endif
+		page_aligned_free((void*)mem, m_paddedSize);
+		m_paddedSize = 0;
+	}
+
+private:
+	size_t m_paddedSize;
+};
+
+
 IoBuf io_buf_Allocate(size_t size)
 {
-	void* p = page_aligned_alloc(size);
+	const size_t paddedSize = (size_t)round_up(size, BLOCK_SIZE);
-	if(!p)
+	const u8* mem = (const u8*)page_aligned_alloc(paddedSize);
+	if(!mem)
 		throw std::bad_alloc();
-#ifndef NDEBUG
-	allocatorChecker.notify_alloc(p, size);
-#endif
-	return (IoBuf)p;
-}
 
-void io_buf_Deallocate(IoBuf buf, size_t size)
-{
-	void* p = (void*)buf;
 #ifndef NDEBUG
-	allocatorChecker.notify_free(p, size);
+	allocatorChecker.notify_alloc((void*)mem, paddedSize);
 #endif
-	page_aligned_free(p, size);
+
+	return IoBuf(mem, IoBufDeleter(paddedSize));
 }

source/lib/file/io/io_buf.h

@@ -11,14 +11,12 @@
 #ifndef INCLUDED_IO_BUF
 #define INCLUDED_IO_BUF
 
-typedef const u8* IoBuf;
+typedef boost::shared_ptr<const u8> IoBuf;
-const IoBuf IO_BUF_TEMP = 0;
+const IoBuf IO_BUF_TEMP((const u8*)0);
-const IoBuf IO_BUF_ALLOC = (IoBuf)1;
 
 // memory will be allocated from the heap, not the (limited) file cache.
 // this makes sense for write buffers that are never used again,
 // because we avoid having to displace some other cached items.
 extern IoBuf io_buf_Allocate(size_t size);
-extern void io_buf_Deallocate(IoBuf buf, size_t size);
 
 #endif	// #ifndef INCLUDED_IO_BUF

source/lib/file/io/io_manager.cpp

@@ -11,8 +11,6 @@
 #include "precompiled.h"
 #include "io_manager.h"
 
-#include <boost/shared_ptr.hpp>
-
 #include "../posix/io_posix.h"
 #include "../file_stats.h"
 #include "block_cache.h"
@@ -48,12 +46,17 @@
 // (apparently since NTFS files are sector-padded anyway?)
 
 
-LibError io_InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed)
+// helper routine used by functions that call back to a IoCallback.
+//
+// bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE }
+// note: don't abort if = 0: zip callback may not actually
+// output anything if passed very little data.
+static LibError InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed)
 {
 	if(cb)
 	{
 		stats_cb_start();
-		LibError ret = cb(cbData, block, size, &bytesProcessed);
+		LibError ret = cb(cbData, block, size, bytesProcessed);
 		stats_cb_finish();
 
 		// failed - reset byte count in case callback didn't
@@ -78,7 +81,7 @@ class BlockIo
 {
 public:
 	BlockIo()
-		: m_blockId(), cachedBlock(0), tempBlock(0), m_posixIo()
+		: m_blockId(), m_cachedBlock(0), m_tempBlock(0), m_posixIo()
 	{
 	}
 
@@ -87,8 +90,7 @@ public:
 		m_blockId = BlockId(file.Pathname(), ofs);
 
 		// block already available in cache?
-		cachedBlock = s_blockCache.Retrieve(m_blockId);
+		if(s_blockCache.Retrieve(m_blockId, m_cachedBlock))
-		if(cachedBlock)
 		{
 			stats_block_cache(CR_HIT);
 			return INFO::OK;
@@ -99,16 +101,16 @@ public:
 
 		// use a temporary block if not writing to a preallocated buffer.
 		if(!buf)
-			buf = tempBlock = s_blockCache.Reserve(m_blockId);
+			buf = m_tempBlock = s_blockCache.Reserve(m_blockId);
 
 		return m_posixIo.Issue(file, ofs, buf, size);
 	}
 
 	LibError WaitUntilComplete(const u8*& block, size_t& blockSize)
 	{
-		if(cachedBlock)
+		block = m_cachedBlock.get();
+		if(block)
 		{
-			block = (u8*)cachedBlock;
 			blockSize = BLOCK_SIZE;
 			return INFO::OK;
 		}
@@ -118,17 +120,17 @@ public:
 
 	void Discard()
 	{
-		if(cachedBlock)
+		if(m_cachedBlock)
 		{
 			s_blockCache.Release(m_blockId);
-			cachedBlock = 0;
+			m_cachedBlock = 0;
 			return;
 		}
 
-		if(tempBlock)
+		if(m_tempBlock)
 		{
 			s_blockCache.MarkComplete(m_blockId);
-			tempBlock = 0;
+			m_tempBlock = 0;
 		}
 	}
 
@@ -136,9 +138,9 @@ private:
 	static BlockCache s_blockCache;
 
 	BlockId m_blockId;
-	IoBuf cachedBlock;
+	IoBuf m_cachedBlock;
 
-	IoBuf tempBlock;
+	IoBuf m_tempBlock;
 
 	Io_Posix m_posixIo;
 };
@@ -197,7 +199,7 @@ private:
 
 		// we have useable data from a previous temp buffer,
 		// but it needs to be copied into the user's buffer
-		if(blockIo.cachedBlock && pbuf != IO_BUF_TEMP)
+		if(blockIo.m_cachedBlock && pbuf != IO_BUF_TEMP)
 			cpu_memcpy((char*)*pbuf+ofs_misalign+m_totalTransferred, block, blockSize);
 
 		m_totalTransferred += blockSize;
@@ -350,3 +352,11 @@ size_t size;
 	//RingBuf<BlockIo, MAX_PENDING_IOS> queue;
 	std::deque<BlockIo> queue;
 };
+
+LibError io(File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb, uintptr_t cbData)
+{
+	debug_printf("IO| size=%d\n", size);
+
+
+	return ERR::NOT_IMPLEMENTED;
+}

source/lib/file/io/io_manager.h

@@ -11,16 +11,10 @@
 #ifndef INCLUDED_IO_MANAGER
 #define INCLUDED_IO_MANAGER
 
-#include "../posix/io_posix.h"
+class File_Posix;
-
-namespace ERR
-{
-	const LibError IO     = -110100;
-	const LibError IO_EOF = -110101;
-}
 
 // called by file_io after a block IO has completed.
-// *bytesProcessed must be set; file_io will return the sum of these values.
+// bytesProcessed must be set; file_io will return the sum of these values.
 // example: when reading compressed data and decompressing in the callback,
 // indicate #bytes decompressed.
 // return value: INFO::CB_CONTINUE to continue calling; anything else:
@@ -30,15 +24,12 @@ namespace ERR
 // for the entire IO. we do not split into fake blocks because it is
 // advantageous (e.g. for decompressors) to have all data at once, if available
 // anyway.
-typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t size, size_t* bytesProcessed);
+typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t size, size_t& bytesProcessed);
 
+extern LibError io_Read(const File_Posix& file, off_t ofs, u8* buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
+extern LibError io_Read(const File_Posix& file, off_t ofs, IoBuf buf, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
 
-// helper routine used by functions that call back to a IoCallback.
+extern LibError io_Write(File_Posix& file, off_t ofs, const u8* data, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
-//
+extern LibError io_Write(File_Posix& file, off_t ofs, IoBuf data, size_t size, IoCallback cb = 0, uintptr_t cbData = 0);
-// bytesProcessed is 0 if return value != { INFO::OK, INFO::CB_CONTINUE }
-// note: don't abort if = 0: zip callback may not actually
-// output anything if passed very little data.
-extern LibError io_InvokeCallback(const u8* block, size_t size, IoCallback cb, uintptr_t cbData, size_t& bytesProcessed);
-
 
 #endif	// #ifndef INCLUDED_IO_MANAGER

source/lib/file/path.cpp

@@ -2,7 +2,7 @@
  * =========================================================================
  * File        : path.cpp
  * Project     : 0 A.D.
- * Description : helper functions for VFS paths.
+ * Description : manage paths relative to a root directory
  * =========================================================================
  */
 
@@ -14,303 +14,117 @@
 #include <string.h>
 
 #include "lib/posix/posix_filesystem.h"
-#include "lib/adts.h"
+#include "lib/sysdep/sysdep.h"	// SYS_DIR_SEP
-#include "lib/rand.h"
+#include "lib/path_util.h"	// ERR::PATH_LENGTH
-#include "lib/allocators/pool.h"
-#include "lib/sysdep/sysdep.h"
-#include "lib/path_util.h"
-#include "lib/sysdep/cpu.h"		// cpu_memcpy
-#include "file_stats.h"
 
 
-ERROR_ASSOCIATE(ERR::ROOT_DIR_ALREADY_SET, "Attempting to set FS root dir more than once", -1);
+ERROR_ASSOCIATE(ERR::PATH_ROOT_DIR_ALREADY_SET, "Attempting to set FS root dir more than once", -1);
-ERROR_ASSOCIATE(ERR::NOT_IN_ROOT_DIR, "Accessing a file that's outside of the root dir", -1);
+ERROR_ASSOCIATE(ERR::PATH_NOT_IN_ROOT_DIR, "Accessing a file that's outside of the root dir", -1);
-
-
-
-// path types:
-// p_*: posix (e.g. mount object name or for open())
-// v_*: vfs (e.g. mount point)
-// fn : filename only (e.g. from readdir)
-// dir_name: directory only, no path (e.g. subdir name)
-//
-// all paths must be relative (no leading '/'); components are separated
-// by '/'; no ':', '\\', "." or ".." allowed; root dir is "".
-//
-// grammar:
-// path ::= dir*file?
-// dir  ::= name/
-// file ::= name
-// name ::= [^/]
-
-enum Conversion
-{
-	TO_NATIVE,
-	TO_PORTABLE
-};
-
-static LibError convert_path(char* dst, const char* src, Conversion conv = TO_NATIVE)
-{
-	// SYS_DIR_SEP is assumed to be a single character!
-
-	const char* s = src;
-	char* d = dst;
-
-	char from = SYS_DIR_SEP, to = '/';
-	if(conv == TO_NATIVE)
-		from = '/', to = SYS_DIR_SEP;
-
-	size_t len = 0;
-
-	for(;;)
-	{
-		len++;
-		if(len >= PATH_MAX)
-			WARN_RETURN(ERR::PATH_LENGTH);
-
-		char c = *s++;
-
-		if(c == from)
-			c = to;
-
-		*d++ = c;
-
-		// end of string - done
-		if(c == '\0')
-			return INFO::OK;
-	}
-}
 
 
 // set by path_SetRoot
-static char n_root_dir[PATH_MAX];
+static char osRootPath[PATH_MAX];
-static size_t n_root_dir_len;
+static size_t osRootPathLength;
 
-
+// security check: only allow path_SetRoot once so that malicious code
-// return the native equivalent of the given relative portable path
+// cannot circumvent the VFS checks that disallow access to anything above
-// (i.e. convert all '/' to the platform's directory separator)
+// the current directory (set here).
-// makes sure length < PATH_MAX.
+// path_SetRoot is called early at startup, so any subsequent attempts
-LibError file_make_native_path(const char* path, char* n_path)
-{
-	return convert_path(n_path, path, TO_NATIVE);
-}
-
-// return the portable equivalent of the given relative native path
-// (i.e. convert the platform's directory separators to '/')
-// makes sure length < PATH_MAX.
-LibError file_make_portable_path(const char* n_path, char* path)
-{
-	return convert_path(path, n_path, TO_PORTABLE);
-}
-
-
-// return the native equivalent of the given portable path
-// (i.e. convert all '/' to the platform's directory separator).
-// also prepends current directory => n_full_path is absolute.
-// makes sure length < PATH_MAX.
-LibError file_make_full_native_path(const char* path, char* n_full_path)
-{
-	debug_assert(path != n_full_path);	// doesn't work in-place
-
-	strcpy_s(n_full_path, PATH_MAX, n_root_dir);
-	return convert_path(n_full_path+n_root_dir_len, path, TO_NATIVE);
-}
-
-// return the portable equivalent of the given relative native path
-// (i.e. convert the platform's directory separators to '/')
-// n_full_path is absolute; if it doesn't match the current dir, fail.
-// (note: portable paths are always relative to the file root dir).
-// makes sure length < PATH_MAX.
-LibError file_make_full_portable_path(const char* n_full_path, char* path)
-{
-	debug_assert(path != n_full_path);	// doesn't work in-place
-
-	if(strncmp(n_full_path, n_root_dir, n_root_dir_len) != 0)
-		WARN_RETURN(ERR::NOT_IN_ROOT_DIR);
-	return convert_path(path, n_full_path+n_root_dir_len, TO_PORTABLE);
-}
-
-
-// security check: only allow attempting to chdir once, so that malicious
-// code cannot circumvent the VFS checks that disallow access to anything
-// above the current directory (set here).
-// this routine is called early at startup, so any subsequent attempts
 // are likely bogus.
 // we provide for resetting this from the self-test to allow clean
 // re-init of the individual tests.
-static bool root_dir_established;
+static bool isRootDirEstablished;
 
 
-// establish the root directory from <rel_path>, which is treated as
-// relative to the executable's directory (determined via argv[0]).
-// all relative file paths passed to this module will be based from
-// this root dir.
-//
-// example: executable in "$install_dir/system"; desired root dir is
-// "$install_dir/data" => rel_path = "../data".
-//
-// argv[0] is necessary because the current directory is unknown at startup
-// (e.g. it isn't set when invoked via batch file), and this is the
-// easiest portable way to find our install directory.
-//
-// can only be called once, by design (see below). rel_path is trusted.
 LibError path_SetRoot(const char* argv0, const char* rel_path)
 {
-	if(root_dir_established)
+	if(isRootDirEstablished)
-		WARN_RETURN(ERR::ROOT_DIR_ALREADY_SET);
+		WARN_RETURN(ERR::PATH_ROOT_DIR_ALREADY_SET);
-	root_dir_established = true;
+	isRootDirEstablished = true;
 
 	// get full path to executable
-	char n_path[PATH_MAX];
+	char osPathname[PATH_MAX];
 	// .. first try safe, but system-dependent version
-	if(sys_get_executable_name(n_path, PATH_MAX) < 0)
+	if(sys_get_executable_name(osPathname, PATH_MAX) < 0)
 	{
 		// .. failed; use argv[0]
-		if(!realpath(argv0, n_path))
+		errno = 0;
+		if(!realpath(argv0, osPathname))
 			return LibError_from_errno();
 	}
 
 	// make sure it's valid
-	if(access(n_path, X_OK) < 0)
+	errno = 0;
+	if(access(osPathname, X_OK) < 0)
 		return LibError_from_errno();
 
-	// strip executable name, append rel_path, convert to native
+	// strip executable name
-	char* start_of_fn = (char*)path_name_only(n_path);
+	char* name = (char*)path_name_only(osPathname);
-	RETURN_ERR(file_make_native_path(rel_path, start_of_fn));
+	*name = '\0';
 
-	// get actual root dir - previous n_path may include ".."
+	strcat_s(osPathname, ARRAY_SIZE(osRootPath), rel_path);
+
+	// get actual root dir - previous osPathname may include ".."
 	// (slight optimization, speeds up path lookup)
-	if(!realpath(n_path, n_root_dir))
+	errno = 0;
+	if(!realpath(osPathname, osRootPath))
 		return LibError_from_errno();
 
-	// .. append SYS_DIR_SEP to simplify code that uses n_root_dir
+	// .. append SYS_DIR_SEP to simplify code that uses osRootPath
-	n_root_dir_len = strlen(n_root_dir)+1;	// +1 for trailing SYS_DIR_SEP
+	osRootPathLength = strlen(osRootPath)+1;	// +1 for trailing SYS_DIR_SEP
-	debug_assert((n_root_dir_len+1) < sizeof(n_root_dir)); // Just checking
+	debug_assert((osRootPathLength+1) < sizeof(osRootPath)); // Just checking
-	n_root_dir[n_root_dir_len-1] = SYS_DIR_SEP;
+	osRootPath[osRootPathLength-1] = SYS_DIR_SEP;
-	// You might think that n_root_dir is already 0-terminated, since it's
+	// You might think that osRootPath is already 0-terminated, since it's
-	// static - but that might not be true after calling file_reset_root_dir!
+	// static - but that might not be true after calling path_ResetRootDir!
-	n_root_dir[n_root_dir_len] = 0;
+	osRootPath[osRootPathLength] = 0;
-	
+
 	return INFO::OK;
 }
 
 
 void path_ResetRootDir()
 {
-	// see comment at root_dir_established.
+	// see comment at isRootDirEstablished.
-	debug_assert(root_dir_established);
+	debug_assert(isRootDirEstablished);
-	n_root_dir[0] = '\0';
+	osRootPath[0] = '\0';
-	n_root_dir_len = 0;
+	osRootPathLength = 0;
-	root_dir_established = false;
+	isRootDirEstablished = false;
 }
 
 
-//-----------------------------------------------------------------------------
+// (this assumes SYS_DIR_SEP is a single character)
-// storage for path strings
+static void ConvertSlashCharacters(char* dst, const char* src, char from, char to)
-//-----------------------------------------------------------------------------
-
-// rationale: we want a constant-time IsAtomFn(string pointer) lookup:
-// this avoids any overhead of calling path_UniqueCopy on
-// already-atomized strings. that requires allocating from one contiguous
-// arena, which is also more memory-efficient than the heap (no headers).
-static Pool atom_pool;
-
-typedef DynHashTbl<const char*, const char*> AtomMap;
-static AtomMap atom_map;
-
-static void InitPool()
 {
-	static bool initialized = false;
+	for(size_t len = 0; len < PATH_MAX; len++)
-	if(initialized)
-		return;
-	pool_create(&atom_pool, 8*MiB, POOL_VARIABLE_ALLOCS);
-	initialized = true;
-
-	// TODO: we currently leak the pool. the following would have to run,
-	// but it's not clear when that would be possible+safe.
-	// atom_map.clear();
-	// (void)pool_destroy(&atom_pool);
-
-}
-
-bool path_is_atom_fn(const char* fn)
-{
-	InitPool();
-	return pool_contains(&atom_pool, (void*)fn);
-}
-
-// allocate a copy of P_fn in our string pool. strings are equal iff
-// their addresses are equal, thus allowing fast comparison.
-//
-// if the (generous) filename storage is full, 0 is returned.
-// this is not ever expected to happen; callers need not check the
-// return value because a warning is raised anyway.
-const char* path_UniqueCopy(const char* P_fn)
-{
-	InitPool();
-
-	// early out: if already an atom, return immediately.
-	if(path_is_atom_fn(P_fn))
-		return P_fn;
-
-	const size_t fn_len = strlen(P_fn);
-	const char* unique_fn;
-
-	// check if already allocated; return existing copy if so.
-	//
-	// rationale: the entire storage could be done via container,
-	// rather than simply using it as a lookup mapping.
-	// however, DynHashTbl together with Pool (see above) is more efficient.
-	unique_fn = atom_map.find(P_fn);
-	if(unique_fn)
-		return unique_fn;
-
-	unique_fn = (const char*)pool_alloc(&atom_pool, fn_len+1);
-	if(!unique_fn)
 	{
-		DEBUG_WARN_ERR(ERR::NO_MEM);
+		char c = *src++;
-		return 0;
+		if(c == from)
-	}
+			c = to;
-	cpu_memcpy((void*)unique_fn, P_fn, fn_len);
+		*dst++ = c;
-	((char*)unique_fn)[fn_len] = '\0';
 
-	atom_map.insert(unique_fn, unique_fn);
+		// end of string - done
-
+		if(c == '\0')
-	stats_unique_name(fn_len);
+			return;
-	return unique_fn;
-}
-
-
-const char* file_get_random_name()
-{
-	InitPool();
-
-	// there had better be names in atom_pool, else this will fail.
-	debug_assert(atom_pool.da.pos != 0);
-
-again:
-	const size_t start_ofs = (size_t)rand(0, (uint)atom_pool.da.pos);
-
-	// scan back to start of string (don't scan ahead; this must
-	// work even if atom_pool only contains one entry).
-	const char* start = (const char*)atom_pool.da.base+start_ofs;
-	for(size_t i = 0; i < start_ofs; i++)
-	{
-		if(*start == '\0')
-			break;
-		start--;
 	}
 
-	// skip past the '\0' we found. loop is needed because there may be
+	DEBUG_WARN_ERR(ERR::PATH_LENGTH);
-	// several if we land in padding (due to pool alignment).
+}
-	size_t chars_left = atom_pool.da.pos - start_ofs;
+
-	for(; *start == '\0'; start++)
+
-	{
+void path_MakeAbsolute(const char* path, char* osPath)
-		// we had landed in padding at the end of the buffer.
+{
-		if(chars_left-- == 0)
+	debug_assert(path != osPath);	// doesn't work in-place
-			goto again;
+
-	}
+	strcpy_s(osPath, PATH_MAX, osRootPath);
-
+	ConvertSlashCharacters(osPath+osRootPathLength, path, '/', SYS_DIR_SEP);
-	const char* next_name = start;
+}
-	return next_name;
+
+
+void path_MakeRelative(const char* osPath, char* path)
+{
+	debug_assert(path != osPath);	// doesn't work in-place
+
+	if(strncmp(osPath, osRootPath, osRootPathLength) != 0)
+		DEBUG_WARN_ERR(ERR::PATH_NOT_IN_ROOT_DIR);
+	ConvertSlashCharacters(path, osPath+osRootPathLength, SYS_DIR_SEP, '/');
 }

source/lib/file/path.h

@@ -2,82 +2,47 @@
  * =========================================================================
  * File        : path.h
  * Project     : 0 A.D.
- * Description : helper functions for VFS paths.
+ * Description : manage paths relative to a root directory
  * =========================================================================
  */
 
 // license: GPL; see lib/license.txt
 
+// path types:
+
+// tag  type      type      separator
+//      portable  relative  /
+// os   native    absolute  SYS_DIR_SEP
+// vfs  vfs       absolute  /
+
+// the vfs root directory is "". no ':', '\\', "." or ".." are allowed.
+
 #ifndef INCLUDED_PATH
 #define INCLUDED_PATH
 
-
 namespace ERR
 {
-	const LibError ROOT_DIR_ALREADY_SET = -110200;
+	const LibError PATH_ROOT_DIR_ALREADY_SET = -110200;
-	const LibError NOT_IN_ROOT_DIR      = -110201;
+	const LibError PATH_NOT_IN_ROOT_DIR      = -110201;
 }
 
-
+/**
-
+ * establish the root OS directory (portable paths are relative to it)
-
+ *
-// establish the root directory from <rel_path>, which is treated as
+ * @param argv0 the value of argv[0] (used to determine the location
-// relative to the executable's directory (determined via argv[0]).
+ * of the executable in case sys_get_executable_path fails). note that
-// all relative file paths passed to this module will be based from
+ * the current directory cannot be used because it's not set when
-// this root dir. 
+ * starting via batch file.
-//
+ * @param rel_path root directory relative to the executable's directory.
-// example: executable in "$install_dir/system"; desired root dir is
+ * the value is considered trusted since it will typically be hard-coded.
-// "$install_dir/data" => rel_path = "../data".
+ *
-//
+ * example: executable in "$install_dir/system"; desired root dir is
-// argv[0] is necessary because the current directory is unknown at startup
+ * "$install_dir/data" => rel_path = "../data".
-// (e.g. it isn't set when invoked via batch file), and this is the
+ *
-// easiest portable way to find our install directory.
+ * can only be called once unless path_ResetRootDir is called.
-//
+ **/
-// can only be called once, by design (see below). rel_path is trusted.
 extern LibError path_SetRoot(const char* argv0, const char* rel_path);
 
-
-//
-// path conversion functions (native <--> portable),
-// for external libraries that require the real filename.
-//
-// replaces '/' with platform's directory separator and vice versa.
-// verifies path length < PATH_MAX (otherwise return ERR::PATH_LENGTH).
-//
-
-// relative paths (relative to root dir)
-extern LibError file_make_native_path(const char* path, char* n_path);
-extern LibError file_make_portable_path(const char* n_path, char* path);
-
-// as above, but with full native paths (portable paths are always relative).
-// prepends current directory, resp. makes sure it matches the given path.
-extern LibError file_make_full_native_path(const char* path, char* n_full_path);
-extern LibError file_make_full_portable_path(const char* n_full_path, char* path);
-
-
-
-
-#define VFS_PATH_IS_DIR(path) (*path == '\0' || path[strlen(path)-1] == '/')
-
-
-extern bool path_is_atom_fn(const char* fn);
-
-
-
-// allocate a copy of P_fn in our string pool. strings are equal iff
-// their addresses are equal, thus allowing fast comparison.
-//
-// if the (generous) filename storage is full, 0 is returned.
-// this is not ever expected to happen; callers need not check the
-// return value because a warning is raised anyway.
-extern const char* path_UniqueCopy(const char* P_fn);
-
-extern const char* file_get_random_name();
-
-
-extern const char* file_get_random_name();
-
-
 /**
  * reset root directory that was previously established via path_SetRoot.
  *
@@ -87,4 +52,19 @@ extern const char* file_get_random_name();
  **/
 extern void path_ResetRootDir();
 
+
+/**
+ * return the absolute OS path for a given relative portable path.
+ *
+ * this is useful for external libraries that require the real filename.
+ **/
+extern void path_MakeAbsolute(const char* path, char* osPath);
+
+/**
+ * return the relative portable path for a given absolute OS path.
+ *
+ * this is useful when receiving paths from external libraries (e.g. FAM).
+ **/
+extern void path_MakeRelative(const char* osPath, char* path);
+
 #endif	// #ifndef INCLUDED_PATH

source/lib/file/posix/fs_posix.cpp

@@ -1,6 +1,6 @@
 /**
  * =========================================================================
- * File        : fp_posix.cpp
+ * File        : fs_posix.cpp
  * Project     : 0 A.D.
  * Description : file layer on top of POSIX. avoids the need for
  *             : absolute paths and provides fast I/O.
@@ -16,217 +16,167 @@
 #include <algorithm>
 #include <string>
 
-#include "../path.h"
+#include "lib/path_util.h"
+#include "lib/file/path.h"
 #include "lib/posix/posix_filesystem.h"
 
 
-//-----------------------------------------------------------------------------
+struct DirDeleter
-// DirectoryIterator_Posix
-//-----------------------------------------------------------------------------
-
-DirectoryIterator_Posix::DirectoryIterator_Posix(const char* P_path)
 {
-	char N_path[PATH_MAX];
+	void operator()(DIR* osDir) const
-	(void)file_make_full_native_path(P_path, N_path);
-	m_osDir = opendir(N_path);
-
-	// note: copying to N_path and then &m_pp.path is inefficient but
-	// more clear/robust. this is only called a few hundred times anyway.
-	(void)path_package_set_dir(&m_pp, N_path);
-}
-
-
-DirectoryIterator_Posix::~DirectoryIterator_Posix()
-{
-	if(m_osDir)
 	{
-		const int ret = closedir(m_osDir);
+		const int ret = closedir(osDir);
 		debug_assert(ret == 0);
 	}
+};
+
+// is name "." or ".."?
+static bool IsDummyDirectory(const char* name)
+{
+	if(name[0] != '.')
+		return false;
+	return (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'));
 }
 
-
+/*virtual*/ LibError Filesystem_Posix::GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories)
-LibError DirectoryIterator_Posix::NextEntry(FilesystemEntry& fsEntry)
 {
-	if(!m_osDir)
+	// open directory
-		return ERR::DIR_NOT_FOUND;
+	char osPath[PATH_MAX];
-
+	(void)path_MakeAbsolute(path, osPath);
-get_another_entry:
 	errno = 0;
-	struct dirent* osEnt = readdir(m_osDir);
+	boost::shared_ptr<DIR> osDir(opendir(osPath), DirDeleter());
-	if(!osEnt)
+	if(!osDir.get())
+		return LibError_from_errno();
+
+	// (we need absolute paths below; using this instead of path_append avoids
+	// a strlen call for each entry.)
+	PathPackage pp;
+	(void)path_package_set_dir(&pp, osPath);
+
+	for(;;)
 	{
-		// no error, just no more entries to return
+		errno = 0;
-		if(!errno)
+		struct dirent* osEnt = readdir(osDir.get());
-			return ERR::DIR_END;	// NOWARN
+		if(!osEnt)
-		return LibError_from_errno();
+		{
-	}
+			// no error, just no more entries to return
+			if(!errno)
+				return INFO::OK;
+			return LibError_from_errno();
+		}
 
-	// copying into the global filename storage avoids the need for users to
+		const char* name = path_Pool()->UniqueCopy(osEnt->d_name);
-	// free fsEntry.name and is convenient+safe.
+		RETURN_ERR(path_component_validate(name));
-	const char* atom_fn = path_UniqueCopy(osEnt->d_name);
 
-	// get file information (mode, size, mtime)
+		// get file information (mode, size, mtime)
-	struct stat s;
+		struct stat s;
 #if OS_WIN
-	// .. wposix readdir has enough information to return dirent
+		// .. wposix readdir has enough information to return dirent
-	//    status directly (much faster than calling stat).
+		//    status directly (much faster than calling stat).
-	RETURN_ERR(readdir_stat_np(m_osDir, &s));
+		RETURN_ERR(readdir_stat_np(osDir.get(), &s));
 #else
-	// .. call regular stat().
+		// .. call regular stat().
-	errno = 0;
+		errno = 0;
-	// (we need the full pathname; don't use path_append because it would
+		path_package_append_file(&pp, name);
-	// unnecessarily call strlen.)
+		if(stat(pp->path, &s) != 0)
-	path_package_append_file(&m_pp, atom_fn);
+			return LibError_from_errno();
-	if(stat(&m_pp->path, &s) != 0)
-		return LibError_from_errno();
 #endif
 
-	// skip "undesirable" entries that POSIX readdir returns:
+		if(files && S_ISREG(s.st_mode))
-	if(S_ISDIR(s.st_mode))
+			files->push_back(FileInfo(name, s.st_size, s.st_mtime));
-	{
+		else if(subdirectories && S_ISDIR(s.st_mode) && !IsDummyDirectory(name))
-		// .. dummy directory entries ("." and "..")
+			subdirectories->push_back(name);
-		if(atom_fn[0] == '.' && (atom_fn[1] == '\0' || (atom_fn[1] == '.' && atom_fn[2] == '\0')))
-			goto get_another_entry;
-
-		s.st_size = -1;	// our way of indicating it's a directory
 	}
-	// .. neither dir nor file
-	else if(!S_ISREG(s.st_mode))
-		goto get_another_entry;
-
-	fsEntry.size  = s.st_size;
-	fsEntry.mtime = s.st_mtime;
-	fsEntry.name  = atom_fn;
-	return INFO::OK;
 }
 
 
-//-----------------------------------------------------------------------------
+LibError Filesystem_Posix::GetFileInfo(const char* pathname, FileInfo& fileInfo) const
-// Filesystem_Posix
-//-----------------------------------------------------------------------------
-
-LibError Filesystem_Posix::GetEntry(const char* P_pathname, FilesystemEntry& fsEntry) const
 {
-	char N_pathname[PATH_MAX];
+	char osPathname[PATH_MAX];
-	RETURN_ERR(file_make_full_native_path(P_pathname, N_pathname));
+	RETURN_ERR(path_MakeAbsolute(pathname, osPathname));
 
 	// if path ends in slash, remove it (required by stat)
-	char* last_char = N_pathname+strlen(N_pathname)-1;
+	char* last_char = osPathname+strlen(osPathname)-1;
 	if(path_is_dir_sep(*last_char))
 		*last_char = '\0';
 
 	errno = 0;
 	struct stat s;
 	memset(&s, 0, sizeof(s));
-	if(stat(N_pathname, &s) != 0)
+	if(stat(osPathname, &s) != 0)
 		return LibError_from_errno();
 
-	fsEntry.size  = s.st_size;
+	const char* name = path_Pool()->UniqueCopy(path_name_only(osPathname));
-	fsEntry.mtime = s.st_mtime;
+	fileInfo = FileInfo(name, s.st_size, s.st_mtime);
-	fsEntry.name = path_UniqueCopy(path_name_only(N_pathname));
-	fsEntry.mount = 0;
 	return INFO::OK;
 }
 
 
-LibError Filesystem_Posix::CreateDirectory(const char* P_dirPath)
+LibError Filesystem_Posix::DeleteFile(const char* pathname)
 {
-	char N_dirPath[PATH_MAX];
+	char osPathname[PATH_MAX+1];
-	RETURN_ERR(file_make_full_native_path(P_dirPath, N_dirPath));
+	RETURN_ERR(path_MakeAbsolute(pathname, osPathname));
+
+	errno = 0;
+	if(unlink(osPathname) != 0)
+		return LibError_from_errno();
+
+	return INFO::OK;
+}
+
+
+LibError Filesystem_Posix::CreateDirectory(const char* path)
+{
+	char osPath[PATH_MAX];
+	RETURN_ERR(path_MakeAbsolute(path, osPath));
 
 	errno = 0;
 	struct stat s;
-	if(stat(N_dirPath, &s) != 0)
+	if(stat(osPath, &s) != 0)
 		return LibError_from_errno();
 
 	errno = 0;
-	if(mkdir(N_dirPath, S_IRWXO|S_IRWXU|S_IRWXG) != 0)
+	if(mkdir(osPath, S_IRWXO|S_IRWXU|S_IRWXG) != 0)
 		return LibError_from_errno();
 
 	return INFO::OK;
 }
 
 
-LibError Filesystem_Posix::DeleteDirectory(const char* P_dirPath)
+LibError Filesystem_Posix::DeleteDirectory(const char* path)
 {
 	// note: we have to recursively empty the directory before it can
 	// be deleted (required by Windows and POSIX rmdir()).
 
-	char N_dirPath[PATH_MAX];
+	char osPath[PATH_MAX];
-	RETURN_ERR(file_make_full_native_path(P_dirPath, N_dirPath));
+	RETURN_ERR(path_MakeAbsolute(path, osPath));
-	PathPackage N_pp;
+	PathPackage pp;
-	RETURN_ERR(path_package_set_dir(&N_pp, N_dirPath));
+	RETURN_ERR(path_package_set_dir(&pp, osPath));
 
+	std::vector<FileInfo> files;
+	std::vector<const char*> subdirectories;
+	RETURN_ERR(GetDirectoryEntries(path, &files, &subdirectories));
+
+	// delete files
+	for(size_t i = 0; i < files.size(); i++)
 	{
-		// (must go out of scope before rmdir)
+		RETURN_ERR(path_package_append_file(&pp, files[i].Name()));
-		DirectoryIterator_Posix di(P_dirPath);
+		errno = 0;
-		for(;;)
+		if(unlink(pp.path) != 0)
-		{
+			return LibError_from_errno();
-			FilesystemEntry fsEntry;
+	}
-			LibError err = di.NextEntry(fsEntry);
-			if(err == ERR::DIR_END)
-				break;
-			RETURN_ERR(err);
 
-			if(fsEntry.IsDirectory())
+	// recurse over subdirectories
-			{
+	for(size_t i = 0; i < subdirectories.size(); i++)
-				char P_subdirPath[PATH_MAX];
+	{
-				RETURN_ERR(path_append(P_subdirPath, P_dirPath, fsEntry.name));
+		char subdirectoryPath[PATH_MAX];
-				RETURN_ERR(DeleteDirectory(P_subdirPath));
+		path_append(subdirectoryPath, path, subdirectories[i]);
-			}
+		RETURN_ERR(DeleteDirectory(subdirectoryPath));
-			else
-			{
-				RETURN_ERR(path_package_append_file(&N_pp, fsEntry.name));
-
-				errno = 0;
-				if(unlink(N_pp.path) != 0)
-					return LibError_from_errno();
-			}
-		}
 	}
 
 	errno = 0;
-	if(rmdir(N_dirPath) != 0)
+	if(rmdir(osPath) != 0)
 		return LibError_from_errno();
 
 	return INFO::OK;
 }
-
-
-IDirectoryIterator* Filesystem_Posix::OpenDirectory(const char* P_dirPath) const
-{
-	return new DirectoryIterator_Posix(P_dirPath);
-}
-
-
-LibError CreateFile(const char* P_pathname, const u8* buf, size_t size, uint flags = 0)
-{
-	File_Posix file;
-	RETURN_ERR(file.Open(P_pathname, 'w', flags));
-	RETURN_ERR(io(file, 0, buf, size));
-	return INFO::OK;
-}
-
-
-LibError Filesystem_Posix::DeleteFile(const char* P_pathname)
-{
-	char N_pathname[PATH_MAX+1];
-	RETURN_ERR(file_make_full_native_path(P_pathname, N_pathname));
-
-	errno = 0;
-	if(unlink(N_pathname) != 0)
-		return LibError_from_errno();
-
-	return INFO::OK;
-}
-
-
-LibError LoadFile(const char* P_pathname, const u8*& buf, size_t size, uint flags = 0, IoCallback cb = 0, uintptr_t cbData = 0)
-{
-	File_Posix file;
-	RETURN_ERR(file.Open(P_pathname, 'r', flags));
-	RETURN_ERR(io(file, 0, buf, size, cb, cbData));
-
-	return INFO::OK;
-}

source/lib/file/posix/fs_posix.h

@@ -1,6 +1,6 @@
 /**
  * =========================================================================
- * File        : fp_posix.h
+ * File        : fs_posix.h
  * Project     : 0 A.D.
  * Description : file layer on top of POSIX. avoids the need for
  *             : absolute paths and provides fast I/O.
@@ -12,49 +12,16 @@
 #ifndef INCLUDED_FS_POSIX
 #define INCLUDED_FS_POSIX
 
-#include "../filesystem.h"
+#include "lib/file/filesystem.h"
-
-#include "lib/path_util.h"	// PathPackage
-#include "../io/io_manager.h"	// IoCallback
-
-// layer on top of POSIX opendir/readdir/closedir that converts paths to
-// the native representation and ignores non-file/directory entries.
-class DirectoryIterator_Posix : public IDirectoryIterator
-{
-public:
-	DirectoryIterator_Posix(const char* P_path);
-	virtual ~DirectoryIterator_Posix();
-	virtual LibError NextEntry(FilesystemEntry& fsEntry);
-
-private:
-	DIR* m_osDir;
-
-	PathPackage m_pp;
-};
-
 
 struct Filesystem_Posix : public IFilesystem
 {
-	virtual char IdentificationCode() const
+	virtual LibError GetFileInfo(const char* pathname, FileInfo& fileInfo) const;
-	{
+	virtual LibError GetDirectoryEntries(const char* path, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
-		return 'F';
-	}
 
-	virtual int Precedence() const
+	LibError DeleteFile(const char* pathname);
-	{
+	LibError CreateDirectory(const char* dirPath);
-		return 1;
+	LibError DeleteDirectory(const char* dirPath);
-	} 
-
-	virtual LibError GetEntry(const char* pathname, FilesystemEntry& fsEntry) const = 0;
-
-	virtual LibError CreateDirectory(const char* dirPath);
-	virtual LibError DeleteDirectory(const char* dirPath);
-	virtual IDirectoryIterator* OpenDirectory(const char* dirPath) const;
-
-	virtual LibError CreateFile(const char* pathname, const u8* buf, size_t size, uint flags = 0);
-	virtual LibError DeleteFile(const char* pathname);
-	virtual ssize_t LoadFile(const char* pathname, IoBuf& buf, uint flags = 0, IoCallback cb = 0, uintptr_t cbData = 0);
 };
 
-
 #endif	// #ifndef INCLUDED_FS_POSIX

source/lib/file/posix/io_posix.cpp

@@ -30,34 +30,22 @@ File_Posix::~File_Posix()
 }
 
 
-LibError File_Posix::Open(const char* P_pathname, char mode, uint flags)
+LibError File_Posix::Open(const char* P_pathname, char mode)
 {
 	debug_assert(mode == 'w' || mode == 'r');
-	debug_assert(flags <= FILE_FLAG_ALL);
 
-	m_pathname = path_UniqueCopy(P_pathname);
+	m_pathname = path_Pool()->UniqueCopy(P_pathname);
 	m_mode = mode;
-	m_flags = flags;
 
 	char N_pathname[PATH_MAX];
-	(void)file_make_full_native_path(P_pathname, N_pathname);
+	(void)path_MakeAbsolute(P_pathname, N_pathname);
 
 	int oflag = (mode == 'r')? O_RDONLY : O_WRONLY|O_CREAT|O_TRUNC;
-
-#if OS_WIN
-	if(flags & FILE_TEXT)
-		oflag |= O_TEXT_NP;
-	else
-		oflag |= O_BINARY_NP;
-
-	// if AIO is disabled at user's behest, inform wposix.
-	if(flags & FILE_NO_AIO)
-		oflag |= O_NO_AIO_NP;
-#endif
-
 	m_fd = open(N_pathname, oflag, S_IRWXO|S_IRWXU|S_IRWXG);
 	if(m_fd < 0)
-		RETURN_ERR(ERR::FILE_ACCESS);
+		WARN_RETURN(ERR::FILE_ACCESS);
+
+	stats_open(m_pathname, Size());
 
 	return INFO::OK;
 }
@@ -72,15 +60,13 @@ void File_Posix::Close()
 
 LibError File_Posix::Validate() const
 {
-	if(path_UniqueCopy(m_pathname) != m_pathname)
+	if(path_Pool()->UniqueCopy(m_pathname) != m_pathname)
 		WARN_RETURN(ERR::_1);
 	if((m_mode != 'w' && m_mode != 'r'))
 		WARN_RETURN(ERR::_2);
-	if(m_flags > FILE_FLAG_ALL)
-		WARN_RETURN(ERR::_3);
 	// >= 0x100 is not necessarily bogus, but suspicious.
 	if(!(3 <= m_fd && m_fd < 0x100))
-		WARN_RETURN(ERR::_4);
+		WARN_RETURN(ERR::_3);
 
 	return INFO::OK;
 }

source/lib/file/posix/io_posix.h

@@ -11,8 +11,6 @@
 #ifndef INCLUDED_IO_POSIX
 #define INCLUDED_IO_POSIX
 
-#include <boost/shared_ptr.hpp>
-
 #include "../io/io_buf.h"
 
 // rationale for using aio instead of mmap:
@@ -64,11 +62,17 @@
 //   idle time to satisfy potential future IOs) requires extra buffers;
 //   this is a bit more complicated than just using the cache as storage.
 
+namespace ERR
+{
+	const LibError FILE_ACCESS = -110200;
+	const LibError IO          = -110201;
+	const LibError IO_EOF      = -110202;
+}
 
 namespace INFO
 {
-	const LibError IO_PENDING  = +110200;
+	const LibError IO_PENDING  = +110203;
-	const LibError IO_COMPLETE = +110201;
+	const LibError IO_COMPLETE = +110204;
 }
 
 
@@ -80,7 +84,7 @@ public:
 	File_Posix();
 	~File_Posix();
 
-	LibError Open(const char* pathname, char mode, uint flags);
+	LibError Open(const char* pathname, char mode);
 	void Close();
 
 	const char* Pathname() const
@@ -93,11 +97,6 @@ public:
 		return m_mode;
 	}
 
-	uint Flags() const
-	{
-		return m_flags;
-	}
-
 	int Handle() const
 	{
 		return m_fd;
@@ -108,7 +107,6 @@ public:
 private:
 	const char* m_pathname;
 	char m_mode;
-	uint m_flags;
 	int m_fd;
 };
 

source/lib/file/vfs/file_cache.cpp

@@ -0,0 +1,233 @@
+/**
+ * =========================================================================
+ * File        : file_cache.cpp
+ * Project     : 0 A.D.
+ * Description : cache of file contents (supports zero-copy IO)
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "file_cache.h"
+
+#include "path.h"
+#include "file_stats.h"
+#include "archive/trace.h"
+#include "lib/cache_adt.h"				// Cache
+#include "lib/bits.h"					// round_up
+#include "lib/allocators/allocators.h"
+#include "lib/allocators/headerless.h"
+#include "lib/allocators/mem_util.h"	// mem_PageSize
+
+
+//-----------------------------------------------------------------------------
+// allocator
+
+/*
+the biggest worry of a file cache is external fragmentation. there are two
+basic ways to combat this:
+1) 'defragment' periodically - move blocks around to increase
+size of available 'holes'.
+2) prevent fragmentation from occurring at all via
+deliberate alloc/free policy.
+
+file contents are returned directly to the user (zero-copy IO), so only
+currently unreferenced blocks can be moved. it is believed that this would
+severely hamper defragmentation; we therefore go with the latter approach.
+
+the basic insight is: fragmentation occurs when a block is freed whose
+neighbors are not free (thus preventing coalescing). this can be prevented by
+allocating objects of similar lifetimes together. typical workloads
+(uniform access frequency) already show such behavior: the Landlord cache
+manager evicts files in an LRU manner, which matches the allocation policy.
+
+references:
+"The Memory Fragmentation Problem - Solved?" (Johnstone and Wilson)
+"Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson)
+*/
+
+class Allocator;
+
+class Deleter
+{
+public:
+	Deleter(size_t size, Allocator* allocator, const char* owner)
+		: m_size(size), m_allocator(allocator), m_owner(owner)
+	{
+	}
+
+	// (this must come after Allocator because it calls Deallocate())
+	void operator()(const u8* data) const;
+
+private:
+	size_t m_size;
+	Allocator* m_allocator;
+	const char* m_owner;
+};
+
+// >= sys_max_sector_size or else waio will have to realign.
+// chosen as exactly 1 page: this allows write-protecting file buffers
+// without worrying about their (non-page-aligned) borders.
+// internal fragmentation is considerable but acceptable.
+static const size_t alignment = mem_PageSize();
+
+class Allocator
+{
+public:
+	Allocator(size_t maxSize)
+		: m_allocator(maxSize)
+	{
+	}
+
+	FileCacheData Allocate(size_t size, const char* owner)
+	{
+		const size_t alignedSize = round_up(size, alignment);
+		stats_buf_alloc(size, alignedSize);
+
+		const u8* data = (const u8*)m_allocator.Allocate(alignedSize);
+#ifndef NDEBUG
+		m_checker.notify_alloc((void*)data, alignedSize);
+#endif
+
+		return FileCacheData(data, Deleter(size, this, owner));
+	}
+
+	void Deallocate(const u8* data, size_t size, const char* owner)
+	{
+		void* const p = (void*)data;
+		const size_t alignedSize = round_up(size, alignment);
+
+		// (re)allow writes. it would be nice to un-map the buffer, but this is
+		// not possible because HeaderlessAllocator needs to affix boundary tags.
+		(void)mprotect(p, size, PROT_READ|PROT_WRITE);
+
+#ifndef NDEBUG
+		m_checker.notify_free(p, alignedSize);
+#endif
+		m_allocator.Deallocate(p, alignedSize);
+
+		stats_buf_free();
+		trace_notify_free(owner);
+	}
+
+private:
+	HeaderlessAllocator m_allocator;
+
+#ifndef NDEBUG
+	AllocatorChecker m_checker;
+#endif
+};
+
+
+void Deleter::operator()(const u8* data) const
+{
+	m_allocator->Deallocate(data, m_size, m_owner);
+}
+
+
+//-----------------------------------------------------------------------------
+// FileCache::Impl
+//-----------------------------------------------------------------------------
+
+// since users are strongly encouraged to only load/process one file at a
+// time, there won't be many active references to cache entries. we could
+// take advantage of this with a separate extant list, but the cache's
+// hash map should be fast enough and this way is less work than maintaining
+// (possibly disjunct) cached and extant lists.
+
+class FileCache::Impl
+{
+public:
+	Impl(size_t size)
+		: m_allocator(size)
+	{
+	}
+
+	FileCacheData Reserve(const char* vfsPathname, size_t size)
+	{
+		// (this probably indicates a bug; caching 0-length files would
+		// have no benefit, anyway)
+		debug_assert(size != 0);
+
+		// (300 iterations have been observed when reserving several MB
+		// of space in a full cache)
+		for(;;)
+		{
+			FileCacheData data = m_allocator.Allocate(size, vfsPathname);
+			if(data.get())
+				return data;
+
+			// remove least valuable entry from cache (if users are holding
+			// references, the contents won't actually be deallocated)
+			FileCacheData discardedData; size_t discardedSize;
+			bool removed = m_cache.remove_least_valuable(&discardedData, &discardedSize);
+			// only false if cache is empty, which can't be the case because
+			// allocation failed.
+			debug_assert(removed);
+		}
+	}
+
+	void Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost)
+	{
+		// zero-copy cache => all users share the contents => must not
+		// allow changes. this will be reverted when deallocating.
+		(void)mprotect((void*)data.get(), size, PROT_READ);
+
+		m_cache.add(vfsPathname, data, size, cost);
+	}
+
+	bool Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size)
+	{
+		// (note: don't call stats_cache because we don't know the file size
+		// in case of a cache miss; doing so is left to the caller.)
+		stats_buf_ref();
+
+		return m_cache.retrieve(vfsPathname, data, &size);
+	}
+
+	void Remove(const char* vfsPathname)
+	{
+		m_cache.remove(vfsPathname);
+
+		// note: we could check if someone is still holding a reference
+		// to the contents, but that currently doesn't matter.
+	}
+
+private:
+	// HACK: due to vfsPathname, we are assured that strings are equal iff their
+	// addresses match. however, Cache's STL (hash_)map stupidly assumes that
+	// const char* keys are "strings". to avoid this behavior, we specify the
+	// key as const void*.
+	static Cache<const void*, FileCacheData> m_cache;
+
+	Allocator m_allocator;
+};
+
+
+//-----------------------------------------------------------------------------
+
+FileCache::FileCache(size_t size)
+	: impl(new Impl(size))
+{
+}
+
+FileCacheData FileCache::Reserve(const char* vfsPathname, size_t size)
+{
+	return impl.get()->Reserve(vfsPathname, size);
+}
+
+void FileCache::Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost)
+{
+	impl.get()->Add(vfsPathname, data, size, cost);
+}
+
+void FileCache::Remove(const char* vfsPathname)
+{
+	impl.get()->Remove(vfsPathname);
+}
+
+bool FileCache::Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size)
+{
+	return impl.get()->Retrieve(vfsPathname, data, size);
+}

source/lib/file/vfs/file_cache.h

@@ -0,0 +1,91 @@
+/**
+ * =========================================================================
+ * File        : file_cache.h
+ * Project     : 0 A.D.
+ * Description : cache of file contents (supports zero-copy IO)
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_FILE_CACHE
+#define INCLUDED_FILE_CACHE
+
+typedef boost::shared_ptr<const u8> FileCacheData;
+
+/**
+ * cache of file contents with support for zero-copy IO.
+ * this works by reserving a region of the cache, using it as the IO buffer,
+ * and returning the memory directly to users. optional write-protection
+ * via MMU ensures that the shared contents aren't inadvertently changed.
+ *
+ * (unique copies of) VFS pathnames are used as lookup key and owner tag.
+ *
+ * to ensure efficient operation and prevent fragmentation, only one
+ * reference should be active at a time. in other words, read a file,
+ * process it, and only then start reading the next file.
+ *
+ * rationale: this is rather similar to BlockCache; however, the differences
+ * (Reserve's size parameter, Add vs. MarkComplete, different eviction
+ * policies) are enough to warrant separate implementations.
+ **/
+class FileCache
+{
+public:
+	/**
+	 * @param size maximum amount [bytes] of memory to use for the cache.
+	 * (managed as a virtual memory region that's committed on-demand)
+	 **/
+	FileCache(size_t size);
+
+	/**
+	 * Reserve a chunk of the cache's memory region.
+	 *
+	 * @param vfsPathname pathname of the file that is to be read; this is
+	 * the key that will be used to Retrieve the file contents.
+	 * @param size required number of bytes (more may be allocated due to
+	 * alignment and/or internal fragmentation)
+	 * @return memory suitably aligned for IO; never fails.
+	 *
+	 * it is expected that this data will be Add()-ed once its IO completes.
+	 **/
+	FileCacheData Reserve(const char* vfsPathname, size_t size);
+
+	/**
+	 * Add a file's contents to the cache.
+	 *
+	 * the cache will be able to satisfy subsequent Retrieve() calls by
+	 * returning this data; if CONFIG_READ_ONLY_CACHE, the buffer is made
+	 * read-only. if need be and no references are currently attached to it,
+	 * the memory can also be commandeered by Reserve().
+	 *
+	 * @param cost is the expected cost of retrieving the file again and
+	 * influences how/when it is evicted from the cache.
+	 **/
+	void Add(const char* vfsPathname, FileCacheData data, size_t size, uint cost = 1);
+
+	/**
+	 * Remove a file's contents from the cache (if it exists).
+	 *
+	 * this ensures subsequent reads of the files see the current, presumably
+	 * recently changed, contents of the file.
+	 *
+	 * this would typically be called in response to a notification that a
+	 * file has changed.
+	 **/
+	void Remove(const char* vfsPathname);
+
+	/**
+	 * Attempt to retrieve a file's contents from the file cache.
+	 *
+	 * @return whether the contents were successfully retrieved; if so,
+	 * data references the read-only file contents.
+	 **/
+	bool Retrieve(const char* vfsPathname, FileCacheData& data, size_t& size);
+
+private:
+	class Impl;
+	boost::shared_ptr<Impl> impl;
+};
+
+#endif	// #ifndef INCLUDED_FILE_CACHE

source/lib/file/vfs/vfs.cpp

@@ -0,0 +1,185 @@
+/**
+ * =========================================================================
+ * File        : vfs.cpp
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "vfs.h"
+
+#include "lib/path_util.h"
+#include "lib/file/file_stats.h"
+#include "file_cache.h"
+#include "vfs_tree.h"
+#include "vfs_path.h"
+#include "vfs_mount.h"
+
+
+class Filesystem_VFS::Impl : public IFilesystem
+{
+public:
+	Impl()
+		: m_fileCache(ChooseCacheSize())
+	{
+	}
+
+	~Impl()
+	{
+		trace_shutdown();
+		mount_shutdown();
+	}
+
+	LibError Mount(const char* vfsPath, const char* path, uint flags /* = 0 */, uint priority /* = 0 */)
+	{
+
+	}
+
+	void Unmount(const char* path)
+	{
+
+	}
+
+	virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const
+	{
+		VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
+		if(!file)
+			return ERR::VFS_FILE_NOT_FOUND;	// NOWARN
+		file->GetFileInfo(fileInfo);
+		return INFO::OK;
+	}
+
+	virtual LibError GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
+	{
+		VfsDirectory* directory = LookupDirectory(vfsPath, &m_tree.Root());
+		if(!directory)
+			WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
+		directory->GetEntries(files, subdirectories);
+		return INFO::OK;
+	}
+
+	// note: only allowing either reads or writes simplifies file cache
+	// coherency (need only invalidate when closing a FILE_WRITE file).
+	LibError CreateFile(const char* vfsPathname, const u8* buf, size_t size)
+	{
+		VfsDirectory* directory = LookupDirectory(vfsPathname, &m_tree.Root());
+		if(!directory)
+			WARN_RETURN(ERR::VFS_DIR_NOT_FOUND);
+		const char* name = path_name_only(vfsPathname);
+		directory->CreateFile(name, buf, size);
+
+		// wipe out any cached blocks. this is necessary to cover the (rare) case
+		// of file cache contents predating the file write.
+		m_fileCache.Remove(vfsPathname);
+	}
+
+	// read the entire file.
+	// return number of bytes transferred (see above), or a negative error code.
+	//
+	// if non-NULL, <cb> is called for each block transferred, passing <cbData>.
+	// it returns how much data was actually transferred, or a negative error
+	// code (in which case we abort the transfer and return that value).
+	// the callback mechanism is useful for user progress notification or
+	// processing data while waiting for the next I/O to complete
+	// (quasi-parallel, without the complexity of threads).
+	LibError LoadFile(const char* vfsPathname, FileContents& contents, size_t& size)
+	{
+		vfsPathname = path_Pool()->UniqueCopy(vfsPathname);
+		debug_printf("VFS| load %s\n", vfsPathname);
+
+		const bool isCacheHit = m_fileCache.Retrieve(vfsPathname, contents, size);
+		if(!isCacheHit)
+		{
+			VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
+			if(!file)
+				WARN_RETURN(ERR::VFS_FILE_NOT_FOUND);
+			contents = m_fileCache.Reserve(vfsPathname, file->Size());
+			RETURN_ERR(file->Load((u8*)contents.get()));
+			m_fileCache.Add(vfsPathname, contents, size);
+		}
+
+		stats_io_user_request(size);
+		stats_cache(isCacheHit? CR_HIT : CR_MISS, size, vfsPathname);
+		trace_notify_io(vfsPathname, size);
+
+		return INFO::OK;
+	}
+
+	void RefreshFileInfo(const char* pathname)
+	{
+		//VfsFile* file = LookupFile(vfsPathname, &m_tree.Root());
+	}
+
+	void Display()
+	{
+		m_tree.Display();
+	}
+
+private:
+	static size_t ChooseCacheSize()
+	{
+		return 96*MiB;
+	}
+
+	void Rebuild()
+	{
+		m_tree.Clear();
+		m_mounts.RedoAll();
+	}
+
+	VfsTree m_tree;
+	FileCache m_fileCache;
+};
+
+//-----------------------------------------------------------------------------
+
+Filesystem_VFS::Filesystem_VFS(void* trace)
+{
+}
+
+/*virtual*/ Filesystem_VFS::~Filesystem_VFS()
+{
+}
+
+/*virtual*/ LibError Filesystem_VFS::GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const
+{
+	return impl.get()->GetFileInfo(vfsPathname, fileInfo);
+}
+
+/*virtual*/ LibError Filesystem_VFS::GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
+{
+	return impl.get()->GetDirectoryEntries(vfsPath, files, subdirectories);
+}
+
+LibError Filesystem_VFS::CreateFile(const char* vfsPathname, const u8* data, size_t size)
+{
+	return impl.get()->CreateFile(vfsPathname, data, size);
+}
+
+LibError Filesystem_VFS::LoadFile(const char* vfsPathname, FileContents& contents, size_t& size)
+{
+	return impl.get()->LoadFile(vfsPathname, contents, size);
+}
+
+LibError Filesystem_VFS::Mount(const char* vfsPath, const char* path, uint flags, uint priority)
+{
+	return impl.get()->Mount(vfsPath, path, flags, priority);
+}
+
+void Filesystem_VFS::Unmount(const char* path)
+{
+	return impl.get()->Unmount(path);
+}
+
+void Filesystem_VFS::RefreshFileInfo(const char* pathname)
+{
+	impl.get()->RefreshFileInfo(pathname);
+}
+
+void Filesystem_VFS::Display() const
+{
+	impl.get()->Display();
+}

source/lib/file/vfs/vfs.h

@@ -0,0 +1,78 @@
+/**
+ * =========================================================================
+ * File        : vfs.h
+ * Project     : 0 A.D.
+ * Description : Virtual File System API - allows transparent access to
+ *             : files in archives and modding via multiple mount points.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_VFS
+#define INCLUDED_VFS
+
+#include "lib/file/filesystem.h"
+
+namespace ERR
+{
+	const LibError VFS_DIR_NOT_FOUND  = -110100;
+	const LibError VFS_FILE_NOT_FOUND = -110101;
+}
+
+typedef boost::shared_ptr<const u8> FileContents;
+
+class Filesystem_VFS : public IFilesystem
+{
+public:
+	Filesystem_VFS(void* trace);
+
+	/**
+	 * mount a directory into the VFS.
+	 *
+	 * @param vfsPath mount point (created if it does not already exist)
+	 * @param path real directory path
+	 *
+	 * if files are encountered that already exist in the VFS (sub)directories,
+	 * the most recent / highest priority/precedence version is preferred.
+	 *
+	 * the contents of archives in this directory (but not its subdirectories!)
+	 * are added as well; they are processed in alphabetical order.
+	 **/
+	LibError Mount(const char* vfsPath, const char* path, uint flags = 0, uint priority = 0);
+
+	/**
+	 * unmount a previously mounted path and rebuild the VFS afterwards.
+	 **/
+	void Unmount(const char* path);
+
+	// (see IFilesystem)
+	virtual ~Filesystem_VFS();
+	virtual LibError GetFileInfo(const char* vfsPathname, FileInfo& fileInfo) const;
+	virtual LibError GetDirectoryEntries(const char* vfsPath, std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
+
+	// note: only allowing either reads or writes simplifies file cache
+	// coherency (need only invalidate when closing a FILE_WRITE file).
+	LibError CreateFile(const char* vfsPathname, const u8* data, size_t size);
+
+	// read the entire file.
+	// return number of bytes transferred (see above), or a negative error code.
+	//
+	// if non-NULL, <cb> is called for each block transferred, passing <cbData>.
+	// it returns how much data was actually transferred, or a negative error
+	// code (in which case we abort the transfer and return that value).
+	// the callback mechanism is useful for user progress notification or
+	// processing data while waiting for the next I/O to complete
+	// (quasi-parallel, without the complexity of threads).
+	LibError LoadFile(const char* vfsPathname, FileContents& contents, size_t& size);
+
+	void RefreshFileInfo(const char* pathname);
+
+	void Display() const;
+
+private:
+	class Impl;
+	boost::shared_ptr<Impl> impl;
+};
+
+#endif	// #ifndef INCLUDED_VFS

source/lib/file/vfs/vfs_mount.cpp

@@ -0,0 +1,293 @@
+/**
+ * =========================================================================
+ * File        : vfs_mount.cpp
+ * Project     : 0 A.D.
+ * Description : mounts files and archives into VFS
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "vfs_mount.h"
+
+#include "lib/file/archive/archive_zip.h"
+
+
+//-----------------------------------------------------------------------------
+// ArchiveEnumerator
+//-----------------------------------------------------------------------------
+
+class ArchiveEnumerator
+{
+public:
+	ArchiveEnumerator(VfsDirectory* directory)
+		: m_startDirectory(directory), m_previousPath(0), m_previousDirectory(0)
+	{
+	}
+
+	LibError Next(const char* pathname, const ArchiveEntry& archiveEntry)
+	{
+		vfs_opt_notify_archived_file(pathname);
+
+		const char* name = path_name_only(pathname);
+		const char* path = path_dir_only2(pathname);
+
+		// into which directory should the file be inserted?
+		VfsDirectory* directory = m_previousDirectory;
+		if(path != m_previousPath)
+		{
+			// we have to create them if missing, since we can't rely on the
+			// archiver placing directories before subdirs or files that
+			// reference them (WinZip doesn't always).
+
+			VfsFile* file;
+			TraverseAndCreate(path, m_startDirectory, directory, file);
+			debug_assert(file == 0);	// should not be a file on the path
+
+			m_previousPath = path;
+			m_previousDirectory = directory;
+		}
+
+		FileInfo fileInfo;
+		fileInfo.name = name;
+		fileInfo.size = archiveEntry.usize;
+		fileInfo.mtime = archiveEntry.mtime;
+		directory->AddFile(name, fileInfo);
+
+		return INFO::CB_CONTINUE;
+	}
+
+private:
+	VfsDirectory* m_startDirectory;
+
+	// optimization: looking up each full path is rather slow, so we
+	// cache the previous directory and use it if the path string
+	// addresses match.
+	const char* m_previousPath;
+	VfsDirectory* m_previousDirectory;
+};
+
+static LibError ArchiveEnumeratorCallback(const char* pathname, const ArchiveEntry& archiveEntry, uintptr_t cbData)
+{
+	ArchiveEnumerator* archiveEnumerator = (ArchiveEnumerator*)cbData;
+	return ArchiveEnumerator->Next(pathname, archiveEntry);
+}
+
+
+//-----------------------------------------------------------------------------
+// Mount
+//-----------------------------------------------------------------------------
+
+// not many instances => don't worry about efficiency.
+// note: only check for archives in the root directory of the mounting
+// because doing so for every single file would entail serious overhead
+class Mount
+{
+	Mount(const char* vfsPath, const char* path, uint flags, uint priority)
+		: m_vfsPath(vfsPath), m_path(path), m_flags(flags), m_priority(priority)
+	{
+	}
+
+	const char* VfsPath() const
+	{
+		return m_vfsPath;
+	}
+
+	const char* Path() const
+	{
+		return m_path;
+	}
+
+	// actually mount the specified entry. split out of vfs_mount,
+	// because when invalidating (reloading) the VFS, we need to
+	// be able to mount without changing the mount list.
+	LibError Mount(VfsDirectory* rootDirectory)
+	{
+		VfsDirectory* directory; VfsFile* file;
+		TraverseAndCreate(m_vfsPath, rootDirectory, directory, file);
+		debug_assert(file == 0);	// should not be a file on the path
+
+		directory->AssociateWithRealDirectory(m_path);
+
+		// add archives
+		directory->Populate();
+		std::vector<FileInfo> files;
+		directory->GetEntries(&files, 0);
+		for(size_t i = 0; i < files.size(); i++)
+		{
+			FileInfo& fileInfo = files[i];
+			const char* pathname = path_append2(m_path, fileInfo.Name());
+			const char* extension = path_extension(fileInfo.Name());
+			boost::shared_ptr<IArchiveReader> archiveReader;
+			if(strcasecmp(extension, "zip") == 0)
+				archiveReader = CreateArchiveReader_Zip(pathname);
+			else
+				continue;	// not a (supported) archive file
+
+			ArchiveEnumerator archiveEnumerator(directory);
+			RETURN_ERR(archiveReader->ReadEntries(ArchiveEnumeratorCallback, (uintptr_t)&archiveEnumerator));
+			m_archiveReaders.push_back(archiveReader);
+		}
+	}
+
+	// "backs off of" all archives - closes their files and allows them to
+	// be rewritten or deleted (required by archive builder).
+	// must call mount_rebuild when done with the rewrite/deletes,
+	// because this call leaves the VFS in limbo!!
+	void ReleaseArchives()
+	{
+		m_archiveReaders.clear();
+	}
+
+private:
+	bool IsArchivable() const
+	{
+		return (m_flags & MOUNT_ARCHIVABLE) != 0;
+	}
+
+	// mounting into this VFS directory;
+	// must end in '/' (unless if root td, i.e. "")
+	const char* m_vfsPath;
+
+	const char* m_path;
+
+	std::vector<boost::shared_ptr<IArchiveReader> > m_archiveReaders;
+
+	uint m_flags;	// MountFlags
+
+	uint m_priority;
+};
+
+
+//-----------------------------------------------------------------------------
+// MountManager
+//-----------------------------------------------------------------------------
+
+
+class MountManager
+{
+public:
+	// mount <P_real_path> into the VFS at <V_mount_point>,
+	//   which is created if it does not yet exist.
+	// files in that directory override the previous VFS contents if
+	//   <pri>(ority) is not lower.
+	// all archives in <P_real_path> are also mounted, in alphabetical order.
+	//
+	// flags determines extra actions to perform; see VfsMountFlags.
+	//
+	// P_real_path = "." or "./" isn't allowed - see implementation for rationale.
+	LibError Add(const char* vfsPath, const char* path, uint flags, uint priority)
+	{
+		// make sure caller didn't forget the required trailing '/'.
+		debug_assert(VFS_PATH_IS_DIR(vfsPath));
+
+		// make sure it's not already mounted, i.e. in mounts.
+		// also prevents mounting a parent directory of a previously mounted
+		// directory, or vice versa. example: mount $install/data and then
+		// $install/data/mods/official - mods/official would also be accessible
+		// from the first mount point - bad.
+		char existingVfsPath[PATH_MAX];
+		if(GetVfsPath(path, existingVfsPath))
+			WARN_RETURN(ERR::ALREADY_MOUNTED);
+
+		// disallow "." because "./" isn't supported on Windows.
+		// "./" and "/." are caught by CHECK_PATH.
+		if(!strcmp(path, "."))
+			WARN_RETURN(ERR::PATH_NON_CANONICAL);
+
+		// (count this as "init" to obviate a separate timer)
+		stats_vfs_init_start();
+		mounts[path] = Mount(vfsPath, path, flags, priority);
+		LibError ret = remount(m);
+		stats_vfs_init_finish();
+		return ret;
+	}
+
+	void Remove(const char* path)
+	{
+		const size_t numRemoved = mounts.erase(path);
+		debug_assert(numRemoved == 1);
+	}
+
+	// rebuild the VFS, i.e. re-mount everything. open files are not affected.
+	// necessary after loose files or directories change, so that the VFS
+	// "notices" the changes and updates file locations. res calls this after
+	// dir_watch reports changes; can also be called from the console after a
+	// rebuild command. there is no provision for updating single VFS dirs -
+	// it's not worth the trouble.
+	void RedoAll()
+	{
+		for(MountIt it = mounts.begin(); it != mounts.end(); ++it)
+		{
+			const Mount& mount = it->second;
+			mount.Apply();
+		}
+	}
+
+	// if <path> or its ancestors are mounted,
+	// return a VFS path that accesses it.
+	// used when receiving paths from external code.
+	bool GetVfsPath(const char* path, char* vfsPath) const
+	{
+		for(MountIt it = mounts.begin(); it != mounts.end(); ++it)
+		{
+			const Mount& mount = it->second;
+
+			const char* remove = mount.Path();
+			const char* replace = mount.VfsPath();
+			if(path_replace(vfsPath, path, remove, replace) == INFO::OK)
+				return true;
+		}
+
+		return false;
+	}
+
+	void ReleaseArchives()
+	{
+		// "backs off of" all archives - closes their files and allows them to
+		// be rewritten or deleted (required by archive builder).
+		// must call mount_rebuild when done with the rewrite/deletes,
+		// because this call leaves the VFS in limbo!!
+
+	}
+
+private:
+	typedef std::map<const char*, Mount> Mounts;
+	typedef Mounts::iterator MountIt;
+	Mounts mounts;
+};
+
+
+
+// 2006-05-09 JW note: we are wanting to move XMB files into a separate
+// folder tree (no longer interspersed with XML), so that deleting them is
+// easier and dirs are less cluttered.
+//
+// if several mods are active, VFS would have several OS directories mounted
+// into a VfsDirectory and could no longer automatically determine the write target.
+//
+// one solution would be to use a set_write_target API to choose the
+// correct dir; however, XMB files may be generated whilst editing
+// (which also requires a write_target to write files that are actually
+// currently in archives), so we'd need to save/restore write_target.
+// this would't be thread-safe => disaster.
+//
+// a vfs_store_to(filename, flags, N_actual_path) API would work, but it'd
+// impose significant burden on users (finding the actual native dir),
+// and be prone to abuse. additionally, it would be difficult to
+// propagate N_actual_path to VFile_reload where it is needed;
+// this would end up messy.
+//
+// instead, we'll write XMB files into VFS path "mods/$MODNAME/..",
+// into which the realdir of the same name (located in some writable folder)
+// is mounted; VFS therefore can write without problems.
+//
+// however, other code (e.g. archive builder) doesn't know about this
+// trick - it only sees the flat VFS namespace, which doesn't
+// include mods/$MODNAME (that is hidden). to solve this, we also mount
+// any active mod's XMB dir into VFS root for read access.
+//
+// write_targets are no longer necessary since files and directories are
+// directly associated with a real directory; when writing files, we can
+// do so directly.

source/lib/file/vfs/vfs_mount.h

@@ -0,0 +1,37 @@
+/**
+ * =========================================================================
+ * File        : vfs_mount.h
+ * Project     : 0 A.D.
+ * Description : mounts files and archives into VFS; provides x_* API
+ *             : that dispatches to file or archive implementation.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_VFS_MOUNT
+#define INCLUDED_VFS_MOUNT
+
+namespace ERR
+{
+	const LibError ALREADY_MOUNTED      = -110700;
+	const LibError NOT_MOUNTED          = -110701;
+	const LibError MOUNT_INVALID_TYPE   = -110702;
+}
+
+// (recursive mounting and mounting archives are no longer optional since they don't hurt)
+enum MountFlags
+{
+	// all real directories mounted during this operation will be watched
+	// for changes. this flag is provided to avoid watches in output-only
+	// directories, e.g. screenshots/ (only causes unnecessary overhead).
+	MOUNT_WATCH = 4,
+
+	// anything mounted from here should be added to archive when
+	// building via vfs_optimizer.
+	MOUNT_ARCHIVABLE = 8
+};
+
+
+
+#endif	// #ifndef INCLUDED_VFS_MOUNT

source/lib/file/vfs/vfs_path.cpp

@@ -0,0 +1,122 @@
+/**
+ * =========================================================================
+ * File        : vfs_path.cpp
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "vfs_path.h"
+
+#include "lib/path_util.h"	// path_foreach_component
+#include "vfs_tree.h"
+
+
+static bool IsVfsPath(const char* path)
+{
+	if(path[0] == '\0')	// root dir
+		return true;
+	if(path[strlen(path)-1] == '/')	// ends in slash => directory path
+		return true;
+	return false;
+}
+
+
+
+class PathResolver
+{
+public:
+	enum Flags
+	{
+		// when encountering subdirectory components in the path(name) that
+		// don't (yet) exist, add them.
+		CreateMissingDirectories = 1,
+	};
+
+	PathResolver(VfsDirectory* startDirectory, uint flags = 0)
+		: m_currentDirectory(startDirectory), m_file(0)
+		, m_createMissingDirectories((flags & CreateMissingDirectories) != 0)
+	{
+	}
+
+	LibError Next(const char* component, bool isDir) const
+	{
+		m_currentDirectory->Populate();
+
+		if(isDir)
+		{
+			if(m_createMissingDirectories)
+				m_currentDirectory->AddSubdirectory(component);
+
+			m_currentDirectory = m_currentDirectory->GetSubdirectory(component);
+			if(!m_currentDirectory)
+				WARN_RETURN(ERR::FAIL);
+		}
+		else
+		{
+			debug_assert(m_file == 0);	// can't have encountered any files yet
+			m_file = m_currentDirectory->GetFile(component);
+			if(!m_file)
+				WARN_RETURN(ERR::FAIL);
+		}
+
+		return INFO::CB_CONTINUE;
+	}
+
+	VfsDirectory* Directory() const
+	{
+		return m_currentDirectory;
+	}
+
+	VfsFile* File() const
+	{
+		return m_file;
+	}
+
+private:
+	mutable VfsDirectory* m_currentDirectory;
+	mutable VfsFile* m_file;
+	bool m_createMissingDirectories;
+};
+
+
+static LibError PathResolverCallback(const char* component, bool isDir, uintptr_t cbData)
+{
+	PathResolver* pathResolver = (PathResolver*)cbData;
+	return pathResolver->Next(component, isDir);
+}
+
+
+VfsFile* LookupFile(const char* vfsPathname, VfsDirectory* startDirectory)
+{
+	debug_assert(!IsVfsPath(vfsPathname));
+
+	PathResolver pathResolver(startDirectory);
+	if(path_foreach_component(vfsPathname, PathResolverCallback, (uintptr_t)&pathResolver) != INFO::OK)
+		return 0;
+	return pathResolver.File();
+}
+
+
+VfsDirectory* LookupDirectory(const char* vfsPath, VfsDirectory* startDirectory)
+{
+	debug_assert(IsVfsPath(vfsPath));
+
+	PathResolver pathResolver(startDirectory);
+	if(path_foreach_component(vfsPath, PathResolverCallback, (uintptr_t)&pathResolver) != INFO::OK)
+		return 0;
+	return pathResolver.Directory();
+}
+
+
+void TraverseAndCreate(const char* vfsPath, VfsDirectory* startDirectory, VfsDirectory*& lastDirectory, VfsFile*& file)
+{
+	PathResolver pathResolver(startDirectory, PathResolver::CreateMissingDirectories);
+	LibError ret = path_foreach_component(vfsPath, PathResolverCallback, (uintptr_t)&pathResolver);
+	debug_assert(ret == INFO::OK);	// should never fail
+	lastDirectory = pathResolver.Directory();
+	file = pathResolver.File();
+}

source/lib/file/vfs/vfs_path.h

@@ -0,0 +1,58 @@
+/**
+ * =========================================================================
+ * File        : vfs_path.h
+ * Project     : 0 A.D.
+ * Description : 
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_VFS_PATH
+#define INCLUDED_VFS_PATH
+
+// VFS paths are of the form: "(dir/)*file?"
+// in English: '/' as path separator; trailing '/' required for dir names;
+// no leading '/', since "" is the root dir.
+
+// there is no restriction on path length; when dimensioning character
+// arrays, prefer PATH_MAX.
+
+// pathnames are case-insensitive.
+// implementation:
+//   when mounting, we get the exact filenames as reported by the OS;
+//   we allow open requests with mixed case to match those,
+//   but still use the correct case when passing to other libraries
+//   (e.g. the actual open() syscall).
+// rationale:
+//   necessary, because some exporters output .EXT uppercase extensions
+//   and it's unreasonable to expect that users will always get it right.
+
+
+class VfsFile;
+class VfsDirectory;
+
+/**
+ * Find a file in the VFS by traversing pathname components.
+ *
+ * @param vfsPath must not end with a slash character.
+ **/
+extern VfsFile* LookupFile(const char* vfsPathname, const VfsDirectory* startDirectory);
+
+/**
+ * Find a directory in the VFS by traversing path components.
+ *
+ * @param vfsPath must end with a slash character.
+ **/
+extern VfsDirectory* LookupDirectory(const char* vfsPath, const VfsDirectory* startDirectory);
+
+/**
+ * Traverse a path, creating subdirectories that didn't yet exist.
+ *
+ * @param lastDirectory receives the last directory that was encountered
+ * @param file references the VFS file represented by vfsPath, or 0 if
+ * it doesn't include a filename.
+ **/
+extern void TraverseAndCreate(const char* vfsPath, VfsDirectory* startDirectory, VfsDirectory*& lastDirectory, VfsFile*& file);
+
+#endif	// #ifndef INCLUDED_VFS_PATH

source/lib/file/vfs/vfs_tree.cpp

@@ -0,0 +1,274 @@
+/**
+ * =========================================================================
+ * File        : vfs_tree.cpp
+ * Project     : 0 A.D.
+ * Description : the actual 'filesystem' and its tree of directories.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "vfs_tree.h"
+
+#include "lib/file/posix/fs_posix.h"
+#include "lib/file/archive/archive.h"
+
+static Filesystem_Posix fsPosix;
+
+static const char* svnName = path_Pool()->UniqueCopy(".svn");
+
+
+LibError VfsFile::Load(u8* buf) const
+{
+	if(m_archiveEntry)
+		RETURN_ERR(m_archiveEntry->archiveReader->LoadFile(*m_archiveEntry, buf));
+	else
+	{
+		const char* pathname = path_append2(m_path, Name());
+		File_Posix file;
+		RETURN_ERR(file.Open(pathname, 'r'));
+		RETURN_ERR(io_Read(file, 0, buf, Size()));
+	}
+
+	return INFO::OK;
+}
+
+VfsDirectory::VfsDirectory(const char* vfsPath, const char* path)
+	: m_vfsPath(vfsPath), m_path(0)
+{
+	if(path)
+		AssociateWithRealDirectory(path);
+}
+
+
+VfsDirectory::~VfsDirectory()
+{
+}
+
+
+
+static bool ShouldReplace(VfsFile& vf1, VfsFile& vf2)
+{
+	// 1) keep old if new priority is lower.
+	if(vf2.Priority() < vf1.Priority())
+		return false;
+
+	// assume they're the same if size and last-modified time match.
+	// note: FAT timestamp only has 2 second resolution
+	const double diff = difftime(vf1.MTime(), vf2.MTime());
+	const bool identical = (vf1.Size() == vf2.Size()) && fabs(diff) <= 2.0;
+
+	// 3) go with more efficient source (if files are identical)
+	//
+	// since priority is not less, we really ought to always go with m_new.
+	// however, there is one special case we handle for performance reasons:
+	// if the file contents are the same, prefer the more efficient source.
+	// note that priority doesn't automatically take care of this,
+	// especially if set incorrectly.
+	if(identical && vf1.Precedence() > vf2.Precedence())
+		return false;
+
+	// 4) don't replace "old" file if modified more recently than "new".
+	// (still provide for 2 sec. FAT tolerance - see above)
+	if(diff > 2.0)
+		return false;
+
+	return true;
+}
+
+
+void VfsDirectory::AddFile(const FileInfo& fileInfo)
+{
+	std::pair<const char*, VfsFile> value = std::make_pair(fileInfo.Name(), VfsFile(fileInfo));
+	std::pair<Files::iterator, bool> ret = m_files.insert(value);
+	if(!ret.second)	// already existed
+	{
+		VfsFile& previousFile = ret.first.second;
+		if(!ShouldReplace(previousFile, value.second))
+			return INFO::ALREADY_EXISTS;
+		replace_in_map
+	}
+
+	stats_vfs_file_add(size);
+}
+
+
+void VfsDirectory::AddSubdirectory(const char* name)
+{
+	const char* vfsPath = path_append2(m_vfsPath, name);
+	std::pair<const char*, VfsDirectory> value = std::make_pair(name, VfsDirectory(vfsPath));
+	(void)m_subdirectories.insert(value);
+}
+
+
+VfsFile* VfsDirectory::GetFile(const char* name) const
+{
+	Files::const_iterator it = m_files.find(name);
+	if(it == m_files.end())
+		return 0;
+	return &it->second;
+}
+
+
+VfsDirectory* VfsDirectory::GetSubdirectory(const char* name) const
+{
+	Subdirectories::const_iterator it = m_subdirectories.find(name);
+	if(it == m_subdirectories.end())
+		return 0;
+	return &it->second;
+}
+
+
+void VfsDirectory::GetEntries(std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const
+{
+	if(files)
+	{
+		files->reserve(m_files.size());
+		for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it)
+			files->push_back(it->second.m_fileInfo);
+	}
+
+	if(subdirectories)
+	{
+		subdirectories->reserve(m_subdirectories.size());
+		for(Subdirectories::const_iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
+			subdirectories->push_back(it->second.m_name);
+	}
+}
+
+
+void VfsDirectory::DisplayR(unsigned depth) const
+{
+	const char indent[] = "    ";
+
+	// build format string (set width of name field so that everything
+	// lines up correctly)
+	char fmt[25];
+	const int maxNameChars = 80 - depth*(sizeof(indent)-1);
+	sprintf(fmt, "%%-%d.%ds (%%c; %%6d; %%s)\n", maxNameChars, maxNameChars);
+
+	for(Files::const_iterator it = m_files.begin(); it != m_files.end(); ++it)
+	{
+		VfsFile& file = it->second;
+
+		char timestamp[25];
+		const time_t mtime = file.MTime();
+		strftime(timestamp, ARRAY_SIZE(timestamp), "%a %b %d %H:%M:%S %Y", localtime(&mtime));
+
+		for(int i = 0; i < depth; i++)
+			printf(indent);
+		printf(fmt, file.Name(), file.LocationCode(), file.Size(), timestamp);
+	}
+
+	for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
+	{
+		const char* name = it->first;
+		const VfsDirectory& directory = it->second;
+
+		for(int i = 0; i < depth+1; i++)
+			printf(indent);
+		printf("[%s/]\n", name);
+		directory.DisplayR(depth+1);
+	}
+}
+
+
+void VfsDirectory::ClearR()
+{
+	for(Subdirectories::iterator it = m_subdirectories.begin(); it != m_subdirectories.end(); ++it)
+		it->second.ClearR();
+
+	m_files.Clear();
+	m_subdirectories.Clear();
+	m_watch.Unregister();
+}
+
+
+
+static const char* AMBIGUOUS = (const char*)(intptr_t)-1;
+
+
+void VfsDirectory::AssociateWithRealDirectory(const char* path)
+{
+	// mounting the same directory twice is probably a bug
+	debug_assert(m_path != path);
+
+	if(!m_path)
+	{
+		m_path = path;
+		m_watch.Register(path);
+	}
+	else
+		m_path = AMBIGUOUS;
+}
+
+
+void VfsDirectory::Populate()
+{
+	std::vector<FileInfo> files; files.reserve(100);
+	std::vector<const char*> subdirectories; subdirectories.reserve(20);
+	fsPosix.GetDirectoryEntries(m_path, &files, &subdirectories);
+
+	for(size_t i = 0; i < files.size(); i++)
+	{
+		AddFile(files[i]);
+
+		// note: check if archivable to exclude stuff like screenshots
+		// from counting towards the threshold.
+		if(m->IsArchivable())
+		{
+			// notify archive builder that this file could be archived but
+			// currently isn't; if there are too many of these, archive will
+			// be rebuilt.
+			const char* pathname = path_append2(m_path, files[i].Name());
+			vfs_opt_notify_loose_file(pathname);
+		}
+	}
+
+	for(size_t i = 0; i < subdirectories.size(); i++)
+	{
+		// skip version control directories - this avoids cluttering the
+		// VFS with hundreds of irrelevant files.
+		if(subdirectories[i] == svnName)
+			continue;
+
+		const char* path = path_append2(m_path, subdirectories[i]);
+		AddSubdirectory(subdirectories[i], path);
+	}
+}
+
+
+LibError VfsDirectory::CreateFile(const char* name, const u8* buf, size_t size, uint flags = 0)
+{
+	debug_assert(m_path != 0 && m_path != AMBIGUOUS);
+
+	const char* pathname = path_append2(pathname, m_path, name);
+	File_Posix file;
+	RETURN_ERR(file.Open(pathname, 'w'));
+	RETURN_ERR(io_Write(file, 0, buf, size));
+
+	AddFile(FileInfo(name, size, time()));
+	return INFO::OK;
+}
+
+
+//-----------------------------------------------------------------------------
+
+
+VfsTree::VfsTree()
+	: m_rootDirectory("")
+{
+}
+
+
+void VfsTree::Display() const
+{
+	m_rootDirectory.DisplayR();
+}
+
+
+void VfsTree::Clear()
+{
+	m_rootDirectory.ClearR();
+}

source/lib/file/vfs/vfs_tree.h

@@ -0,0 +1,198 @@
+/**
+ * =========================================================================
+ * File        : vfs_tree.h
+ * Project     : 0 A.D.
+ * Description : the actual 'filesystem' and its tree of directories.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_VFS_TREE
+#define INCLUDED_VFS_TREE
+
+#include "lib/file/filesystem.h"		// FileInfo
+
+// we add/cancel directory watches from the VFS mount code for convenience -
+// it iterates through all subdirectories anyway (*) and provides storage for
+// a key to identify the watch (obviates separate TDir -> watch mapping).
+//
+// define this to strip out that code - removes .watch from struct TDir,
+// and calls to res_watch_dir / res_cancel_watch.
+//
+// *: the add_watch code would need to iterate through subdirs and watch
+//    each one, because the monitor API (e.g. FAM) may only be able to
+//    watch single directories, instead of a whole subdirectory tree.
+#define NO_DIR_WATCH
+
+class DirectoryWatch
+{
+public:
+	DirectoryWatch()
+#ifndef NO_DIR_WATCH
+		: m_watch(0)
+#endif
+	{
+	}
+
+	// 'watch' this directory for changes to support hotloading.
+	void Register(const char* path)
+	{
+#ifndef NO_DIR_WATCH
+		char osPath[PATH_MAX];
+		(void)path_MakeAbsolute(path, osPath);
+		(void)dir_add_watch(osPath, &m_watch);
+#else
+		UNUSED2(path);
+#endif
+	}
+
+	void Unregister()
+	{
+#ifndef NO_DIR_WATCH
+		if(m_watch)	// avoid dir_cancel_watch complaining
+		{
+			dir_cancel_watch(m_watch);
+			m_watch = 0;
+		}
+#endif
+	}
+
+#ifndef NO_DIR_WATCH
+private:
+	intptr_t m_watch;
+#endif
+};
+
+
+//-----------------------------------------------------------------------------
+
+struct ArchiveEntry;
+
+class VfsFile
+{
+	friend class VfsDirectory;
+
+public:
+	VfsFile(const FileInfo& fileInfo, unsigned priority)
+		: m_fileInfo(fileInfo), m_priority(priority), m_path(0), m_archiveEntry(0)
+	{
+	}
+
+	void GetFileInfo(FileInfo& fileInfo) const
+	{
+		fileInfo = m_fileInfo;
+	}
+
+	const char* Name() const
+	{
+		return m_fileInfo.Name();
+	}
+
+	off_t Size() const
+	{
+		return m_fileInfo.Size();
+	}
+
+	time_t MTime() const
+	{
+		return m_fileInfo.MTime();
+	}
+
+	LibError Load(u8* buf) const;
+
+	unsigned Priority() const
+	{
+		return m_priority;
+	}
+
+	unsigned Precedence() const
+	{
+		return m_archiveEntry? 1u : 2u;
+	}
+
+	char LocationCode() const
+	{
+		return m_archiveEntry? 'A' : 'F';
+	}
+
+private:
+	FileInfo m_fileInfo;
+
+	unsigned m_priority;
+
+	// location (directory or archive) of the file. this allows opening
+	// the file in O(1) even when there are multiple mounted directories.
+	const char* m_path;
+	ArchiveEntry* m_archiveEntry;
+};
+
+
+//-----------------------------------------------------------------------------
+
+class VfsDirectory
+{
+public:
+	VfsDirectory(const char* vfsPath);
+	~VfsDirectory();
+
+	void AddFile(const FileInfo& fileInfo);
+	void AddSubdirectory(const char* name);
+
+	VfsFile* GetFile(const char* name) const;
+	VfsDirectory* GetSubdirectory(const char* name) const;
+
+	// (don't split this into 2 functions because POSIX can't implement
+	// that efficiently)
+	void GetEntries(std::vector<FileInfo>* files, std::vector<const char*>* subdirectories) const;
+
+	void DisplayR() const;
+	void ClearR();
+
+	void AssociateWithRealDirectory(const char* path);
+	void Populate();
+
+	void CreateFile(const char* name, const u8* buf, size_t size, uint flags = 0);
+
+private:
+	typedef std::map<const char*, VfsFile> Files;
+	Files m_files;
+
+	typedef std::map<const char*, VfsDirectory> Subdirectories;
+	Subdirectories m_subdirectories;
+
+	const char* m_vfsPath;
+
+	// if exactly one real directory is mounted into this virtual dir,
+	// this points to its location. used to add files to VFS when writing.
+	const char* m_path;
+
+	DirectoryWatch m_watch;
+};
+
+
+//-----------------------------------------------------------------------------
+
+class VfsTree
+{
+public:
+	VfsTree();
+
+	VfsDirectory& Root()
+	{
+		return m_rootDirectory;
+	}
+
+	const VfsDirectory& Root() const
+	{
+		return m_rootDirectory;
+	}
+
+	void Display() const;
+	void Clear();
+
+private:
+	VfsDirectory m_rootDirectory;
+};
+
+#endif	// #ifndef INCLUDED_VFS_TREE