huge change:

- replace std::map with custom filename lookup container (more
efficient)
- split into 3 parts (was too big)
also added struct stat-like fields to vfsDirEnt

still under heavy development.

This was SVN commit r2058.

source/lib/res/vfs.h

@@ -29,6 +29,8 @@
 // VFS tree
 //
 
+extern void vfs_init();
+
 // the VFS doesn't require this length restriction - VFS internal storage
 // is not fixed-length. the purpose here is to give an indication of how
 // large fixed-size user buffers should be. length includes trailing '\0'.
@@ -91,11 +93,16 @@ extern void vfs_display(void);
 struct vfsDirEnt
 {
 	// name of directory entry - does not include path.
-	// valid until the directory handle is closed. must not be modified!
-	// rationale for pointer and invalidation: see vfs_next_dirent.
+	// valid until the next VFS rebuild. must not be modified!
 	const char* name;
+
+	off_t size;
+
+	time_t mtime;
 };
 
+#define VFS_ENT_IS_DIR(ent) ((ent).size == -1)
+
 // open the directory for reading its entries via vfs_next_dirent.
 // <v_dir> need not end in '/'; we add it if not present.
 // directory contents are cached here; subsequent changes to the dir
@@ -110,10 +117,10 @@ extern int vfs_close_dir(Handle& hd);
 // return 0 on success, ERR_DIR_END if no matching entry was found,
 // or a negative error code on failure.
 // filter values:
-// - 0: any file;
+// - 0: anything;
 // - "/": any subdirectory
 // - anything else: pattern for name (may include '?' and '*' wildcards)
-extern int vfs_next_dirent(Handle hd, vfsDirEnt* ent, const char* filter);
+extern int vfs_next_dirent(Handle hd, vfsDirEnt* ent, const char* filter = 0);
 
 
 //

source/lib/res/vfs_path.cpp

@@ -0,0 +1,150 @@
+#include "precompiled.h"
+
+#include <string.h>
+
+#include "lib.h"
+#include "vfs.h"
+#include "vfs_path.h"
+
+
+// path types:
+// p_*: posix (e.g. mount object name or for open())
+// v_*: vfs (e.g. mount point)
+// fn : filename only (e.g. from readdir)
+// dir_name: directory only, no path (e.g. subdir name)
+//
+// all paths must be relative (no leading '/'); components are separated
+// by '/'; no ':', '\\', "." or ".." allowed; root dir is "".
+//
+// grammar:
+// path ::= dir*file?
+// dir  ::= name/
+// file ::= name
+// name ::= [^/]
+
+
+// if path is invalid (see source for criteria), print a diagnostic message
+// (indicating line number of the call that failed) and
+// return a negative error code. used by CHECK_PATH.
+int path_validate(const uint line, const char* path)
+{
+	size_t path_len = 0;	// counted as we go; checked against max.
+
+	const char* msg = 0;	// error occurred <==> != 0
+	int err = -1;			// what we pass to caller
+
+	int c = 0, last_c;		// used for ./ detection
+
+	// disallow "/", because it would create a second 'root' (with name = "").
+	// root dir is "".
+	if(path[0] == '/')
+	{
+		msg = "starts with '/'";
+		goto fail;
+	}
+
+	// scan each char in path string; count length.
+	for(;;)
+	{
+		last_c = c;
+		c = path[path_len++];
+
+		// whole path is too long
+		if(path_len >= VFS_MAX_PATH)
+		{
+			msg = "path too long";
+			goto fail;
+		}
+
+		// disallow:
+		// - ".." (prevent going above the VFS root dir)
+		// - "./" (security hole when mounting and not supported on Windows).
+		// allow "/.", because CVS backup files include it.
+		if(last_c == '.' && (c == '.' || c == '/'))
+		{
+			msg = "contains '..' or './'";
+			goto fail;
+		}
+
+		// disallow OS-specific dir separators
+		if(c == '\\' || c == ':')
+		{
+			msg = "contains OS-specific dir separator (e.g. '\\', ':')";
+			goto fail;
+		}
+
+		// end of string, all is well.
+		if(c == '\0')
+			goto ok;
+	}
+
+	// failed somewhere - err is the error code,
+	// or -1 if not set specifically above.
+fail:
+	debug_out("path_validate at line %d failed: %s (error code %d)\n", line, msg, err);
+	debug_warn("path_validate failed");
+	return err;
+
+ok:
+	return 0;
+}
+
+#define CHECK_PATH(path) CHECK_ERR(path_validate(__LINE__, path))
+
+
+// convenience function
+inline void path_copy(char* dst, const char* src)
+{
+	strcpy_s(dst, VFS_MAX_PATH, src);
+}
+
+
+// combine <path1> and <path2> into one path, and write to <dst>.
+// if necessary, a directory separator is added between the paths.
+// each may be empty, filenames, or full paths.
+// total path length (including '\0') must not exceed VFS_MAX_PATH.
+int path_append(char* dst, const char* path1, const char* path2)
+{
+	const size_t len1 = strlen(path1);
+	const size_t len2 = strlen(path2);
+	size_t total_len = len1 + len2 + 1;	// includes '\0'
+
+	// check if we need to add '/' between path1 and path2
+	// note: the second can't start with '/' (not allowed by path_validate)
+	bool need_separator = false;
+	if(len1 != 0 && path1[len1-1] != '/')
+	{
+		total_len++;	// for '/'
+		need_separator = true;
+	}
+
+	if(total_len+1 > VFS_MAX_PATH)
+		return ERR_PATH_LENGTH;
+
+	strcpy(dst, path1);	// safe
+	dst += len1;
+	if(need_separator)
+		*dst++ = '/';
+	strcpy(dst, path2);	// safe
+	return 0;
+}
+
+// strip <remove> from the start of <src>, prepend <replace>,
+// and write to <dst>.
+// used when converting VFS <--> real paths.
+int path_replace(char* dst, const char* src, const char* remove, const char* replace)
+{
+	// remove doesn't match start of <src>
+	const size_t remove_len = strlen(remove);
+	if(strncmp(src, remove, remove_len) != 0)
+		return -1;
+
+	// get rid of trailing / in src (must not be included in remove)
+	const char* start = src+remove_len;
+	if(*start == '/' || *start == DIR_SEP)
+		start++;
+
+	// prepend replace.
+	CHECK_ERR(path_append(dst, replace, start));
+	return 0;
+}

source/lib/res/vfs_path.h

@@ -0,0 +1,27 @@
+#ifndef VFS_UTIL_H__
+#define VFS_UTIL_H__
+
+#include "lib.h"
+
+// if path is invalid (see source for criteria), print a diagnostic message
+// (indicating line number of the call that failed) and
+// return a negative error code. used by CHECK_PATH.
+extern int path_validate(const uint line, const char* path);
+
+#define CHECK_PATH(path) CHECK_ERR(path_validate(__LINE__, path))
+
+// convenience function
+extern void path_copy(char* dst, const char* src);
+
+// combine <path1> and <path2> into one path, and write to <dst>.
+// if necessary, a directory separator is added between the paths.
+// each may be empty, filenames, or full paths.
+// total path length (including '\0') must not exceed VFS_MAX_PATH.
+extern int path_append(char* dst, const char* path1, const char* path2);
+
+// strip <remove> from the start of <src>, prepend <replace>,
+// and write to <dst>.
+// used when converting VFS <--> real paths.
+extern int path_replace(char* dst, const char* src, const char* remove, const char* replace);
+
+#endif	// #ifndef VFS_UTIL_H__

source/lib/res/vfs_tree.cpp

@@ -0,0 +1,748 @@
+#include "precompiled.h"
+
+#include <string.h>
+
+#include <string>
+#include <vector>
+
+#include "lib.h"
+#include "res.h"
+#include "vfs_path.h"
+#include "vfs_tree.h"
+#include "hotload.h"	// see NO_DIR_WATCH
+
+
+// TMountPoint  = location of a file in the tree.
+// TFile = all information about a file stored in the tree.
+// TDir  = container holding TFile-s representing a dir. in the tree.
+
+
+
+
+// CONTAINER RATIONALE (see philip discussion)
+
+
+struct TNode;
+
+enum TNodeType
+{
+	N_NONE,
+	N_DIR,
+	N_FILE
+};
+
+
+//////////////////////////////////////////////////////////////////////////////
+//
+//
+//
+//////////////////////////////////////////////////////////////////////////////
+
+
+class TChildren
+{
+public: // xxx
+	TNode** tbl;
+	short num_entries;
+	short max_entries;	// when initialized, = 2**n for faster modulo
+
+public:
+	void init();
+	void clear();
+	TNode** get_slot(const char* fn);
+	bool expand_tbl();
+	TNode* add(const char* fn);
+	TNode* find(const char* fn);
+
+	size_t size() { return num_entries; }
+
+	class iterator;
+	iterator begin() const;
+	iterator end() const;
+};
+
+typedef TChildren::iterator TChildIt;
+
+
+
+
+class TDir
+{
+public:
+
+	// if exactly one real directory is mounted into this virtual dir,
+	// this points to its location. used to add files to VFS when writing.
+	//
+	// the TMountPoint is actually in the mount info and is invalid when
+	// that's unmounted, but the VFS would then be rebuilt anyway.
+	//
+	// = 0 if no real dir mounted here; = -1 if more than one.
+	const TMountPoint* mount_point;
+
+#ifndef NO_DIR_WATCH
+	intptr_t watch;
+#endif
+	TChildren children;
+
+	// documented below
+	void init();
+	TNode* find(const char* fn, TNodeType desired_type);
+	TNode* add(const char* fn, TNodeType new_type);
+	TDir* add_dir(const char* path, const TMountPoint* mount_point_);
+	int lookup(const char* path, uint flags, TNode** pnode, char* exact_path);
+	void clearR();
+	void displayR(int indent_level);
+};
+
+
+
+// can't inherit, since exact_name must come at end of record
+struct TNode
+{
+	// must be at start of TNode to permit casting back and forth!
+	// (see TDir::lookup)
+	union TNodeUnion
+	{
+		TDir dir;
+		TFile file;
+	} u;
+
+	TNodeType type;
+
+	//used by callers needing the exact case,
+	// e.g. for case-sensitive syscalls; also key for lookup
+	// set by TChildren
+	char exact_name[1];
+};
+
+
+
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// "bucket" allocator for TNodes; used by TChildren
+//
+//////////////////////////////////////////////////////////////////////////////
+
+
+const size_t BUCKET_SIZE = 8*KiB;
+
+static u8* bucket_pos;
+
+
+TNode* node_alloc(size_t size)
+{
+	// would overflow a bucket
+	if(size > BUCKET_SIZE-sizeof(u8*))
+	{
+		debug_warn("node_alloc: size doesn't fit in a bucket");
+		return 0;
+	}
+
+	size = round_up(size, 8);
+		// ensure alignment, since size includes a string
+	const uintptr_t addr = (uintptr_t)bucket_pos;
+	const size_t bytes_used = addr % BUCKET_SIZE;
+	// addr = 0 on first call (no bucket yet allocated)
+	// bytes_used == 0 if a node fit exactly into a bucket
+	if(addr == 0 || bytes_used == 0 || bytes_used+size > BUCKET_SIZE)
+	{
+		u8* const prev_bucket = (u8*)addr - bytes_used;
+		u8* bucket = (u8*)mem_alloc(BUCKET_SIZE, BUCKET_SIZE);
+		if(!bucket)
+			return 0;
+		*(u8**)bucket = prev_bucket;
+		bucket_pos = bucket+round_up(sizeof(u8*), 8);
+	}
+
+	TNode* node = (TNode*)bucket_pos;
+	bucket_pos = (u8*)node+size;
+	return node;
+}
+
+
+void node_free_all()
+{
+	const uintptr_t addr = (uintptr_t)bucket_pos;
+	u8* bucket = bucket_pos - (addr % BUCKET_SIZE);
+
+	// covers bucket_pos == 0 case
+	while(bucket)
+	{
+		u8* prev_bucket = *(u8**)bucket;
+		mem_free(bucket);
+		bucket = prev_bucket;
+	}
+}
+
+
+//////////////////////////////////////////////////////////////////////////////
+//
+// TChildren implementation
+//
+//////////////////////////////////////////////////////////////////////////////
+
+
+
+class TChildren::iterator
+{
+public:
+	typedef std::random_access_iterator_tag iterator_category;
+	typedef TNode* T;
+	typedef T value_type;
+	typedef ptrdiff_t difference_type;
+	typedef const TNode** pointer;
+	typedef const TNode*& reference;
+
+	iterator()
+		{}
+	iterator(T* pos_) : pos(pos_)
+		{}
+	T& operator[](int idx) const
+		{ return pos[idx]; }
+	T& operator*() const
+		{ return *pos; }
+	const T* operator->() const
+		{ return &**this; }
+	iterator& operator++()	// pre
+		{ ++pos; return (*this); }
+	iterator operator++(int)	// post
+		{ iterator tmp =  *this; ++*this; return tmp; }
+	bool operator==(const iterator& rhs) const
+		{ return pos == rhs.pos; }
+	bool operator!=(const iterator& rhs) const
+		{ return !(*this == rhs); }
+	bool operator<(const iterator& rhs) const
+		{ return (pos < rhs.pos); }
+
+protected:
+	T* pos;
+};
+
+TChildren::iterator TChildren::begin() const
+{ return iterator(tbl); }
+TChildren::iterator TChildren::end() const
+{ return iterator(tbl+max_entries);	}
+
+
+void TChildren::init()
+{
+	tbl = 0;
+	num_entries = 0;
+	max_entries = 16;	// will be doubled in expand_tbl
+	expand_tbl();
+}
+
+
+void TChildren::clear()
+{
+memset(tbl, 0, max_entries*sizeof(TNode*));
+	free(tbl);
+	tbl = 0;
+	num_entries = max_entries = 0;
+}
+
+
+// note: add is only called once per file, so we can do the hash
+// here without duplication
+TNode** TChildren::get_slot(const char* fn)
+{
+	const uint mask = max_entries-1;
+	u32 hash = fnv_lc_hash(fn);
+	TNode** pnode;
+	for(;;)
+	{
+		pnode = &tbl[hash & mask];
+		hash++;
+		TNode* const node = *pnode;
+		if(!node)
+			break;
+		if(!stricmp(node->exact_name, fn))
+			break;
+	}
+
+	return pnode;
+}
+
+
+bool TChildren::expand_tbl()
+{
+	// alloc a new table (but don't assign it to <tbl> unless successful)
+	TNode** old_tbl = tbl;
+	tbl = (TNode**)calloc(max_entries*2, sizeof(TNode*));
+	if(!tbl)
+	{
+		tbl = old_tbl;
+		return false;
+	}
+
+	max_entries += max_entries;
+	// must be set before get_slot
+
+	// newly initialized, nothing to copy - done
+	if(!old_tbl)
+		return true;
+
+	// re-hash from old table into the new one
+	for(int i = 0; i < max_entries/2; i++)
+	{
+		TNode* const node = old_tbl[i];
+		if(node)
+			*get_slot(node->exact_name) = node;
+	}
+	free(old_tbl);
+
+	return true;
+}
+
+
+// return existing, or add if not present
+TNode* TChildren::add(const char* fn)
+{
+	// expand before determining slot; this will invalidate previous pnodes.
+	if(num_entries*2 >= max_entries)
+	{
+		if(!expand_tbl())
+			return 0;
+	}
+
+	TNode** pnode = get_slot(fn);
+	if(*pnode)
+		return *pnode;
+
+	const size_t size = sizeof(TNode)+strlen_s(fn, VFS_MAX_PATH)+1;
+	TNode* node = node_alloc(size);
+	if(!node)
+		return 0;
+
+	// commit
+	*pnode = node;
+	num_entries++;
+	strcpy(node->exact_name, fn);	// safe
+	node->type = N_NONE;
+	return node;
+}
+
+
+TNode* TChildren::find(const char* fn)
+{
+	return *get_slot(fn);
+}
+
+
+
+//////////////////////////////////////////////////////////////////////////////
+//
+//
+//
+//////////////////////////////////////////////////////////////////////////////
+
+void TDir::init()
+{
+	mount_point = 0;
+	children.init();
+}
+
+
+TNode* TDir::find(const char* fn, TNodeType desired_type)
+{
+	TNode* node = children.find(fn);
+	if(node && node->type != desired_type)
+		return 0;
+	return node;
+}
+
+
+TNode* TDir::add(const char* fn, TNodeType new_type)
+{
+	TNode* node = children.add(fn);
+	if(!node)
+		return 0;
+	// already initialized
+	if(node->type != N_NONE)
+		return node;
+
+	node->type = new_type;
+
+	// note: this is called from lookup, which needs to create nodes.
+	// therefore, we need to initialize here.
+	if(new_type == N_FILE)
+		node->u.file.init();
+	else
+		node->u.dir.init();
+
+	return node;
+}
+
+
+// note: full VFS path is needed for the dir watch.
+TDir* TDir::add_dir(const char* path, const TMountPoint* mp)
+{
+	const char* subdir_name = strrchr(path, '/');
+	if(!subdir_name++)
+		return 0;
+
+	TNode* node = add(subdir_name, N_DIR);
+	if(!node)
+		return 0;
+	TDir* subdir = &node->u.dir;
+
+	// more than one real dir mounted into VFS dir
+	// (=> can't create files for writing here)
+	if(subdir->mount_point)
+		subdir->mount_point = (TMountPoint*)-1;
+	else
+		subdir->mount_point = mp;
+
+#ifndef NO_DIR_WATCH
+	int ret = res_watch_dir(path, &subdir->watch);
+	assert(ret == 0);
+#endif
+
+	return subdir;
+}
+
+
+int TDir::lookup(const char* path, uint flags, TNode** pnode, char* exact_path)
+{
+	// cleared on failure / if returning root dir node (= "")
+	if(exact_path)
+		exact_path[0] = '\0';
+
+	// early out: "" => return this directory (usually VFS root)
+	if(path[0] == '\0')
+	{
+		*pnode = (TNode*)this;	// HACK: TDir is at start of TNode
+		return 0;
+	}
+
+	CHECK_PATH(path);
+	assert( (flags & ~(LF_CREATE_MISSING|LF_START_DIR)) == 0 );
+		// no undefined bits set
+
+	const bool create_missing = !!(flags & LF_CREATE_MISSING);
+
+	// copy into (writeable) buffer so we can 'tokenize' path components
+	// by replacing '/' with '\0'.
+	char v_path[VFS_MAX_PATH];
+	strcpy_s(v_path, sizeof(v_path), path);
+	char* cur_component = v_path;
+
+	TDir* dir = this;
+	TNodeType type = N_DIR;
+
+	// successively navigate to the next component in <path>.
+	TNode* node;
+	for(;;)
+	{
+		// "extract" cur_component string (0-terminate by replacing '/')
+		char* slash = (char*)strchr(cur_component, '/');
+		if(!slash)
+		{
+			// string ended in slash => return the current dir node
+			if(*cur_component == '\0')
+				break;
+			// it's a filename
+			type = N_FILE;
+		}
+		// normal operation (cur_component is a directory)
+		else
+			*slash = '\0';
+
+		// create <cur_component> (no-op if it already exists)
+		if(create_missing)
+		{
+			node = dir->add(cur_component, type);
+			if(!node)
+				return ERR_NO_MEM;
+			if(type == N_FILE)	// xxx move to ctor i.e. init()?
+			{
+				node->u.file.mount_point = dir->mount_point;
+				node->u.file.pri = 0;
+				node->u.file.in_archive = 0;
+			}
+		}
+		else
+		{
+			node = dir->find(cur_component, type);
+			if(!node)
+				return slash? ERR_PATH_NOT_FOUND : ERR_FILE_NOT_FOUND;
+		}
+		dir = &node->u.dir;
+
+		if(exact_path)
+			exact_path += sprintf(exact_path, "%s/", node->exact_name);
+		// no length check needed: length is the same as path
+
+		// cur_component was a filename => we're done
+		if(!slash)
+		{
+			// strip trailing '/' that was added above
+			if(exact_path)
+				exact_path[-1] = '\0';
+			break;
+		}
+		// else: it was a directory; advance
+		cur_component = slash+1;
+	}
+
+	// success.
+	*pnode = node;
+	return 0;
+}
+
+
+// empty this directory and all subdirectories; used when rebuilding VFS.
+void TDir::clearR()
+{
+	// recurse for all subdirs
+	// (preorder traversal - need to do this before clearing the list)
+	for(TChildIt it = children.begin(); it != children.end(); ++it)
+	{
+		TNode* node = *it;
+		if(node && node->type == N_DIR)
+			node->u.dir.clearR();
+	}
+
+	// wipe out this directory
+	children.clear();
+#ifndef NO_DIR_WATCH
+	res_cancel_watch(watch);
+	watch = 0;
+#endif
+}
+
+
+void TDir::displayR(int indent_level)
+{
+	const char indent[] = "    ";
+
+	TChildIt it;
+
+	// list all files in this dir
+	for(it = children.begin(); it != children.end(); ++it)
+	{
+		TNode* node = (*it);
+		if(node->type != N_FILE)
+			continue;
+
+		TFile& file = node->u.file;
+		const char* name = node->exact_name;
+		char is_archive = file.in_archive? 'A' : 'L';
+		char* timestamp = ctime(&file.mtime);
+		timestamp[24] = '\0';	// remove '\n'
+		const off_t size = file.size;
+
+		for(int i = 0; i < indent_level; i++)
+			printf(indent);
+		char fmt[25];
+		int chars = 80 - indent_level*(sizeof(indent)-1);
+		sprintf(fmt, "%%-%d.%ds (%%c; %%6d; %%s)\n", chars, chars);
+		// build format string: tell it how long the filename may be,
+		// so that it takes up all space before file info column.
+		printf(fmt, name, is_archive, size, timestamp);
+	}
+
+	// recurse over all subdirs
+	for(it = children.begin(); it != children.end(); ++it)
+	{
+		TNode* node = (*it);
+		if(node->type != N_DIR)
+			continue;
+
+		TDir& subdir = node->u.dir;
+		const char* subdir_name = node->exact_name;
+
+		// write subdir's name
+		// note: do it now, instead of in recursive call so that:
+		// - we don't have to pass dir_name parameter;
+		// - the VFS root node isn't displayed.
+		for(int i = 0; i < indent_level; i++)
+			printf(indent);
+		printf("[%s/]\n", subdir_name);
+
+		subdir.displayR(indent_level+1);
+	}
+}
+
+
+//////////////////////////////////////////////////////////////////////////////
+//
+//
+//
+//////////////////////////////////////////////////////////////////////////////
+
+static TNode tree_root;
+	// => exact_name = ""
+static TDir* tree_root_dir = &tree_root.u.dir;
+
+
+void tree_init()
+{
+	tree_root_dir->init();
+}
+
+inline void tree_clear()
+{
+	tree_root_dir->clearR();
+}
+
+
+// write a representation of the VFS tree to stdout.
+inline void tree_display()
+{
+	tree_root_dir->displayR(0);
+}
+
+
+TFile* tree_add_file(TDir* dir, const char* name)
+{
+	TNode* node = dir->add(name, N_FILE);
+	return node? &node->u.file : 0;
+}
+
+
+TDir* tree_add_dir(TDir* dir, const char* path, const TMountPoint* mount_point)
+{
+	return dir->add_dir(path, mount_point);
+}
+
+
+int tree_lookup_dir(const char* path, TDir** pdir, uint flags, char* exact_path)
+{
+	// TDir::lookup would return a file node
+	if(path[0] != '\0' && path[strlen(path)-1] != '/')
+		return -1;
+
+	TDir* dir = (flags & LF_START_DIR)? *pdir : tree_root_dir;
+	TNode* node;
+	int err = dir->lookup(path, flags, &node, exact_path);
+	if(err <= 0)
+		CHECK_ERR(err);
+	*pdir = &node->u.dir;
+	return 0;
+}
+
+
+int tree_lookup(const char* path, TFile** pfile, uint flags, char* exact_path)
+{
+	// TDir::lookup would return a directory node
+	if(path[0] == '\0' || path[strlen(path)-1] == '/')
+		return -1;
+
+	TNode* node;
+	int ret = tree_root_dir->lookup(path, flags, &node, exact_path);
+	if(ret < 0)	// xxx
+		return ret;
+	*pfile = &node->u.file;
+	return 0;
+}
+
+
+//////////////////////////////////////////////////////////////////////////////
+
+struct NodeLatch
+{
+	size_t i;
+	std::vector<const TNode*> v;
+
+	static bool ci_less(const TNode* n1, const TNode* n2)
+	{
+		return stricmp(n1->exact_name, n2->exact_name) <= 0;
+	}
+
+	NodeLatch(TChildren& c)
+	{
+		i = 0;
+
+		v.reserve(c.size());
+		// don't std::copy because c contains a lot of NULL entries
+		// (which we must not return); this way is easier than having the
+		// iterator strip them.
+		for(TChildIt it = c.begin(); it != c.end(); ++it)
+			if(*it)
+				v.push_back(*it);
+
+		std::sort(v.begin(), v.end(), ci_less);
+	}
+
+	bool empty() const { return i == v.size(); }
+
+	const TNode* get_next()
+	{
+		assert(!empty());
+		return v[i++];
+	}
+};
+
+
+// must not be held across rebuild! refcount to make sure
+int tree_open_dir(const char* path_slash, void** latch)
+{
+	TDir* dir;
+	CHECK_ERR(tree_lookup_dir(path_slash, &dir));
+	*latch = new NodeLatch(dir->children);
+	return 0;
+}
+
+
+int tree_next_dirent(void* latch_, const char* filter, TDirent* dirent)
+{
+	bool want_dir = true;
+	if(filter)
+	{
+		if(filter[0] == '/')
+		{
+			if(filter[1] == '|')
+				filter += 2;
+		}
+		else
+			want_dir = false;
+	}
+
+	// loop until a TNode matches what is requested, or end of list.
+	NodeLatch* latch = (NodeLatch*)latch_;
+	const TNode* node;
+	for(;;)
+	{
+		if(latch->empty())
+			return ERR_DIR_END;
+		node = latch->get_next();
+
+		if(node->type == N_DIR)
+		{
+			if(want_dir)
+			{
+				dirent->size = -1;
+				dirent->mtime = 0;	// not currently supported for dirs
+				break;
+			}
+		}
+		else if(node->type == N_FILE)
+		{
+			// (note: filter = 0 matches anything)
+			if(match_wildcard(node->exact_name, filter))
+			{
+				dirent->size  = node->u.file.size;
+				dirent->mtime = node->u.file.mtime;
+				break;
+			}
+		}
+#ifndef NDEBUG
+		else
+			debug_warn("invalid TNode type");
+#endif
+	}
+
+	// success; set shared fields
+	dirent->name = node->exact_name;
+	return 0;
+}
+
+
+int tree_close_dir(void* latch_)
+{
+	NodeLatch* latch = (NodeLatch*)latch_;
+	delete latch;
+	return 0;
+}

source/lib/res/vfs_tree.h

@@ -0,0 +1,87 @@
+struct TMountPoint;
+
+class TDir;
+
+struct TFile
+{
+	// required:
+	const TMountPoint* mount_point;
+	// allocated and owned by caller (mount code)
+
+	time_t mtime;
+	off_t size;
+
+	// these can also be extracted from TMountPoint,
+	// but better cache coherency when accessing them here.
+	u32 pri : 16;
+	u32 in_archive : 1;
+
+	// note: this is basically the constructor (C++ can't call it directly
+	// since this object is stored in a union)
+	void init()
+	{
+		mount_point = 0;
+		mtime = 0;
+		size = 0;
+	}
+};
+
+// keep in sync with vfs.h vfsDirEnt!
+struct TDirent
+{
+	const char* name;
+	off_t size;
+	time_t mtime;
+};
+
+enum TreeLookupFlags
+{
+	LF_CREATE_MISSING = 1,
+	LF_START_DIR      = 2
+};
+
+
+
+extern void tree_init();
+extern void tree_clear();
+extern void tree_display();
+
+extern TFile* tree_add_file(TDir* dir, const char* name);
+extern TDir* tree_add_dir(TDir* parent, const char* path, const TMountPoint*);
+
+
+// starting at VFS root, traverse <path> and pass back information
+// for its last directory component.
+//
+// if <flags> & LF_CREATE_MISSING, all missing subdirectory components are
+//   added to the VFS.
+// if <flags> & LF_START_DIR, traversal starts at *pdir
+//   (used when looking up paths relative to a mount point).
+// if <exact_path> != 0, it receives a copy of <path> with the exact
+//   case of each component as returned by the OS (useful for calling
+//   external case-sensitive code). must hold at least VFS_MAX_PATH chars.
+//
+// <path> can be to a file or dir (in which case it must end in '/',
+// to make sure the last component is treated as a directory).
+//
+// return 0 on success, or a negative error code
+// (in which case output params are undefined).
+extern int tree_lookup_dir(const char* path, TDir** pdir, uint flags = 0, char* exact_path = 0);
+
+
+// pass back file information for <path> (relative to VFS root).
+//
+// if <flags> & LF_CREATE_MISSING, the file is added to VFS unless
+//   a higher-priority file of the same name already exists
+//   (used by VFile_reload when opening for writing).
+// if <exact_path> != 0, it receives a copy of <path> with the exact
+//   case of each component as returned by the OS (useful for calling
+//   external case-sensitive code). must hold at least VFS_MAX_PATH chars.
+//
+// return 0 on success, or a negative error code
+// (in which case output params are undefined).
+extern int tree_lookup(const char* path, TFile** pfile, uint flags = 0, char* exact_path = 0);
+
+extern int tree_open_dir(const char* path_slash, void** latch);
+extern int tree_next_dirent(void* latch_, const char* filter, TDirent* dirent);
+extern int tree_close_dir(void* latch_);

source/lib/res/zip.cpp

@@ -418,29 +418,6 @@ struct LookupInfo
 };
 
 
-// write a lower-case copy of <src> to <dst>, which holds <buf_size> bytes.
-// up to buf_size-1 chars are written; we always 0-terminate the output!
-//
-// this routine is used to convert OS and user-specified filenames
-// to lowercase before hashing them and then comparing.
-static void copy_lower_case(char* dst, const char* src, size_t buf_size)
-{
-	assert(buf_size > 0);	// otherwise, no room for trailing '\0'
-
-	int c;
-	do
-	{
-		c = *src++;
-		// this is the last remaining byte in the buffer.
-		// loop will exit below after writing 0-terminator.
-		if(--buf_size == 0)
-			c = '\0';
-		*dst++ = (char)tolower(c);
-	}
-	while(c != '\0');
-}
-
-
 // add file <fn> to the lookup data structure.
 // called from z_enum_files in order (0 <= idx < num_entries).
 // the first call notifies us of # entries, so we can allocate memory.
@@ -478,14 +455,7 @@ static int lookup_add_file_cb(uintptr_t user, i32 idx,
 	// adding a regular file.
 
 	assert(idx < li->num_entries);
-
-	// hash (lower case!) filename
-	char lc_fn[PATH_MAX];
-	size_t max_size = fn_len+1;	// fn not 0-terminated
-	if(max_size > PATH_MAX)		// (this avoids stupid min() type warning)
-		max_size = PATH_MAX;	// clamp to actual buffer size
-	copy_lower_case(lc_fn, fn, max_size);
-	FnHash fn_hash = fnv_hash(lc_fn);
+	FnHash fn_hash = fnv_lc_hash(fn, fn_len);
 
 	// fill ZEnt
 	ZEnt* ent = li->ents + idx;
@@ -558,10 +528,7 @@ static int lookup_free(LookupInfo* li)
 // look up ZLoc, given filename (untrusted!).
 static int lookup_get_file_info(LookupInfo* li, const char* fn, ZLoc* loc)
 {
-	// hash (lower case!) filename
-	char lc_fn[PATH_MAX];
-	copy_lower_case(lc_fn, fn, sizeof(lc_fn));
-	const FnHash fn_hash = fnv_hash(lc_fn);
+	const FnHash fn_hash = fnv_lc_hash(fn);
 
 	const FnHash* fn_hashes = li->fn_hashes;
 	const i32 num_files = li->num_files;