file: pp_set_dir fix: safely handle case where pp.path already contains data; add TFile* member to DirEnt

vfs: add VFS_MOUNT_ARCHIVABLE flag; allow archive builder to only
include files from mount points with this set.

VFSUtil: make EnumDirEnts more efficient while at it (uses PathPackage
and filename storage)

This was SVN commit r3618.

source/lib/res/file/file.cpp

@@ -63,15 +63,29 @@
 // '/'. this is to allow use on portable paths; the function otherwise
 // does not care if paths are relative/portable/absolute.
 LibError pp_set_dir(PathPackage* pp, const char* dir)
-{
+ {
 	// -1 allows for trailing DIR_SEP that will be added if not
 	// already present.
 	if(strcpy_s(pp->path, ARRAY_SIZE(pp->path)-1, dir) != 0)
 		WARN_RETURN(ERR_PATH_LENGTH);
 	size_t len = strlen(pp->path);
-	// didn't end with directory separator: add one
+	// add directory separator if not already present
-	if(pp->path[len-1] != '/' && pp->path[len-1] != DIR_SEP)
+	// .. but only check this if dir != "" (=> len-1 is safe)
-		pp->path[len++] = '/';
+	if(len != 0)
+	{
+		char* last_char = pp->path+len-1;
+		// note: must handle both portable and native separators -
+		// <dir> may be either.
+		if(*last_char != '/' && *last_char != DIR_SEP)
+		{
+			*(last_char+1) = '/';
+			// note: need to 0-terminate because pp.path is uninitialized
+			// and we overwrite strcpy_s's terminator above.
+			*(last_char+2) = '\0';
+			// only bump by 1 - filename must overwrite '\0'.
+			len++;
+		}
+	}
 
 	pp->end = pp->path+len;
 	pp->chars_left = ARRAY_SIZE(pp->path)-len;

source/lib/res/file/file.h

@@ -118,6 +118,8 @@ struct DirIterator
 	char opaque[PATH_MAX+32];
 };
 
+class TFile;
+
 // information about a directory entry filled by dir_next_ent.
 struct DirEnt
 {
@@ -134,6 +136,8 @@ struct DirEnt
 	// rationale: we don't want to return a pointer to a copy because
 	// users would have to free it (won't happen).
 	const char* name;
+
+	const TFile* tf;
 };
 
 // return [bool] indicating whether the given DirEnt* (filled by

source/lib/res/file/vfs.h

@@ -262,7 +262,11 @@ enum VfsMountFlags
 	// all real directories mounted during this operation will be watched
 	// for changes. this flag is provided to avoid watches in output-only
 	// directories, e.g. screenshots/ (only causes unnecessary overhead).
-	VFS_MOUNT_WATCH = 4
+	VFS_MOUNT_WATCH = 4,
+
+	// anything mounted from here should be added to archive when
+	// building via vfs_optimizer.
+	VFS_MOUNT_ARCHIVABLE = 8
 };
 
 // mount <P_real_dir> into the VFS at <V_mount_point>,

source/lib/res/file/vfs_mount.cpp

@@ -94,6 +94,12 @@ char mount_get_type(const Mount* m)
 }
 
 
+bool mount_is_archivable(const Mount* m)
+{
+	return (m->flags & VFS_MOUNT_ARCHIVES) != 0;
+}
+
+
 bool mount_should_replace(const Mount* m_old, const Mount* m_new,
 	size_t size_old, size_t size_new, time_t mtime_old, time_t mtime_new)
 {

source/lib/res/file/vfs_mount.h

@@ -97,7 +97,9 @@ extern LibError x_unmap(XFile* xf);
 // accessor routines that obviate the need to access Mount fields directly:
 //
 
-bool mount_should_replace(const Mount* m_old, const Mount* m_new,
+extern bool mount_is_archivable(const Mount* m);
+
+extern bool mount_should_replace(const Mount* m_old, const Mount* m_new,
 	size_t size_old, size_t size_new, time_t mtime_old, time_t mtime_new);
 
 extern char mount_get_type(const Mount* m);

source/lib/res/file/vfs_optimizer.cpp

@@ -35,6 +35,16 @@ typedef std::vector<FileNode> FileNodes;
 
 //-----------------------------------------------------------------------------
 
+// check if the file is supposed to be added to archive.
+// this avoids adding e.g. screenshots (wasteful because they're never used)
+// or config (bad because they are written to and that's not supported for
+// archived files).
+static bool is_archivable(const TFile* tf)
+{
+	const Mount* m = tfile_get_mount(tf);
+	return mount_is_archivable(m);
+}
+
 class IdMgr
 {
 	FileId cur;
@@ -101,13 +111,17 @@ static IdMgr id_mgr;
 //
 // time cost: 13ms for 5500 files; we therefore do not bother with
 // optimizations like reading from vfs_tree container directly.
-class FileNodeGatherer
+class FileGatherer
 {
 	static void EntCb(const char* path, const DirEnt* ent, void* context)
 	{
 		FileNodes* file_nodes = (FileNodes*)context;
 
-		if(!DIRENT_IS_DIR(ent))
+		// we only want files
+		if(DIRENT_IS_DIR(ent))
+			return;
+
+		if(is_archivable(ent->tf))
 		{
 			const char* atom_fn = file_make_unique_fn_copy(path);
 			file_nodes->push_back(FileNode(atom_fn));
@@ -115,7 +129,7 @@ class FileNodeGatherer
 	}
 
 public:
-	FileNodeGatherer(FileNodes& file_nodes)
+	FileGatherer(FileNodes& file_nodes)
 	{
 		// jump-start allocation (avoids frequent initial reallocs)
 		file_nodes.reserve(500);
@@ -292,20 +306,23 @@ class ConnectionBuilder
 		for(Runs::const_reverse_iterator it = runs.rbegin(); it != runs.rend(); ++it)
 		{
 			const Run& run = *it;
-			const TraceEntry* ent = run.first;
+			for(uint i = 0; i < run.count; i++)
-			for(uint i = 0; i < run.count; i++, ent++)
 			{
+				const TraceEntry* te = run.first + i;
 				// improvement: postprocess the trace and remove all IOs that would be
 				// satisfied by our cache. often repeated IOs would otherwise potentially
 				// be arranged badly.
-				if(trace_entry_causes_io(ent))
+				if(trace_entry_causes_io(te))
 				{
-					// only add connection if this file exists. otherwise,
+					// only add connection if this file exists and is in
-					// ConnectionAdder's id_from_fn call will fail.
+					// file_nodes list. otherwise, ConnectionAdder's
-					// note: this is relevant when trace contains
+					// id_from_fn call will fail.
-					// by now deleted files.
+					// note: this happens when trace contains by now
-					if(vfs_exists(ent->atom_fn))
+					// deleted or unarchivable files.
-						add_connection(connections, ent->atom_fn);
+					TFile* tf;
+					if(tree_lookup(te->atom_fn, &tf) == ERR_OK)
+						if(is_archivable(tf))
+							add_connection(connections, te->atom_fn);
 				}
 			}
 
@@ -577,7 +594,7 @@ static LibError vfs_opt_init(const char* trace_filename, const char* archive_fn_
 
 	// build 'graph' (nodes only) of all files that must be added.
 	FileNodes file_nodes;
-	FileNodeGatherer gatherer(file_nodes);
+	FileGatherer gatherer(file_nodes);
 	if(file_nodes.empty())
 		WARN_RETURN(ERR_DIR_END);
 

source/lib/res/file/vfs_tree.cpp

@@ -584,12 +584,14 @@ LibError tree_dir_next_ent(TreeDirIterator* d_, DirEnt* ent)
 	case NT_DIR:
 		ent->size = -1;
 		ent->mtime = 0;	// not currently supported for dirs
+		ent->tf    = 0;
 		break;
 	case NT_FILE:
 	{
 		TFile* tf = (TFile*)node;
 		ent->size  = tf->size;
 		ent->mtime = tf->mtime;
+		ent->tf    = tf;
 		break;
 	}
 	default:

source/ps/GameSetup/GameSetup.cpp

@@ -551,7 +551,7 @@ static void InitVfs(const char* argv0)
 	(void)file_set_root_dir(argv0, "../data");
 
 	vfs_init();
-	vfs_mount("", "mods/official", VFS_MOUNT_RECURSIVE|VFS_MOUNT_ARCHIVES|VFS_MOUNT_WATCH);
+	vfs_mount("", "mods/official", VFS_MOUNT_RECURSIVE|VFS_MOUNT_ARCHIVES|VFS_MOUNT_WATCH|VFS_MOUNT_ARCHIVABLE);
 	vfs_mount("screenshots/", "screenshots");
 	vfs_mount("profiles/", "profiles", VFS_MOUNT_RECURSIVE);
 

source/ps/VFSUtil.cpp

@@ -74,8 +74,8 @@ LibError VFSUtil::EnumDirEnts(const CStr start_path, int flags, const char* user
 	// (less stack usage; avoids seeks by reading all entries in a
 	// directory consecutively)
 
-	std::deque<CStr> dir_queue;
+	std::queue<const char*> dir_queue;
-	dir_queue.push_back(start_path);
+	dir_queue.push(file_make_unique_fn_copy(start_path.c_str()));
 
 	// for each directory:
 	do
@@ -83,13 +83,11 @@ LibError VFSUtil::EnumDirEnts(const CStr start_path, int flags, const char* user
 		// get current directory path from queue
 		// note: can't refer to the queue contents - those are invalidated
 		// as soon as a directory is pushed onto it.
-		char path[VFS_MAX_PATH];
+		PathPackage pp;
-		vfs_path_append(path, dir_queue.front().c_str(), "");
+		(void)pp_set_dir(&pp, dir_queue.front());
-			// vfs_open_dir checks this, so ignore failure
+		dir_queue.pop();
-		const size_t path_len = strlen(path);
-		dir_queue.pop_front();
 
-		Handle hdir = vfs_dir_open(path);
+		Handle hdir = vfs_dir_open(pp.path);
 		if(hdir <= 0)
 		{
 			debug_warn("vfs_open_dir failed");
@@ -101,18 +99,19 @@ LibError VFSUtil::EnumDirEnts(const CStr start_path, int flags, const char* user
 		while(vfs_dir_next_ent(hdir, &ent, filter) == 0)
 		{
 			// build complete path (DirEnt only stores entry name)
-			strcpy_s(path+path_len, VFS_MAX_PATH-path_len, ent.name);
+			(void)pp_append_file(&pp, ent.name);
+			const char* atom_path = file_make_unique_fn_copy(pp.path);
 
 			if(DIRENT_IS_DIR(&ent))
 			{
 				if(recursive)
-					dir_queue.push_back(path);
+					dir_queue.push(atom_path);
 
 				if(want_dir)
-					cb(path, &ent, context);
+					cb(atom_path, &ent, context);
 			}
 			else
-				cb(path, &ent, context);
+				cb(atom_path, &ent, context);
 		}
 
 		vfs_dir_close(hdir);