From 201aae41d9f2d8715560a60e8e60dd06ddabc56b Mon Sep 17 00:00:00 2001 From: janwas Date: Thu, 9 Mar 2006 21:37:23 +0000 Subject: [PATCH] file: pp_set_dir fix: safely handle case where pp.path already contains data; add TFile* member to DirEnt vfs: add VFS_MOUNT_ARCHIVABLE flag; allow archive builder to only include files from mount points with this set. VFSUtil: make EnumDirEnts more efficient while at it (uses PathPackage and filename storage) This was SVN commit r3618. --- source/lib/res/file/file.cpp | 22 +++++++++++--- source/lib/res/file/file.h | 4 +++ source/lib/res/file/vfs.h | 6 +++- source/lib/res/file/vfs_mount.cpp | 6 ++++ source/lib/res/file/vfs_mount.h | 4 ++- source/lib/res/file/vfs_optimizer.cpp | 43 +++++++++++++++++++-------- source/lib/res/file/vfs_tree.cpp | 2 ++ source/ps/GameSetup/GameSetup.cpp | 2 +- source/ps/VFSUtil.cpp | 23 +++++++------- 9 files changed, 80 insertions(+), 32 deletions(-) diff --git a/source/lib/res/file/file.cpp b/source/lib/res/file/file.cpp index 2a20b9b619..91906f29b8 100755 --- a/source/lib/res/file/file.cpp +++ b/source/lib/res/file/file.cpp @@ -63,15 +63,29 @@ // '/'. this is to allow use on portable paths; the function otherwise // does not care if paths are relative/portable/absolute. LibError pp_set_dir(PathPackage* pp, const char* dir) -{ + { // -1 allows for trailing DIR_SEP that will be added if not // already present. if(strcpy_s(pp->path, ARRAY_SIZE(pp->path)-1, dir) != 0) WARN_RETURN(ERR_PATH_LENGTH); size_t len = strlen(pp->path); - // didn't end with directory separator: add one - if(pp->path[len-1] != '/' && pp->path[len-1] != DIR_SEP) - pp->path[len++] = '/'; + // add directory separator if not already present + // .. but only check this if dir != "" (=> len-1 is safe) + if(len != 0) + { + char* last_char = pp->path+len-1; + // note: must handle both portable and native separators - + // may be either. + if(*last_char != '/' && *last_char != DIR_SEP) + { + *(last_char+1) = '/'; + // note: need to 0-terminate because pp.path is uninitialized + // and we overwrite strcpy_s's terminator above. + *(last_char+2) = '\0'; + // only bump by 1 - filename must overwrite '\0'. + len++; + } + } pp->end = pp->path+len; pp->chars_left = ARRAY_SIZE(pp->path)-len; diff --git a/source/lib/res/file/file.h b/source/lib/res/file/file.h index 9492992d0f..1cf3935c89 100755 --- a/source/lib/res/file/file.h +++ b/source/lib/res/file/file.h @@ -118,6 +118,8 @@ struct DirIterator char opaque[PATH_MAX+32]; }; +class TFile; + // information about a directory entry filled by dir_next_ent. struct DirEnt { @@ -134,6 +136,8 @@ struct DirEnt // rationale: we don't want to return a pointer to a copy because // users would have to free it (won't happen). const char* name; + + const TFile* tf; }; // return [bool] indicating whether the given DirEnt* (filled by diff --git a/source/lib/res/file/vfs.h b/source/lib/res/file/vfs.h index 0d7f3ddc69..1e9c97bb47 100755 --- a/source/lib/res/file/vfs.h +++ b/source/lib/res/file/vfs.h @@ -262,7 +262,11 @@ enum VfsMountFlags // all real directories mounted during this operation will be watched // for changes. this flag is provided to avoid watches in output-only // directories, e.g. screenshots/ (only causes unnecessary overhead). - VFS_MOUNT_WATCH = 4 + VFS_MOUNT_WATCH = 4, + + // anything mounted from here should be added to archive when + // building via vfs_optimizer. + VFS_MOUNT_ARCHIVABLE = 8 }; // mount into the VFS at , diff --git a/source/lib/res/file/vfs_mount.cpp b/source/lib/res/file/vfs_mount.cpp index b2c33f8d70..fec9c697e0 100644 --- a/source/lib/res/file/vfs_mount.cpp +++ b/source/lib/res/file/vfs_mount.cpp @@ -94,6 +94,12 @@ char mount_get_type(const Mount* m) } +bool mount_is_archivable(const Mount* m) +{ + return (m->flags & VFS_MOUNT_ARCHIVES) != 0; +} + + bool mount_should_replace(const Mount* m_old, const Mount* m_new, size_t size_old, size_t size_new, time_t mtime_old, time_t mtime_new) { diff --git a/source/lib/res/file/vfs_mount.h b/source/lib/res/file/vfs_mount.h index e76dc19ffc..af878aa084 100644 --- a/source/lib/res/file/vfs_mount.h +++ b/source/lib/res/file/vfs_mount.h @@ -97,7 +97,9 @@ extern LibError x_unmap(XFile* xf); // accessor routines that obviate the need to access Mount fields directly: // -bool mount_should_replace(const Mount* m_old, const Mount* m_new, +extern bool mount_is_archivable(const Mount* m); + +extern bool mount_should_replace(const Mount* m_old, const Mount* m_new, size_t size_old, size_t size_new, time_t mtime_old, time_t mtime_new); extern char mount_get_type(const Mount* m); diff --git a/source/lib/res/file/vfs_optimizer.cpp b/source/lib/res/file/vfs_optimizer.cpp index 3e1f435571..2b15ee6bb4 100644 --- a/source/lib/res/file/vfs_optimizer.cpp +++ b/source/lib/res/file/vfs_optimizer.cpp @@ -35,6 +35,16 @@ typedef std::vector FileNodes; //----------------------------------------------------------------------------- +// check if the file is supposed to be added to archive. +// this avoids adding e.g. screenshots (wasteful because they're never used) +// or config (bad because they are written to and that's not supported for +// archived files). +static bool is_archivable(const TFile* tf) +{ + const Mount* m = tfile_get_mount(tf); + return mount_is_archivable(m); +} + class IdMgr { FileId cur; @@ -101,13 +111,17 @@ static IdMgr id_mgr; // // time cost: 13ms for 5500 files; we therefore do not bother with // optimizations like reading from vfs_tree container directly. -class FileNodeGatherer +class FileGatherer { static void EntCb(const char* path, const DirEnt* ent, void* context) { FileNodes* file_nodes = (FileNodes*)context; - if(!DIRENT_IS_DIR(ent)) + // we only want files + if(DIRENT_IS_DIR(ent)) + return; + + if(is_archivable(ent->tf)) { const char* atom_fn = file_make_unique_fn_copy(path); file_nodes->push_back(FileNode(atom_fn)); @@ -115,7 +129,7 @@ class FileNodeGatherer } public: - FileNodeGatherer(FileNodes& file_nodes) + FileGatherer(FileNodes& file_nodes) { // jump-start allocation (avoids frequent initial reallocs) file_nodes.reserve(500); @@ -292,20 +306,23 @@ class ConnectionBuilder for(Runs::const_reverse_iterator it = runs.rbegin(); it != runs.rend(); ++it) { const Run& run = *it; - const TraceEntry* ent = run.first; - for(uint i = 0; i < run.count; i++, ent++) + for(uint i = 0; i < run.count; i++) { + const TraceEntry* te = run.first + i; // improvement: postprocess the trace and remove all IOs that would be // satisfied by our cache. often repeated IOs would otherwise potentially // be arranged badly. - if(trace_entry_causes_io(ent)) + if(trace_entry_causes_io(te)) { - // only add connection if this file exists. otherwise, - // ConnectionAdder's id_from_fn call will fail. - // note: this is relevant when trace contains - // by now deleted files. - if(vfs_exists(ent->atom_fn)) - add_connection(connections, ent->atom_fn); + // only add connection if this file exists and is in + // file_nodes list. otherwise, ConnectionAdder's + // id_from_fn call will fail. + // note: this happens when trace contains by now + // deleted or unarchivable files. + TFile* tf; + if(tree_lookup(te->atom_fn, &tf) == ERR_OK) + if(is_archivable(tf)) + add_connection(connections, te->atom_fn); } } @@ -577,7 +594,7 @@ static LibError vfs_opt_init(const char* trace_filename, const char* archive_fn_ // build 'graph' (nodes only) of all files that must be added. FileNodes file_nodes; - FileNodeGatherer gatherer(file_nodes); + FileGatherer gatherer(file_nodes); if(file_nodes.empty()) WARN_RETURN(ERR_DIR_END); diff --git a/source/lib/res/file/vfs_tree.cpp b/source/lib/res/file/vfs_tree.cpp index 0737a1589c..d6e77376d3 100644 --- a/source/lib/res/file/vfs_tree.cpp +++ b/source/lib/res/file/vfs_tree.cpp @@ -584,12 +584,14 @@ LibError tree_dir_next_ent(TreeDirIterator* d_, DirEnt* ent) case NT_DIR: ent->size = -1; ent->mtime = 0; // not currently supported for dirs + ent->tf = 0; break; case NT_FILE: { TFile* tf = (TFile*)node; ent->size = tf->size; ent->mtime = tf->mtime; + ent->tf = tf; break; } default: diff --git a/source/ps/GameSetup/GameSetup.cpp b/source/ps/GameSetup/GameSetup.cpp index 6b3641c0e8..a8260423a6 100644 --- a/source/ps/GameSetup/GameSetup.cpp +++ b/source/ps/GameSetup/GameSetup.cpp @@ -551,7 +551,7 @@ static void InitVfs(const char* argv0) (void)file_set_root_dir(argv0, "../data"); vfs_init(); - vfs_mount("", "mods/official", VFS_MOUNT_RECURSIVE|VFS_MOUNT_ARCHIVES|VFS_MOUNT_WATCH); + vfs_mount("", "mods/official", VFS_MOUNT_RECURSIVE|VFS_MOUNT_ARCHIVES|VFS_MOUNT_WATCH|VFS_MOUNT_ARCHIVABLE); vfs_mount("screenshots/", "screenshots"); vfs_mount("profiles/", "profiles", VFS_MOUNT_RECURSIVE); diff --git a/source/ps/VFSUtil.cpp b/source/ps/VFSUtil.cpp index 2fe71f6296..ef04d84529 100755 --- a/source/ps/VFSUtil.cpp +++ b/source/ps/VFSUtil.cpp @@ -74,8 +74,8 @@ LibError VFSUtil::EnumDirEnts(const CStr start_path, int flags, const char* user // (less stack usage; avoids seeks by reading all entries in a // directory consecutively) - std::deque dir_queue; - dir_queue.push_back(start_path); + std::queue dir_queue; + dir_queue.push(file_make_unique_fn_copy(start_path.c_str())); // for each directory: do @@ -83,13 +83,11 @@ LibError VFSUtil::EnumDirEnts(const CStr start_path, int flags, const char* user // get current directory path from queue // note: can't refer to the queue contents - those are invalidated // as soon as a directory is pushed onto it. - char path[VFS_MAX_PATH]; - vfs_path_append(path, dir_queue.front().c_str(), ""); - // vfs_open_dir checks this, so ignore failure - const size_t path_len = strlen(path); - dir_queue.pop_front(); + PathPackage pp; + (void)pp_set_dir(&pp, dir_queue.front()); + dir_queue.pop(); - Handle hdir = vfs_dir_open(path); + Handle hdir = vfs_dir_open(pp.path); if(hdir <= 0) { debug_warn("vfs_open_dir failed"); @@ -101,18 +99,19 @@ LibError VFSUtil::EnumDirEnts(const CStr start_path, int flags, const char* user while(vfs_dir_next_ent(hdir, &ent, filter) == 0) { // build complete path (DirEnt only stores entry name) - strcpy_s(path+path_len, VFS_MAX_PATH-path_len, ent.name); + (void)pp_append_file(&pp, ent.name); + const char* atom_path = file_make_unique_fn_copy(pp.path); if(DIRENT_IS_DIR(&ent)) { if(recursive) - dir_queue.push_back(path); + dir_queue.push(atom_path); if(want_dir) - cb(path, &ent, context); + cb(atom_path, &ent, context); } else - cb(path, &ent, context); + cb(atom_path, &ent, context); } vfs_dir_close(hdir);