phosit/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

file_cache: make infinite loop warning less likely

Browse Source 
vfs_mount: only notify archive builder of files that are archivable
vfs_optimizer: fix should-rebuild logic; make 2 criteria optional and
disable in final release for convenience

This was SVN commit r3651.
This commit is contained in:
janwas 2006-03-16 18:57:09 +00:00
parent 3f95dcb6ba
commit 6eda8c2209
4 changed files with 75 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
source/lib/res/file/file_cache.cpp
View File

@ -1016,13 +1016,20 @@ FileIOBuf file_buf_alloc(size_t size, const char* atom_fn, bool long_lived)
// discarded_buf may be the least valuable entry in cache, but if
// still in use (i.e. extant), it must not actually be freed yet!
if(extant_bufs.find(discarded_buf) == -1)
{
free_padded_buf(discarded_buf, size);
// optional: this iteration doesn't really count because no
// memory was actually freed. helps prevent infinite loop
// warning without having to raise the limit really high.
attempts--;
}
// note: this may seem hefty, but 300 is known to be reached.
// (after building archive, file cache is full; attempting to
// allocate ~4MB while only freeing small blocks scattered over
// the entire cache can take a while)
if(attempts++ > 500)
if(++attempts > 500)
debug_warn("possible infinite loop: failed to make room in cache");
}

19
source/lib/res/file/vfs_mount.cpp
View File

@ -378,11 +378,20 @@ static LibError add_ent(TDir* td, DirEnt* ent, const char* P_parent_path, const
// (see enqueue_archive)
return ERR_OK;
// prepend parent path to get complete pathname.
char V_path[PATH_MAX];
CHECK_ERR(vfs_path_append(V_path, tfile_get_atom_fn((TFile*)td), name));
const char* atom_fn = file_make_unique_fn_copy(V_path);
vfs_opt_notify_loose_file(atom_fn);
// notify archive builder that this file could be archived but
// currently isn't; if there are too many of these, archive will be
// rebuilt.
// note: check if archivable to exclude stuff like screenshots
// from counting towards the threshold.
if(mount_is_archivable(m))
{
// prepend parent path to get complete pathname.
char V_path[PATH_MAX];
CHECK_ERR(vfs_path_append(V_path, tfile_get_atom_fn((TFile*)td), name));
const char* atom_fn = file_make_unique_fn_copy(V_path);
vfs_opt_notify_loose_file(atom_fn);
}
// it's a regular data file; add it to the directory.
return tree_add_file(td, name, m, ent->size, ent->mtime, 0);

85
source/lib/res/file/vfs_optimizer.cpp
View File

@ -475,33 +475,48 @@ public:
// autobuild logic: decides when to (re)build an archive.
//-----------------------------------------------------------------------------
// for each loose or archived file encountered during mounting: add to a
// std::set; if there are more than *_THRESHOLD non-archived files, rebuild.
// this ends up costing 50ms for 5000 files, so disable it in final release.
#ifndef FINAL
# define AB_COUNT_LOOSE_FILES 1
#else
# define AB_COUNT_LOOSE_FILES 0
#endif
// rebuild if the archive is much older than most recent VFS timestamp.
// this makes sense during development: the archive will periodically be
// rebuilt with the newest trace. however, it would be annoying in the
// final release, where users will frequently mod things, which should not
// end up rebuilding the main archive.
#ifndef FINAL
# define AB_COMPARE_MTIME 1
#else
# define AB_COMPARE_MTIME 0
#endif
#if AB_COUNT_LOOSE_FILES
static const ssize_t REBUILD_MAIN_ARCHIVE_THRESHOLD = 50;
static const ssize_t BUILD_MINI_ARCHIVE_THRESHOLD = 20;
typedef std::vector<const char*> FnVector;
static FnVector loose_files;
static ssize_t loose_file_total, non_loose_file_total;
static std::set<const char*> loose;
static std::set<const char*> archive;
typedef std::set<const char*> FnSet;
static FnSet loose_files;
static FnSet archived_files;
#endif
void vfs_opt_notify_loose_file(const char* atom_fn)
{
loose_file_total++;
loose.insert(atom_fn);
// only add if it's not yet clear the main archive will be
// rebuilt anyway (otherwise we'd just waste time and memory)
if(loose_files.size() > REBUILD_MAIN_ARCHIVE_THRESHOLD)
loose_files.push_back(atom_fn);
#if AB_COUNT_LOOSE_FILES
// note: files are added before archives, so we can't stop adding to
// set after one of the above thresholds are reached.
loose_files.insert(atom_fn);
#endif
}
void vfs_opt_notify_non_loose_file(const char* atom_fn)
{
archive.insert(atom_fn);
non_loose_file_total++;
#if AB_COUNT_LOOSE_FILES
archived_files.insert(atom_fn);
#endif
}
@ -544,29 +559,30 @@ static bool should_rebuild_main_archive(const char* trace_filename,
const char* archive_fn_fmt, DirEnts& dirents)
{
// if there's no trace file, no point in building a main archive.
struct stat s;
if(file_stat(trace_filename, &s) != ERR_OK)
// (we wouldn't know how to order the files)
if(!file_exists(trace_filename))
return false;
// otherwise, if trace is up-to-date, stop recording a new one.
const time_t vfs_mtime = tree_most_recent_mtime();
if(s.st_mtime >= vfs_mtime)
trace_enable(false);
const ssize_t loose_files_only = loose_file_total - non_loose_file_total;
#if AB_COUNT_LOOSE_FILES
// too many (eligible for archiving!) loose files not in archive: rebuild.
const ssize_t loose_files_only = (ssize_t)loose_files.size() - (ssize_t)archived_files.size();
if(loose_files_only >= REBUILD_MAIN_ARCHIVE_THRESHOLD)
return true;
#endif
// scan dir and see what archives are already present
// scan dir and see what archives are already present..
{
ArchiveScanner archive_scanner(archive_fn_fmt);
for(DirEnts::iterator it = dirents.begin(); it != dirents.end(); ++it)
// note: a loop is more convenient than std::for_each, which would
// require referencing the returned functor (since param is a copy).
for(DirEnts::const_iterator it = dirents.begin(); it != dirents.end(); ++it)
archive_scanner(*it);
// .. 3 or more archives in dir: rebuild so that they'll be
// merged into one archive and the rest deleted
if(archive_scanner.num_archives >= 3)
// .. no archive yet OR 'lots' of them: rebuild so that they'll be
// merged into one archive and the rest deleted.
if(archive_scanner.num_archives == 0 || archive_scanner.num_archives >= 4)
return true;
// .. dev builds only: archive is much older than most recent data.
#ifndef FINAL
#if AB_COMPARE_MTIME
// .. archive is much older than most recent data: rebuild.
const double max_diff = 14*86400; // 14 days
if(difftime(tree_most_recent_mtime(), archive_scanner.most_recent_archive_mtime) > max_diff)
return true;
@ -576,6 +592,8 @@ static bool should_rebuild_main_archive(const char* trace_filename,
return false;
}
//-----------------------------------------------------------------------------
static ArchiveBuildState ab;
static std::vector<const char*> fn_vector;
@ -651,9 +669,12 @@ static int vfs_opt_continue()
static bool should_build_mini_archive(const char* UNUSED(mini_archive_fn_fmt))
{
const ssize_t loose_files_only = loose_file_total - non_loose_file_total;
#if AB_COUNT_LOOSE_FILES
// too many (eligible for archiving!) loose files not in archive
const ssize_t loose_files_only = (ssize_t)loose_files.size() - (ssize_t)archived_files.size();
if(loose_files_only >= BUILD_MINI_ARCHIVE_THRESHOLD)
return true;
#endif
return false;
}

2
source/lib/res/file/vfs_tree.cpp
View File

@ -425,8 +425,6 @@ static LibError lookup(TDir* td, const char* path, uint flags, TNode** pnode)
// - no NLSO shutdown order issues; validity is well defined
// (namely between tree_init and tree_shutdown)
// - bonus: tree_init can use it when checking if called twice.
//
//
static TDir* tree_root;