Remove VFS cache, because it is less effective and less efficient than the OS cache (and partially redundant with higher level application caches).
Patch By: Sandarac Discussed with: Philip, echotangoecho, Bezerra Fixes #4072. Differential Revision: https://code.wildfiregames.com/D587 This was SVN commit r20639.
This commit is contained in:
parent
3bfe10f63f
commit
89e339dd16
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2012 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
* This file is part of 0 A.D.
|
||||
*
|
||||
* 0 A.D. is free software: you can redistribute it and/or modify
|
||||
@ -58,7 +58,7 @@ class TestMeshManager : public CxxTest::TestSuite
|
||||
if(DirectoryExists(CACHE_PATH))
|
||||
DeleteDirectory(CACHE_PATH);
|
||||
|
||||
g_VFS = CreateVfs(20*MiB);
|
||||
g_VFS = CreateVfs();
|
||||
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", MOD_PATH));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"collada/", DataDir()/"tests"/"collada", VFS_MOUNT_MUST_EXIST));
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2010 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
* This file is part of 0 A.D.
|
||||
*
|
||||
* 0 A.D. is free software: you can redistribute it and/or modify
|
||||
@ -19,7 +19,6 @@
|
||||
|
||||
#include "graphics/TextureConverter.h"
|
||||
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/file/vfs/vfs.h"
|
||||
#include "lib/res/h_mgr.h"
|
||||
#include "lib/tex/tex.h"
|
||||
@ -35,7 +34,7 @@ public:
|
||||
{
|
||||
DeleteDirectory(DataDir()/"_testcache"); // clean up in case the last test run failed
|
||||
|
||||
m_VFS = CreateVfs(20*MiB);
|
||||
m_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(m_VFS->Mount(L"", DataDir()/"mods"/"_test.tex", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(m_VFS->Mount(L"cache/", DataDir()/"_testcache"));
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2012 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
* This file is part of 0 A.D.
|
||||
*
|
||||
* 0 A.D. is free software: you can redistribute it and/or modify
|
||||
@ -18,7 +18,6 @@
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include "graphics/TextureManager.h"
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/external_libraries/libsdl.h"
|
||||
#include "lib/file/vfs/vfs.h"
|
||||
#include "lib/res/h_mgr.h"
|
||||
@ -36,7 +35,7 @@ public:
|
||||
{
|
||||
DeleteDirectory(DataDir()/"_testcache"); // clean up in case the last test run failed
|
||||
|
||||
m_VFS = CreateVfs(20*MiB);
|
||||
m_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(m_VFS->Mount(L"", DataDir()/"mods"/"_test.tex", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(m_VFS->Mount(L"cache/", DataDir()/"_testcache"));
|
||||
|
||||
|
@ -210,30 +210,6 @@ void stats_cb_finish()
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// file_cache
|
||||
//
|
||||
|
||||
void stats_cache(CacheRet cr, size_t size)
|
||||
{
|
||||
ENSURE(cr == CR_HIT || cr == CR_MISS);
|
||||
|
||||
#if 0
|
||||
if(cr == CR_MISS)
|
||||
{
|
||||
PairIB ret = ever_cached_files.insert(atom_fn);
|
||||
if(!ret.second) // was already cached once
|
||||
{
|
||||
conflict_miss_size_total += size;
|
||||
conflict_misses++;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
cache_count[cr]++;
|
||||
cache_size_total[cr] += size;
|
||||
}
|
||||
|
||||
void stats_block_cache(CacheRet cr)
|
||||
{
|
||||
ENSURE(cr == CR_HIT || cr == CR_MISS);
|
||||
|
@ -78,7 +78,6 @@ extern void stats_cb_start();
|
||||
extern void stats_cb_finish();
|
||||
|
||||
// file_cache
|
||||
extern void stats_cache(CacheRet cr, size_t size);
|
||||
extern void stats_block_cache(CacheRet cr);
|
||||
|
||||
// archive builder
|
||||
@ -108,7 +107,6 @@ public:
|
||||
};
|
||||
#define stats_cb_start()
|
||||
#define stats_cb_finish()
|
||||
#define stats_cache(cr, size)
|
||||
#define stats_block_cache(cr)
|
||||
#define stats_ab_connection(already_exists)
|
||||
#define file_stats_dump()
|
||||
|
@ -1,73 +0,0 @@
|
||||
/* Copyright (C) 2010 Wildfire Games.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "lib/self_test.h"
|
||||
|
||||
#include "lib/res/file/file_cache.h"
|
||||
#include "lib/rand.h"
|
||||
|
||||
class TestFileCache : public CxxTest::TestSuite
|
||||
{
|
||||
enum { TEST_ALLOC_TOTAL = 100*1000*1000 };
|
||||
public:
|
||||
void test_cache_allocator()
|
||||
{
|
||||
// allocated address -> its size
|
||||
typedef std::map<void*, size_t> AllocMap;
|
||||
AllocMap allocations;
|
||||
|
||||
// put allocator through its paces by allocating several times
|
||||
// its capacity (this ensures memory is reused)
|
||||
srand(1);
|
||||
size_t total_size_used = 0;
|
||||
while(total_size_used < TEST_ALLOC_TOTAL)
|
||||
{
|
||||
size_t size = rand(1, TEST_ALLOC_TOTAL/16);
|
||||
total_size_used += size;
|
||||
void* p;
|
||||
// until successful alloc:
|
||||
for(;;)
|
||||
{
|
||||
p = file_cache_allocator_alloc(size);
|
||||
if(p)
|
||||
break;
|
||||
// out of room - remove a previous allocation
|
||||
// .. choose one at random
|
||||
size_t chosen_idx = (size_t)rand(0, (size_t)allocations.size());
|
||||
AllocMap::iterator it = allocations.begin();
|
||||
for(; chosen_idx != 0; chosen_idx--)
|
||||
++it;
|
||||
file_cache_allocator_free(it->first, it->second);
|
||||
allocations.erase(it);
|
||||
}
|
||||
|
||||
// must not already have been allocated
|
||||
TS_ASSERT_EQUALS(allocations.find(p), allocations.end());
|
||||
allocations[p] = size;
|
||||
}
|
||||
|
||||
// reset to virginal state
|
||||
// note: even though everything has now been freed, this is
|
||||
// necessary since the freelists may be a bit scattered already.
|
||||
file_cache_allocator_reset();
|
||||
}
|
||||
};
|
@ -1,253 +0,0 @@
|
||||
/* Copyright (C) 2010 Wildfire Games.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* cache of file contents (supports zero-copy IO)
|
||||
*/
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "lib/file/vfs/file_cache.h"
|
||||
|
||||
#include "lib/external_libraries/suppress_boost_warnings.h"
|
||||
|
||||
#include "lib/file/common/file_stats.h"
|
||||
#include "lib/adts/cache_adt.h"
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "lib/allocators/allocator_checker.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "lib/allocators/headerless.h"
|
||||
#include "lib/sysdep/os_cpu.h" // os_cpu_PageSize
|
||||
#include "lib/posix/posix_mman.h" // mprotect
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// allocator
|
||||
|
||||
/*
|
||||
the biggest worry of a file cache is external fragmentation. there are two
|
||||
basic ways to combat this:
|
||||
1) 'defragment' periodically - move blocks around to increase
|
||||
size of available 'holes'.
|
||||
2) prevent fragmentation from occurring at all via
|
||||
deliberate alloc/free policy.
|
||||
|
||||
file contents are returned directly to the user (zero-copy IO), so only
|
||||
currently unreferenced blocks can be moved. it is believed that this would
|
||||
severely hamper defragmentation; we therefore go with the latter approach.
|
||||
|
||||
the basic insight is: fragmentation occurs when a block is freed whose
|
||||
neighbors are not free (thus preventing coalescing). this can be prevented by
|
||||
allocating objects of similar lifetimes together. typical workloads
|
||||
(uniform access frequency) already show such behavior: the Landlord cache
|
||||
manager evicts files in an LRU manner, which matches the allocation policy.
|
||||
|
||||
references:
|
||||
"The Memory Fragmentation Problem - Solved?" (Johnstone and Wilson)
|
||||
"Dynamic Storage Allocation - A Survey and Critical Review" (Johnstone and Wilson)
|
||||
*/
|
||||
|
||||
// shared_ptr<u8>s must own a reference to their allocator to ensure it's extant when
|
||||
// they are freed. it is stored in the shared_ptr deleter.
|
||||
class Allocator;
|
||||
typedef shared_ptr<Allocator> PAllocator;
|
||||
|
||||
class FileCacheDeleter
|
||||
{
|
||||
public:
|
||||
FileCacheDeleter(size_t size, const PAllocator& allocator)
|
||||
: m_size(size), m_allocator(allocator)
|
||||
{
|
||||
}
|
||||
|
||||
// (this uses Allocator and must come after its definition)
|
||||
void operator()(u8* mem) const;
|
||||
|
||||
private:
|
||||
size_t m_size;
|
||||
PAllocator m_allocator;
|
||||
};
|
||||
|
||||
|
||||
// adds statistics and AllocatorChecker to a HeaderlessAllocator
|
||||
class Allocator
|
||||
{
|
||||
public:
|
||||
Allocator(size_t maxSize)
|
||||
: m_allocator(maxSize)
|
||||
{
|
||||
}
|
||||
|
||||
shared_ptr<u8> Allocate(size_t size, const PAllocator& pthis)
|
||||
{
|
||||
const size_t alignedSize = Align<maxSectorSize>(size);
|
||||
|
||||
u8* mem = (u8*)m_allocator.Allocate(alignedSize);
|
||||
if(!mem)
|
||||
return DummySharedPtr<u8>(0); // (prevent FileCacheDeleter from seeing a null pointer)
|
||||
|
||||
#ifndef NDEBUG
|
||||
m_checker.OnAllocate(mem, alignedSize);
|
||||
#endif
|
||||
|
||||
stats_buf_alloc(size, alignedSize);
|
||||
return shared_ptr<u8>(mem, FileCacheDeleter(size, pthis));
|
||||
}
|
||||
|
||||
void Deallocate(u8* mem, size_t size)
|
||||
{
|
||||
const size_t alignedSize = Align<maxSectorSize>(size);
|
||||
|
||||
// (re)allow writes in case the buffer was made read-only. it would
|
||||
// be nice to unmap the buffer, but this is not possible because
|
||||
// HeaderlessAllocator needs to affix boundary tags.
|
||||
(void)mprotect(mem, size, PROT_READ|PROT_WRITE);
|
||||
|
||||
#ifndef NDEBUG
|
||||
m_checker.OnDeallocate(mem, alignedSize);
|
||||
#endif
|
||||
m_allocator.Deallocate(mem, alignedSize);
|
||||
|
||||
stats_buf_free();
|
||||
}
|
||||
|
||||
private:
|
||||
HeaderlessAllocator m_allocator;
|
||||
|
||||
#ifndef NDEBUG
|
||||
AllocatorChecker m_checker;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
void FileCacheDeleter::operator()(u8* mem) const
|
||||
{
|
||||
m_allocator->Deallocate(mem, m_size);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// FileCache::Impl
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// since users are strongly encouraged to only load/process one file at a
|
||||
// time, there won't be many active references to cache entries. we could
|
||||
// take advantage of this with a separate extant list, but the cache's
|
||||
// hash map should be fast enough and this way is less work than maintaining
|
||||
// (possibly disjunct) cached and extant lists.
|
||||
|
||||
class FileCache::Impl
|
||||
{
|
||||
public:
|
||||
Impl(size_t maxSize)
|
||||
: m_allocator(new Allocator(maxSize))
|
||||
{
|
||||
}
|
||||
|
||||
shared_ptr<u8> Reserve(size_t size)
|
||||
{
|
||||
// (should never happen because the VFS ensures size != 0.)
|
||||
ENSURE(size != 0);
|
||||
|
||||
// (300 iterations have been observed when reserving several MB
|
||||
// of space in a full cache)
|
||||
for(;;)
|
||||
{
|
||||
{
|
||||
shared_ptr<u8> data = m_allocator->Allocate(size, m_allocator);
|
||||
if(data)
|
||||
return data;
|
||||
}
|
||||
|
||||
// remove least valuable entry from cache (if users are holding
|
||||
// references, the contents won't actually be deallocated)
|
||||
{
|
||||
shared_ptr<u8> discardedData; size_t discardedSize;
|
||||
bool removed = m_cache.remove_least_valuable(&discardedData, &discardedSize);
|
||||
// the cache is empty, and allocation still failed.
|
||||
// apparently the cache is full of data that's still
|
||||
// referenced, so we can't reserve any more space.
|
||||
if(!removed)
|
||||
return shared_ptr<u8>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Add(const VfsPath& pathname, const shared_ptr<u8>& data, size_t size, size_t cost)
|
||||
{
|
||||
// zero-copy cache => all users share the contents => must not
|
||||
// allow changes. this will be reverted when deallocating.
|
||||
(void)mprotect((void*)data.get(), size, PROT_READ);
|
||||
|
||||
m_cache.add(pathname, data, size, cost);
|
||||
}
|
||||
|
||||
bool Retrieve(const VfsPath& pathname, shared_ptr<u8>& data, size_t& size)
|
||||
{
|
||||
// (note: don't call stats_cache because we don't know the file size
|
||||
// in case of a cache miss; doing so is left to the caller.)
|
||||
stats_buf_ref();
|
||||
|
||||
return m_cache.retrieve(pathname, data, &size);
|
||||
}
|
||||
|
||||
void Remove(const VfsPath& pathname)
|
||||
{
|
||||
m_cache.remove(pathname);
|
||||
|
||||
// note: we could check if someone is still holding a reference
|
||||
// to the contents, but that currently doesn't matter.
|
||||
}
|
||||
|
||||
private:
|
||||
typedef Cache<VfsPath, shared_ptr<u8> > CacheType;
|
||||
CacheType m_cache;
|
||||
|
||||
PAllocator m_allocator;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
FileCache::FileCache(size_t size)
|
||||
: impl(new Impl(size))
|
||||
{
|
||||
}
|
||||
|
||||
shared_ptr<u8> FileCache::Reserve(size_t size)
|
||||
{
|
||||
return impl->Reserve(size);
|
||||
}
|
||||
|
||||
void FileCache::Add(const VfsPath& pathname, const shared_ptr<u8>& data, size_t size, size_t cost)
|
||||
{
|
||||
impl->Add(pathname, data, size, cost);
|
||||
}
|
||||
|
||||
void FileCache::Remove(const VfsPath& pathname)
|
||||
{
|
||||
impl->Remove(pathname);
|
||||
}
|
||||
|
||||
bool FileCache::Retrieve(const VfsPath& pathname, shared_ptr<u8>& data, size_t& size)
|
||||
{
|
||||
return impl->Retrieve(pathname, data, size);
|
||||
}
|
@ -1,108 +0,0 @@
|
||||
/* Copyright (C) 2010 Wildfire Games.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* cache of file contents (supports zero-copy IO)
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_FILE_CACHE
|
||||
#define INCLUDED_FILE_CACHE
|
||||
|
||||
#include "lib/file/vfs/vfs_path.h"
|
||||
|
||||
/**
|
||||
* cache of file contents with support for zero-copy IO.
|
||||
* this works by reserving a region of the cache, using it as the IO buffer,
|
||||
* and returning the memory directly to users. optional write-protection
|
||||
* via MMU ensures that the shared contents aren't inadvertently changed.
|
||||
*
|
||||
* (unique copies of) VFS pathnames are used as lookup key and owner tag.
|
||||
*
|
||||
* to ensure efficient operation and prevent fragmentation, only one
|
||||
* reference should be active at a time. in other words, read a file,
|
||||
* process it, and only then start reading the next file.
|
||||
*
|
||||
* rationale: this is rather similar to BlockCache; however, the differences
|
||||
* (Reserve's size parameter, eviction policies) are enough to warrant
|
||||
* separate implementations.
|
||||
**/
|
||||
class FileCache
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* @param size maximum amount [bytes] of memory to use for the cache.
|
||||
* (managed as a virtual memory region that's committed on-demand)
|
||||
**/
|
||||
FileCache(size_t size);
|
||||
|
||||
/**
|
||||
* Reserve a chunk of the cache's memory region.
|
||||
*
|
||||
* @param size required number of bytes (more may be allocated due to
|
||||
* alignment and/or internal fragmentation)
|
||||
* @return memory suitably aligned for IO; never fails.
|
||||
*
|
||||
* it is expected that this data will be Add()-ed once its IO completes.
|
||||
**/
|
||||
shared_ptr<u8> Reserve(size_t size);
|
||||
|
||||
/**
|
||||
* Add a file's contents to the cache.
|
||||
*
|
||||
* The cache will be able to satisfy subsequent Retrieve() calls by
|
||||
* returning this data; if CONFIG2_CACHE_READ_ONLY, the buffer is made
|
||||
* read-only. If need be and no references are currently attached to it,
|
||||
* the memory can also be commandeered by Reserve().
|
||||
*
|
||||
* @param data
|
||||
* @param size
|
||||
* @param pathname key that will be used to Retrieve file contents.
|
||||
* @param cost is the expected cost of retrieving the file again and
|
||||
* influences how/when it is evicted from the cache.
|
||||
**/
|
||||
void Add(const VfsPath& pathname, const shared_ptr<u8>& data, size_t size, size_t cost = 1);
|
||||
|
||||
/**
|
||||
* Remove a file's contents from the cache (if it exists).
|
||||
*
|
||||
* this ensures subsequent reads of the files see the current, presumably
|
||||
* recently changed, contents of the file.
|
||||
*
|
||||
* this would typically be called in response to a notification that a
|
||||
* file has changed.
|
||||
**/
|
||||
void Remove(const VfsPath& pathname);
|
||||
|
||||
/**
|
||||
* Attempt to retrieve a file's contents from the file cache.
|
||||
*
|
||||
* @return whether the contents were successfully retrieved; if so,
|
||||
* data references the read-only file contents.
|
||||
**/
|
||||
bool Retrieve(const VfsPath& pathname, shared_ptr<u8>& data, size_t& size);
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
shared_ptr<Impl> impl;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_FILE_CACHE
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2014 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
@ -33,7 +33,6 @@
|
||||
#include "lib/file/vfs/vfs_tree.h"
|
||||
#include "lib/file/vfs/vfs_lookup.h"
|
||||
#include "lib/file/vfs/vfs_populate.h"
|
||||
#include "lib/file/vfs/file_cache.h"
|
||||
|
||||
static const StatusDefinition vfsStatusDefinitions[] = {
|
||||
{ ERR::VFS_DIR_NOT_FOUND, L"VFS directory not found" },
|
||||
@ -54,9 +53,7 @@ struct ScopedLock
|
||||
class VFS : public IVFS
|
||||
{
|
||||
public:
|
||||
VFS(size_t cacheSize)
|
||||
: m_cacheSize(cacheSize), m_fileCache(m_cacheSize)
|
||||
, m_trace(CreateDummyTrace(8*MiB))
|
||||
VFS() : m_trace(CreateDummyTrace(8*MiB))
|
||||
{
|
||||
}
|
||||
|
||||
@ -82,7 +79,9 @@ public:
|
||||
virtual Status GetFileInfo(const VfsPath& pathname, CFileInfo* pfileInfo) const
|
||||
{
|
||||
ScopedLock s;
|
||||
VfsDirectory* directory; VfsFile* file;
|
||||
VfsDirectory* directory;
|
||||
VfsFile* file;
|
||||
|
||||
Status ret = vfs_Lookup(pathname, &m_rootDirectory, directory, &file);
|
||||
if(!pfileInfo) // just indicate if the file exists without raising warnings.
|
||||
return ret;
|
||||
@ -145,10 +144,6 @@ public:
|
||||
const OsPath name = pathname.Filename();
|
||||
RETURN_STATUS_IF_ERR(realDirectory->Store(name, fileContents, size));
|
||||
|
||||
// wipe out any cached blocks. this is necessary to cover the (rare) case
|
||||
// of file cache contents predating the file write.
|
||||
m_fileCache.Remove(pathname);
|
||||
|
||||
const VfsFile file(name, size, time(0), realDirectory->Priority(), realDirectory);
|
||||
directory->AddFile(file);
|
||||
|
||||
@ -176,9 +171,6 @@ public:
|
||||
RealDirectory realDirectory(file->Loader()->Path(), file->Priority(), directory->AssociatedDirectory()->Flags());
|
||||
RETURN_STATUS_IF_ERR(realDirectory.Store(pathname.Filename(), fileContents, size));
|
||||
|
||||
// See comment in CreateFile
|
||||
m_fileCache.Remove(pathname);
|
||||
|
||||
directory->AddFile(*file);
|
||||
|
||||
m_trace->NotifyStore(pathname, size);
|
||||
@ -188,36 +180,20 @@ public:
|
||||
virtual Status LoadFile(const VfsPath& pathname, shared_ptr<u8>& fileContents, size_t& size)
|
||||
{
|
||||
ScopedLock s;
|
||||
const bool isCacheHit = m_fileCache.Retrieve(pathname, fileContents, size);
|
||||
if(!isCacheHit)
|
||||
{
|
||||
VfsDirectory* directory; VfsFile* file;
|
||||
// per 2010-05-01 meeting, this shouldn't raise 'scary error
|
||||
// dialogs', which might fail to display the culprit pathname
|
||||
// instead, callers should log the error, including pathname.
|
||||
RETURN_STATUS_IF_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, &file));
|
||||
|
||||
fileContents = DummySharedPtr((u8*)0);
|
||||
size = file->Size();
|
||||
if(size != 0) // (the file cache can't handle zero-length allocations)
|
||||
{
|
||||
if(size < m_cacheSize/2) // (avoid evicting lots of previous data)
|
||||
fileContents = m_fileCache.Reserve(size);
|
||||
if(fileContents)
|
||||
{
|
||||
RETURN_STATUS_IF_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size()));
|
||||
m_fileCache.Add(pathname, fileContents, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
RETURN_STATUS_IF_ERR(AllocateAligned(fileContents, size, maxSectorSize));
|
||||
RETURN_STATUS_IF_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size()));
|
||||
}
|
||||
}
|
||||
}
|
||||
VfsDirectory* directory; VfsFile* file;
|
||||
// per 2010-05-01 meeting, this shouldn't raise 'scary error
|
||||
// dialogs', which might fail to display the culprit pathname
|
||||
// instead, callers should log the error, including pathname.
|
||||
RETURN_STATUS_IF_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, &file));
|
||||
|
||||
fileContents = DummySharedPtr((u8*)0);
|
||||
size = file->Size();
|
||||
|
||||
RETURN_STATUS_IF_ERR(AllocateAligned(fileContents, size, maxSectorSize));
|
||||
RETURN_STATUS_IF_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size()));
|
||||
|
||||
stats_io_user_request(size);
|
||||
stats_cache(isCacheHit? CR_HIT : CR_MISS, size);
|
||||
m_trace->NotifyLoad(pathname, size);
|
||||
|
||||
return INFO::OK;
|
||||
@ -263,7 +239,6 @@ public:
|
||||
virtual Status RemoveFile(const VfsPath& pathname)
|
||||
{
|
||||
ScopedLock s;
|
||||
m_fileCache.Remove(pathname);
|
||||
|
||||
VfsDirectory* directory; VfsFile* file;
|
||||
RETURN_STATUS_IF_ERR(vfs_Lookup(pathname, &m_rootDirectory, directory, &file));
|
||||
@ -312,15 +287,13 @@ private:
|
||||
return ERR::PATH_NOT_FOUND; // NOWARN
|
||||
}
|
||||
|
||||
size_t m_cacheSize;
|
||||
FileCache m_fileCache;
|
||||
PITrace m_trace;
|
||||
mutable VfsDirectory m_rootDirectory;
|
||||
};
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
PIVFS CreateVfs(size_t cacheSize)
|
||||
PIVFS CreateVfs()
|
||||
{
|
||||
return PIVFS(new VFS(cacheSize));
|
||||
return PIVFS(new VFS());
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2013 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
@ -140,9 +140,6 @@ struct IVFS
|
||||
* @param fileContents
|
||||
* @param size [bytes] of the contents, will match that of the file.
|
||||
* @return Status.
|
||||
*
|
||||
* rationale: disallowing partial writes simplifies file cache coherency
|
||||
* (we need only invalidate cached data when closing a newly written file).
|
||||
**/
|
||||
virtual Status CreateFile(const VfsPath& pathname, const shared_ptr<u8>& fileContents, size_t size) = 0;
|
||||
|
||||
@ -162,10 +159,6 @@ struct IVFS
|
||||
*
|
||||
* @param pathname
|
||||
* @param fileContents receives a smart pointer to the contents.
|
||||
* CAVEAT: this will be taken from the file cache if the VFS was
|
||||
* created with cacheSize != 0 and size < cacheSize. There is no
|
||||
* provision for Copy-on-Write, which means that such buffers
|
||||
* must not be modified (this is enforced via mprotect).
|
||||
* @param size receives the size [bytes] of the file contents.
|
||||
* @return Status.
|
||||
**/
|
||||
@ -202,8 +195,7 @@ struct IVFS
|
||||
virtual Status GetVirtualPath(const OsPath& realPathname, VfsPath& pathname) = 0;
|
||||
|
||||
/**
|
||||
* remove file from the virtual directory listing and evict its
|
||||
* data from the cache.
|
||||
* remove file from the virtual directory listing.
|
||||
**/
|
||||
virtual Status RemoveFile(const VfsPath& pathname) = 0;
|
||||
|
||||
@ -228,13 +220,9 @@ typedef shared_ptr<IVFS> PIVFS;
|
||||
/**
|
||||
* create an instance of a Virtual File System.
|
||||
*
|
||||
* @param cacheSize size [bytes] of memory to reserve for a file cache,
|
||||
* or zero to disable it. if small enough to fit, file contents are
|
||||
* stored here until no references remain and they are evicted.
|
||||
*
|
||||
* note: there is no limitation to a single instance, it may make sense
|
||||
* to create and destroy VFS instances during each unit test.
|
||||
**/
|
||||
LIB_API PIVFS CreateVfs(size_t cacheSize);
|
||||
LIB_API PIVFS CreateVfs();
|
||||
|
||||
#endif // #ifndef INCLUDED_VFS
|
||||
|
@ -515,7 +515,7 @@ static void RunGameOrAtlas(int argc, const char* argv[])
|
||||
}
|
||||
|
||||
Paths paths(args);
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
g_VFS->Mount(L"cache/", paths.Cache(), VFS_MOUNT_ARCHIVABLE);
|
||||
MountMods(paths, GetMods(args, INIT_MODS));
|
||||
|
||||
|
@ -39,7 +39,7 @@ class TestNetComms : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"public", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
CXeromyces::Startup();
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2016 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
* This file is part of 0 A.D.
|
||||
*
|
||||
* 0 A.D. is free software: you can redistribute it and/or modify
|
||||
@ -31,7 +31,7 @@
|
||||
CArchiveBuilder::CArchiveBuilder(const OsPath& mod, const OsPath& tempdir) :
|
||||
m_TempDir(tempdir), m_NumBaseMods(0)
|
||||
{
|
||||
m_VFS = CreateVfs(20*MiB);
|
||||
m_VFS = CreateVfs();
|
||||
|
||||
DeleteDirectory(m_TempDir/"_archivecache"); // clean up in case the last run failed
|
||||
|
||||
|
@ -27,13 +27,6 @@
|
||||
#include "lib/res/h_mgr.h"
|
||||
#include "lib/res/graphics/cursor.h"
|
||||
#include "lib/sysdep/cursor.h"
|
||||
#include "lib/sysdep/cpu.h"
|
||||
#include "lib/sysdep/gfx.h"
|
||||
#include "lib/sysdep/os_cpu.h"
|
||||
#include "lib/tex/tex.h"
|
||||
#if OS_WIN
|
||||
#include "lib/sysdep/os/win/wversion.h"
|
||||
#endif
|
||||
|
||||
#include "graphics/CinemaManager.h"
|
||||
#include "graphics/FontMetrics.h"
|
||||
@ -337,56 +330,6 @@ void Render()
|
||||
ogl_WarnIfError();
|
||||
}
|
||||
|
||||
|
||||
static size_t OperatingSystemFootprint()
|
||||
{
|
||||
#if OS_WIN
|
||||
switch(wversion_Number())
|
||||
{
|
||||
case WVERSION_2K:
|
||||
case WVERSION_XP:
|
||||
return 150;
|
||||
case WVERSION_XP64:
|
||||
return 200;
|
||||
default: // newer Windows version: assume the worst, and don't warn
|
||||
case WVERSION_VISTA:
|
||||
return 300;
|
||||
case WVERSION_7:
|
||||
return 250;
|
||||
}
|
||||
#else
|
||||
return 200;
|
||||
#endif
|
||||
}
|
||||
|
||||
static size_t ChooseCacheSize()
|
||||
{
|
||||
// (all sizes in MiB and signed to allow temporarily negative computations)
|
||||
|
||||
const ssize_t total = (ssize_t)os_cpu_MemorySize();
|
||||
// (NB: os_cpu_MemoryAvailable is useless on Linux because free memory
|
||||
// is marked as "in use" by OS caches.)
|
||||
const ssize_t os = (ssize_t)OperatingSystemFootprint();
|
||||
const ssize_t game = 300; // estimated working set
|
||||
|
||||
ssize_t cache = 400; // upper bound: total size of our data
|
||||
|
||||
// the cache reserves contiguous address space, which is a precious
|
||||
// resource on 32-bit systems, so don't use too much:
|
||||
if(ARCH_IA32 || sizeof(void*) == 4)
|
||||
cache = std::min(cache, (ssize_t)200);
|
||||
|
||||
// try to leave over enough memory for the OS and game
|
||||
cache = std::min(cache, total-os-game);
|
||||
|
||||
// always provide at least this much to ensure correct operation
|
||||
cache = std::max(cache, (ssize_t)64);
|
||||
|
||||
debug_printf("Cache: %d (total: %d) MiB\n", (int)cache, (int)total);
|
||||
return size_t(cache)*MiB;
|
||||
}
|
||||
|
||||
|
||||
ErrorReactionInternal psDisplayError(const wchar_t* UNUSED(text), size_t UNUSED(flags))
|
||||
{
|
||||
// If we're fullscreen, then sometimes (at least on some particular drivers on Linux)
|
||||
@ -488,8 +431,7 @@ static void InitVfs(const CmdLineArgs& args, int flags)
|
||||
hooks.display_error = psDisplayError;
|
||||
app_hooks_update(&hooks);
|
||||
|
||||
const size_t cacheSize = ChooseCacheSize();
|
||||
g_VFS = CreateVfs(cacheSize);
|
||||
g_VFS = CreateVfs();
|
||||
|
||||
const OsPath readonlyConfig = paths.RData()/"config"/"";
|
||||
g_VFS->Mount(L"config/", readonlyConfig);
|
||||
|
@ -51,7 +51,7 @@ JS::Value Mod::GetAvailableMods(const ScriptInterface& scriptInterface)
|
||||
// Sort modDirs so that we can do a fast lookup below
|
||||
std::sort(modDirs.begin(), modDirs.end());
|
||||
|
||||
PIVFS vfs = CreateVfs(1); // No cache needed; TODO but 0 crashes
|
||||
PIVFS vfs = CreateVfs();
|
||||
|
||||
for (DirectoryNames::iterator iter = modDirs.begin(); iter != modDirs.end(); ++iter)
|
||||
{
|
||||
|
@ -102,7 +102,7 @@ class TestScriptConversions : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"_test.sim", VFS_MOUNT_MUST_EXIST));
|
||||
}
|
||||
|
||||
|
@ -34,7 +34,7 @@ class TestCmpPathfinder : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
g_VFS->Mount(L"", DataDir()/"mods"/"mod", VFS_MOUNT_MUST_EXIST);
|
||||
g_VFS->Mount(L"", DataDir()/"mods"/"public", VFS_MOUNT_MUST_EXIST, 1); // ignore directory-not-found errors
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
|
@ -24,7 +24,7 @@ class TestComponentScripts : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
g_VFS->Mount(L"", DataDir()/"mods"/"mod", VFS_MOUNT_MUST_EXIST);
|
||||
g_VFS->Mount(L"", DataDir()/"mods"/"public", VFS_MOUNT_MUST_EXIST, 1); // ignore directory-not-found errors
|
||||
CXeromyces::Startup();
|
||||
|
@ -36,7 +36,7 @@ class TestCmpTemplateManager : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"_test.sim", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
CXeromyces::Startup();
|
||||
@ -221,7 +221,7 @@ class TestCmpTemplateManager_2 : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"mod", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"public", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
|
@ -43,7 +43,7 @@ class TestComponentManager : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"_test.sim", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
CXeromyces::Startup();
|
||||
|
@ -824,7 +824,7 @@ public:
|
||||
{
|
||||
CXeromyces::Startup();
|
||||
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"public", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2010 Wildfire Games.
|
||||
/* Copyright (C) 2017 Wildfire Games.
|
||||
* This file is part of 0 A.D.
|
||||
*
|
||||
* 0 A.D. is free software: you can redistribute it and/or modify
|
||||
@ -42,7 +42,7 @@ class TestSimulation2 : public CxxTest::TestSuite
|
||||
public:
|
||||
void setUp()
|
||||
{
|
||||
g_VFS = CreateVfs(20 * MiB);
|
||||
g_VFS = CreateVfs();
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"", DataDir()/"mods"/"_test.sim", VFS_MOUNT_MUST_EXIST));
|
||||
TS_ASSERT_OK(g_VFS->Mount(L"cache", DataDir()/"_testcache"));
|
||||
CXeromyces::Startup();
|
||||
|
Loading…
Reference in New Issue
Block a user