major refactor of file/io and alignment code. requires update-workspaces
. completely rewrite waio - use IOCP, add several hardcore optimizations. now outperforms the AS SSD and ATTO benchmarks when writing . refactor file interface - use LIO_READ instead of 'r', allow access to file descriptor. . completely rewrite the IO wrapper. now much more simple, less CPU overhead, adds support for pre-issue/post-completion hooks and preallocation. io::Run defaults to simple synchronous IO; use io::Parameters to get asynchronous. . add alignment.h with constants and Align() function template (more efficient than round_up for compile-time constants) . add UniqueRange - similar to C++0x unique_ptr (emulated for C++03), plus a built-in size. avoids expensive thread-safe reference counting in shared_ptr. cleanup: - move fat_time functions into archive_zip - remove no longer needed io_align and block_cache - reduce dependencies in sysdep/compiler (move parts to code_annotation.h) - move IOCP into separate file (reused by waio) This was SVN commit r9350.
This commit is contained in:
parent
7ed6a164ba
commit
2374caac3e
@ -458,7 +458,8 @@ bool CTextureConverter::Poll(CTexturePtr& texture, VfsPath& dest, bool& ok)
|
||||
|
||||
// Move output into a correctly-aligned buffer
|
||||
size_t size = result->output.buffer.size();
|
||||
shared_ptr<u8> file = io_Allocate(size);
|
||||
shared_ptr<u8> file;
|
||||
AllocateAligned(file, size, maxSectorSize);
|
||||
memcpy(file.get(), &result->output.buffer[0], size);
|
||||
if (m_VFS->CreateFile(result->dest, file, size) < 0)
|
||||
{
|
||||
|
@ -20,6 +20,7 @@
|
||||
#include "lib/file/file_system_util.h"
|
||||
#include "lib/file/vfs/vfs.h"
|
||||
#include "lib/file/io/io.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
|
||||
#include "graphics/ColladaManager.h"
|
||||
#include "graphics/MeshManager.h"
|
||||
@ -116,7 +117,8 @@ public:
|
||||
{
|
||||
copyFile(srcDAE, testDAE);
|
||||
//buildArchive();
|
||||
shared_ptr<u8> buf = io_Allocate(100);
|
||||
shared_ptr<u8> buf;
|
||||
AllocateAligned(buf, 100, maxSectorSize);
|
||||
strcpy_s((char*)buf.get(), 5, "Test");
|
||||
g_VFS->CreateFile(testDAE, buf, 4);
|
||||
}
|
||||
@ -178,7 +180,8 @@ public:
|
||||
TestLogger logger;
|
||||
|
||||
copyFile(srcDAE, testDAE);
|
||||
shared_ptr<u8> buf = io_Allocate(100);
|
||||
shared_ptr<u8> buf;
|
||||
AllocateAligned(buf, 100, maxSectorSize);
|
||||
strcpy_s((char*)buf.get(), 100, "Not valid XML");
|
||||
g_VFS->CreateFile(testSkeletonDefs, buf, 13);
|
||||
|
||||
@ -192,7 +195,8 @@ public:
|
||||
TestLogger logger;
|
||||
|
||||
copyFile(srcSkeletonDefs, testSkeletonDefs);
|
||||
shared_ptr<u8> buf = io_Allocate(100);
|
||||
shared_ptr<u8> buf;
|
||||
AllocateAligned(buf, 100, maxSectorSize);
|
||||
strcpy_s((char*)buf.get(), 100, "Not valid XML");
|
||||
g_VFS->CreateFile(testDAE, buf, 13);
|
||||
|
||||
|
105
source/lib/alignment.h
Normal file
105
source/lib/alignment.h
Normal file
@ -0,0 +1,105 @@
|
||||
#ifndef INCLUDED_ALIGNMENT
|
||||
#define INCLUDED_ALIGNMENT
|
||||
|
||||
#include "lib/sysdep/compiler.h" // MSC_VERSION
|
||||
|
||||
template<typename T>
|
||||
inline bool IsAligned(T t, uintptr_t multiple)
|
||||
{
|
||||
return (uintptr_t(t) % multiple) == 0;
|
||||
}
|
||||
|
||||
template<size_t multiple>
|
||||
inline size_t Align(size_t n)
|
||||
{
|
||||
cassert(multiple != 0 && ((multiple & (multiple-1)) == 0)); // is power of 2
|
||||
return (n + multiple-1) & ~(multiple-1);
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// SIMD vector
|
||||
//
|
||||
|
||||
static const size_t vectorSize = 16;
|
||||
|
||||
#define VERIFY_VECTOR_MULTIPLE(size)\
|
||||
VERIFY(IsAligned(size, vectorSize))
|
||||
|
||||
#define VERIFY_VECTOR_ALIGNED(pointer)\
|
||||
VERIFY_VECTOR_MULTIPLE(pointer);\
|
||||
ASSUME_ALIGNED(pointer, vectorSize)
|
||||
|
||||
|
||||
//
|
||||
// CPU cache
|
||||
//
|
||||
|
||||
static const size_t cacheLineSize = 64; // (L2)
|
||||
|
||||
#if MSC_VERSION
|
||||
#define CACHE_ALIGNED __declspec(align(64)) // align() requires a literal; keep in sync with cacheLineSize
|
||||
#endif
|
||||
|
||||
|
||||
//
|
||||
// MMU pages
|
||||
//
|
||||
|
||||
static const size_t pageSize = 0x1000; // 4 KB
|
||||
static const size_t largePageSize = 0x200000; // 2 MB
|
||||
|
||||
|
||||
// waio opens files with FILE_FLAG_NO_BUFFERING, so Windows requires
|
||||
// file offsets / buffers and sizes to be sector-aligned. querying the
|
||||
// actual sector size via GetDiskFreeSpace is inconvenient and slow.
|
||||
// we always request large blocks anyway, so just check whether inputs
|
||||
// are aligned to a `maximum' sector size. this catches common mistakes
|
||||
// before they cause scary "IO failed" errors. if the value turns out
|
||||
// to be too low, the Windows APIs will still complain.
|
||||
static const uintptr_t maxSectorSize = 0x1000;
|
||||
|
||||
#endif // #ifndef INCLUDED_ALIGNMENT
|
||||
#ifndef INCLUDED_ALIGNMENT
|
||||
#define INCLUDED_ALIGNMENT
|
||||
|
||||
template<typename T>
|
||||
inline bool IsAligned(T t, uintptr_t multiple)
|
||||
{
|
||||
return (uintptr_t(t) % multiple) == 0;
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// SIMD vector
|
||||
//
|
||||
|
||||
static const size_t vectorSize = 16;
|
||||
|
||||
#define VERIFY_VECTOR_MULTIPLE(size)\
|
||||
VERIFY(IsAligned(size, vectorSize))
|
||||
|
||||
#define VERIFY_VECTOR_ALIGNED(pointer)\
|
||||
VERIFY_VECTOR_MULTIPLE(pointer);\
|
||||
ASSUME_ALIGNED(pointer, vectorSize)
|
||||
|
||||
|
||||
//
|
||||
// CPU cache
|
||||
//
|
||||
|
||||
static const size_t cacheLineSize = 64; // (L2)
|
||||
|
||||
#if MSC_VERSION
|
||||
#define CACHE_ALIGNED __declspec(align(64)) // align() requires a literal; keep in sync with cacheLineSize
|
||||
#endif
|
||||
|
||||
|
||||
//
|
||||
// MMU pages
|
||||
//
|
||||
|
||||
static const size_t pageSize = 0x1000; // 4 KB
|
||||
static const size_t largePageSize = 0x200000; // 2 MB
|
||||
|
||||
#endif // #ifndef INCLUDED_ALIGNMENT
|
@ -27,9 +27,8 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/allocators/allocators.h"
|
||||
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/sysdep/cpu.h" // cpu_CAS
|
||||
#include "lib/bits.h"
|
||||
|
||||
#include "lib/allocators/mem_util.h"
|
||||
|
||||
|
||||
@ -157,7 +156,7 @@ void single_free(void* storage, volatile intptr_t* in_use_flag, void* p)
|
||||
|
||||
void* static_calloc(StaticStorage* ss, size_t size)
|
||||
{
|
||||
void* p = (void*)round_up((uintptr_t)ss->pos, (uintptr_t)16u);
|
||||
void* p = (void*)Align<16>((uintptr_t)ss->pos);
|
||||
ss->pos = (u8*)p+size;
|
||||
debug_assert(ss->pos <= ss->end);
|
||||
return p;
|
||||
|
@ -28,6 +28,7 @@
|
||||
#include "lib/allocators/mem_util.h"
|
||||
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/posix/posix_mman.h"
|
||||
#include "lib/sysdep/os_cpu.h" // os_cpu_PageSize
|
||||
|
||||
@ -45,8 +46,7 @@ size_t mem_RoundUpToPage(size_t size)
|
||||
size_t mem_RoundUpToAlignment(size_t size)
|
||||
{
|
||||
// all allocators should align to at least this many bytes:
|
||||
const size_t alignment = 8;
|
||||
return round_up(size, alignment);
|
||||
return Align<8>(size);
|
||||
}
|
||||
|
||||
|
||||
|
@ -29,6 +29,10 @@
|
||||
|
||||
#include "lib/allocators/mem_util.h"
|
||||
|
||||
#include "lib/timer.h"
|
||||
|
||||
TIMER_ADD_CLIENT(tc_pool_alloc);
|
||||
|
||||
|
||||
LibError pool_create(Pool* p, size_t max_size, size_t el_size)
|
||||
{
|
||||
@ -66,6 +70,7 @@ bool pool_contains(const Pool* p, void* el)
|
||||
|
||||
void* pool_alloc(Pool* p, size_t size)
|
||||
{
|
||||
TIMER_ACCRUE(tc_pool_alloc);
|
||||
// if pool allows variable sizes, go with the size parameter,
|
||||
// otherwise the pool el_size setting.
|
||||
const size_t el_size = p->el_size? p->el_size : mem_RoundUpToAlignment(size);
|
||||
|
@ -23,7 +23,7 @@
|
||||
#ifndef INCLUDED_SHARED_PTR
|
||||
#define INCLUDED_SHARED_PTR
|
||||
|
||||
#include "lib/sysdep/arch/x86_x64/cache.h"
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/sysdep/rtl.h" // rtl_AllocateAligned
|
||||
|
||||
struct DummyDeleter
|
||||
@ -61,6 +61,7 @@ struct FreeDeleter
|
||||
// (note: uses CheckedArrayDeleter)
|
||||
LIB_API shared_ptr<u8> Allocate(size_t size);
|
||||
|
||||
|
||||
struct AlignedDeleter
|
||||
{
|
||||
template<class T>
|
||||
@ -71,9 +72,13 @@ struct AlignedDeleter
|
||||
};
|
||||
|
||||
template<class T>
|
||||
inline shared_ptr<T> AllocateAligned(size_t size)
|
||||
static inline LibError AllocateAligned(shared_ptr<T>& p, size_t size, size_t alignment = cacheLineSize)
|
||||
{
|
||||
return shared_ptr<T>((T*)rtl_AllocateAligned(size, x86_x64_Caches(L2D)->entrySize), AlignedDeleter());
|
||||
void* mem = rtl_AllocateAligned(size, alignment);
|
||||
if(!mem)
|
||||
WARN_RETURN(ERR::NO_MEM);
|
||||
p.reset((T*)mem, AlignedDeleter());
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
#endif // #ifndef INCLUDED_SHARED_PTR
|
||||
|
40
source/lib/allocators/unique_range.cpp
Normal file
40
source/lib/allocators/unique_range.cpp
Normal file
@ -0,0 +1,40 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/allocators/unique_range.h"
|
||||
|
||||
#include "lib/sysdep/cpu.h" // cpu_AtomicAdd
|
||||
#include "lib/sysdep/rtl.h" // rtl_FreeAligned
|
||||
|
||||
|
||||
static void UniqueRangeDeleterNone(void* UNUSED(pointer), size_t UNUSED(size))
|
||||
{
|
||||
// (introducing this do-nothing function avoids having to check whether deleter != 0)
|
||||
}
|
||||
|
||||
static void UniqueRangeDeleterAligned(void* pointer, size_t UNUSED(size))
|
||||
{
|
||||
return rtl_FreeAligned(pointer);
|
||||
}
|
||||
|
||||
|
||||
static UniqueRangeDeleter deleters[idxDeleterBits+1] = { UniqueRangeDeleterNone, UniqueRangeDeleterAligned };
|
||||
|
||||
static IdxDeleter numDeleters = 2;
|
||||
|
||||
|
||||
IdxDeleter AddUniqueRangeDeleter(UniqueRangeDeleter deleter)
|
||||
{
|
||||
debug_assert(deleter);
|
||||
IdxDeleter idxDeleter = cpu_AtomicAdd(&numDeleters, 1);
|
||||
debug_assert(idxDeleter < (IdxDeleter)ARRAY_SIZE(deleters));
|
||||
deleters[idxDeleter] = deleter;
|
||||
return idxDeleter;
|
||||
}
|
||||
|
||||
|
||||
void CallUniqueRangeDeleter(void* pointer, size_t size, IdxDeleter idxDeleter) throw()
|
||||
{
|
||||
ASSERT(idxDeleter < numDeleters);
|
||||
// (some deleters do not tolerate null pointers)
|
||||
if(pointer)
|
||||
deleters[idxDeleter](pointer, size);
|
||||
}
|
185
source/lib/allocators/unique_range.h
Normal file
185
source/lib/allocators/unique_range.h
Normal file
@ -0,0 +1,185 @@
|
||||
#ifndef INCLUDED_UNIQUE_RANGE
|
||||
#define INCLUDED_UNIQUE_RANGE
|
||||
|
||||
|
||||
#define ASSERT debug_assert
|
||||
|
||||
|
||||
#include "lib/lib_api.h"
|
||||
|
||||
// we usually don't hold multiple references to allocations, so unique_ptr
|
||||
// can be used instead of the more complex (ICC generated incorrect code on
|
||||
// 2 occasions) and expensive shared_ptr.
|
||||
// a custom deleter is required because allocators such as ReserveAddressSpace need to
|
||||
// pass the size to their deleter. we want to mix pointers from various allocators, but
|
||||
// unique_ptr's deleter is fixed at compile-time, so it would need to be general enough
|
||||
// to handle all allocators.
|
||||
// storing the size and a function pointer would be one such solution, with the added
|
||||
// bonus of no longer requiring a complete type at the invocation of ~unique_ptr.
|
||||
// however, this inflates the pointer size to 3 words. if only a few allocator types
|
||||
// are needed, we can replace the function pointer with an index stashed into the
|
||||
// lower bits of the pointer (safe because allocations are always aligned to the
|
||||
// word size).
|
||||
typedef intptr_t IdxDeleter;
|
||||
|
||||
// no-op deleter (use when returning part of an existing allocation)
|
||||
// must be zero because reset() sets address (which includes idxDeleter) to zero.
|
||||
static const IdxDeleter idxDeleterNone = 0;
|
||||
|
||||
static const IdxDeleter idxDeleterAligned = 1;
|
||||
|
||||
// (temporary value to prevent concurrent calls to AddUniqueRangeDeleter)
|
||||
static const IdxDeleter idxDeleterBusy = -IdxDeleter(1);
|
||||
|
||||
// governs the maximum number of IdxDeleter and each pointer's alignment requirements
|
||||
static const IdxDeleter idxDeleterBits = 0x7;
|
||||
|
||||
typedef void (*UniqueRangeDeleter)(void* pointer, size_t size);
|
||||
|
||||
/**
|
||||
* @return the next available IdxDeleter and associate it with the deleter.
|
||||
* halts the program if the idxDeleterBits limit has been reached.
|
||||
*
|
||||
* thread-safe, but no attempt is made to detect whether the deleter has already been
|
||||
* registered (would require a mutex). each allocator must ensure they only call this once.
|
||||
**/
|
||||
LIB_API IdxDeleter AddUniqueRangeDeleter(UniqueRangeDeleter deleter);
|
||||
|
||||
LIB_API void CallUniqueRangeDeleter(void* pointer, size_t size, IdxDeleter idxDeleter) throw();
|
||||
|
||||
|
||||
// unfortunately, unique_ptr allows constructing without a custom deleter. to ensure callers can
|
||||
// rely upon pointers being associated with a size, we introduce a `UniqueRange' replacement.
|
||||
// its interface is identical to unique_ptr except for the constructors, the addition of
|
||||
// size() and the removal of operator bool (which avoids implicit casts to int).
|
||||
class UniqueRange
|
||||
{
|
||||
public:
|
||||
typedef void* pointer;
|
||||
typedef void element_type;
|
||||
|
||||
UniqueRange()
|
||||
{
|
||||
Set(0, 0, idxDeleterNone);
|
||||
}
|
||||
|
||||
UniqueRange(pointer p, size_t size, IdxDeleter deleter)
|
||||
{
|
||||
Set(p, size, deleter);
|
||||
}
|
||||
|
||||
UniqueRange(RVREF(UniqueRange) rvref)
|
||||
{
|
||||
UniqueRange& rhs = LVALUE(rvref);
|
||||
address_ = rhs.address_;
|
||||
size_ = rhs.size_;
|
||||
rhs.address_ = 0;
|
||||
}
|
||||
|
||||
UniqueRange& operator=(RVREF(UniqueRange) rvref)
|
||||
{
|
||||
UniqueRange& rhs = LVALUE(rvref);
|
||||
if(this != &rhs)
|
||||
{
|
||||
Delete();
|
||||
address_ = rhs.address_;
|
||||
size_ = rhs.size_;
|
||||
rhs.address_ = 0;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
~UniqueRange()
|
||||
{
|
||||
Delete();
|
||||
}
|
||||
|
||||
pointer get() const
|
||||
{
|
||||
return pointer(address_ & ~idxDeleterBits);
|
||||
}
|
||||
|
||||
IdxDeleter get_deleter() const
|
||||
{
|
||||
return IdxDeleter(address_ & idxDeleterBits);
|
||||
}
|
||||
|
||||
size_t size() const
|
||||
{
|
||||
return size_;
|
||||
}
|
||||
|
||||
// side effect: subsequent get_deleter will return idxDeleterNone
|
||||
pointer release() // relinquish ownership
|
||||
{
|
||||
pointer ret = get();
|
||||
address_ = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
void reset()
|
||||
{
|
||||
Delete();
|
||||
address_ = 0;
|
||||
}
|
||||
|
||||
void reset(pointer p, size_t size, IdxDeleter deleter)
|
||||
{
|
||||
Delete();
|
||||
Set(p, size, deleter);
|
||||
}
|
||||
|
||||
void swap(UniqueRange& rhs)
|
||||
{
|
||||
std::swap(address_, rhs.address_);
|
||||
std::swap(size_, rhs.size_);
|
||||
}
|
||||
|
||||
private:
|
||||
void Set(pointer p, size_t size, IdxDeleter deleter)
|
||||
{
|
||||
ASSERT((uintptr_t(p) & idxDeleterBits) == 0);
|
||||
ASSERT(deleter <= idxDeleterBits);
|
||||
|
||||
address_ = uintptr_t(p) | deleter;
|
||||
size_ = size;
|
||||
|
||||
ASSERT(get() == p);
|
||||
ASSERT(get_deleter() == deleter);
|
||||
ASSERT(this->size() == size);
|
||||
}
|
||||
|
||||
void Delete()
|
||||
{
|
||||
CallUniqueRangeDeleter(get(), size(), get_deleter());
|
||||
}
|
||||
|
||||
// disallow construction and assignment from lvalue
|
||||
UniqueRange(const UniqueRange&);
|
||||
UniqueRange& operator=(const UniqueRange&);
|
||||
|
||||
// (IdxDeleter is stored in the lower bits of address since size might not even be a multiple of 4.)
|
||||
uintptr_t address_;
|
||||
size_t size_;
|
||||
};
|
||||
|
||||
namespace std {
|
||||
|
||||
static inline void swap(UniqueRange& p1, UniqueRange& p2)
|
||||
{
|
||||
p1.swap(p2);
|
||||
}
|
||||
|
||||
static inline void swap(RVREF(UniqueRange) p1, UniqueRange& p2)
|
||||
{
|
||||
p2.swap(LVALUE(p1));
|
||||
}
|
||||
|
||||
static inline void swap(UniqueRange& p1, RVREF(UniqueRange) p2)
|
||||
{
|
||||
p1.swap(LVALUE(p2));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif // #ifndef INCLUDED_UNIQUE_RANGE
|
@ -230,13 +230,6 @@ inline T round_down(T n, T multiple)
|
||||
}
|
||||
|
||||
|
||||
template<typename T>
|
||||
inline bool IsAligned(T t, uintptr_t multiple)
|
||||
{
|
||||
return ((uintptr_t)t % multiple) == 0;
|
||||
}
|
||||
|
||||
|
||||
template<typename T>
|
||||
inline T MaxPowerOfTwoDivisor(T value)
|
||||
{
|
||||
|
@ -28,6 +28,7 @@
|
||||
#define INCLUDED_CODE_ANNOTATION
|
||||
|
||||
#include "lib/sysdep/compiler.h"
|
||||
#include "lib/sysdep/arch.h" // ARCH_AMD64
|
||||
|
||||
/**
|
||||
* mark a function local variable or parameter as unused and avoid
|
||||
@ -78,6 +79,20 @@
|
||||
#define UNREACHABLE // actually defined below.. this is for
|
||||
# undef UNREACHABLE // CppDoc's benefit only.
|
||||
|
||||
// this macro should not generate any fallback code; it is merely the
|
||||
// compiler-specific backend for UNREACHABLE.
|
||||
// #define it to nothing if the compiler doesn't support such a hint.
|
||||
#define HAVE_ASSUME_UNREACHABLE 1
|
||||
#if MSC_VERSION && !ICC_VERSION // (ICC ignores this)
|
||||
# define ASSUME_UNREACHABLE __assume(0)
|
||||
#elif GCC_VERSION >= 450
|
||||
# define ASSUME_UNREACHABLE __builtin_unreachable()
|
||||
#else
|
||||
# define ASSUME_UNREACHABLE
|
||||
# undef HAVE_ASSUME_UNREACHABLE
|
||||
# define HAVE_ASSUME_UNREACHABLE 0
|
||||
#endif
|
||||
|
||||
// compiler supports ASSUME_UNREACHABLE => allow it to assume the code is
|
||||
// never reached (improves optimization at the cost of undefined behavior
|
||||
// if the annotation turns out to be incorrect).
|
||||
@ -214,4 +229,150 @@ private:\
|
||||
# define COMPILER_FENCE
|
||||
#endif
|
||||
|
||||
|
||||
// try to define _W64, if not already done
|
||||
// (this is useful for catching pointer size bugs)
|
||||
#ifndef _W64
|
||||
# if MSC_VERSION
|
||||
# define _W64 __w64
|
||||
# elif GCC_VERSION
|
||||
# define _W64 __attribute__((mode (__pointer__)))
|
||||
# else
|
||||
# define _W64
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
// C99-like restrict (non-standard in C++, but widely supported in various forms).
|
||||
//
|
||||
// May be used on pointers. May also be used on member functions to indicate
|
||||
// that 'this' is unaliased (e.g. "void C::m() RESTRICT { ... }").
|
||||
// Must not be used on references - GCC supports that but VC doesn't.
|
||||
//
|
||||
// We call this "RESTRICT" to avoid conflicts with VC's __declspec(restrict),
|
||||
// and because it's not really the same as C99's restrict.
|
||||
//
|
||||
// To be safe and satisfy the compilers' stated requirements: an object accessed
|
||||
// by a restricted pointer must not be accessed by any other pointer within the
|
||||
// lifetime of the restricted pointer, if the object is modified.
|
||||
// To maximise the chance of optimisation, any pointers that could potentially
|
||||
// alias with the restricted one should be marked as restricted too.
|
||||
//
|
||||
// It would probably be a good idea to write test cases for any code that uses
|
||||
// this in an even very slightly unclear way, in case it causes obscure problems
|
||||
// in a rare compiler due to differing semantics.
|
||||
//
|
||||
// .. GCC
|
||||
#if GCC_VERSION
|
||||
# define RESTRICT __restrict__
|
||||
// .. VC8 provides __restrict
|
||||
#elif MSC_VERSION >= 1400
|
||||
# define RESTRICT __restrict
|
||||
// .. ICC supports the keyword 'restrict' when run with the /Qrestrict option,
|
||||
// but it always also supports __restrict__ or __restrict to be compatible
|
||||
// with GCC/MSVC, so we'll use the underscored version. One of {GCC,MSC}_VERSION
|
||||
// should have been defined in addition to ICC_VERSION, so we should be using
|
||||
// one of the above cases (unless it's an old VS7.1-emulating ICC).
|
||||
#elif ICC_VERSION
|
||||
# error ICC_VERSION defined without either GCC_VERSION or an adequate MSC_VERSION
|
||||
// .. unsupported; remove it from code
|
||||
#else
|
||||
# define RESTRICT
|
||||
#endif
|
||||
|
||||
|
||||
// C99-style __func__
|
||||
// .. newer GCC already have it
|
||||
#if GCC_VERSION >= 300
|
||||
// nothing need be done
|
||||
// .. old GCC and MSVC have __FUNCTION__
|
||||
#elif GCC_VERSION >= 200 || MSC_VERSION
|
||||
# define __func__ __FUNCTION__
|
||||
// .. unsupported
|
||||
#else
|
||||
# define __func__ "(unknown)"
|
||||
#endif
|
||||
|
||||
|
||||
// extern "C", but does the right thing in pure-C mode
|
||||
#if defined(__cplusplus)
|
||||
# define EXTERN_C extern "C"
|
||||
#else
|
||||
# define EXTERN_C extern
|
||||
#endif
|
||||
|
||||
|
||||
#if MSC_VERSION
|
||||
# define INLINE __forceinline
|
||||
#else
|
||||
# define INLINE inline
|
||||
#endif
|
||||
|
||||
|
||||
#if MSC_VERSION
|
||||
# define CALL_CONV __cdecl
|
||||
#else
|
||||
# define CALL_CONV
|
||||
#endif
|
||||
|
||||
|
||||
#if MSC_VERSION && !ARCH_AMD64
|
||||
# define DECORATED_NAME(name) _##name
|
||||
#else
|
||||
# define DECORATED_NAME(name) name
|
||||
#endif
|
||||
|
||||
|
||||
// workaround for preprocessor limitation: macro args aren't expanded
|
||||
// before being pasted.
|
||||
#define STRINGIZE2(id) # id
|
||||
#define STRINGIZE(id) STRINGIZE2(id)
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// partial emulation of C++0x rvalue references (required for UniqueRange)
|
||||
|
||||
#if HAVE_CPP0X
|
||||
|
||||
#define RVREF(T) T&& // the type of an rvalue reference
|
||||
#define LVALUE(rvalue) rvalue // (a named rvalue reference is an lvalue)
|
||||
#define RVALUE(lvalue) std::move(lvalue)
|
||||
#define RVALUE_FROM_R(rvalue) RVALUE(rvalue) // (see above)
|
||||
|
||||
#else
|
||||
|
||||
// RVALUE wraps an lvalue reference in this class for later use by a
|
||||
// "move ctor" that takes an RValue.
|
||||
template<typename T>
|
||||
class RValue
|
||||
{
|
||||
public:
|
||||
explicit RValue(T& lvalue): lvalue(lvalue) {}
|
||||
T& LValue() const { return lvalue; }
|
||||
|
||||
private:
|
||||
T& lvalue;
|
||||
};
|
||||
|
||||
// from rvalue or const lvalue
|
||||
template<class T>
|
||||
static inline RValue<T> ToRValue(const T& t)
|
||||
{
|
||||
return RValue<T>((T&)t);
|
||||
}
|
||||
|
||||
// from lvalue
|
||||
template<class T>
|
||||
static inline RValue<T> ToRValue(T& t)
|
||||
{
|
||||
return RValue<T>(t);
|
||||
}
|
||||
|
||||
#define RVREF(T) const RValue<T>& // the type of an rvalue reference
|
||||
#define LVALUE(rvalue) rvalue.LValue()
|
||||
#define RVALUE(lvalue) ToRValue(lvalue)
|
||||
#define RVALUE_FROM_R(rvalue) rvalue
|
||||
|
||||
#endif // #if !HAVE_CPP0X
|
||||
|
||||
#endif // #ifndef INCLUDED_CODE_ANNOTATION
|
||||
|
@ -1,77 +0,0 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* timestamp conversion: DOS FAT <-> Unix time_t
|
||||
*/
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "lib/fat_time.h"
|
||||
|
||||
#include <ctime>
|
||||
|
||||
#include "lib/bits.h"
|
||||
|
||||
|
||||
time_t time_t_from_FAT(u32 fat_timedate)
|
||||
{
|
||||
const u32 fat_time = bits(fat_timedate, 0, 15);
|
||||
const u32 fat_date = bits(fat_timedate, 16, 31);
|
||||
|
||||
struct tm t; // struct tm format:
|
||||
t.tm_sec = bits(fat_time, 0,4) * 2; // [0,59]
|
||||
t.tm_min = bits(fat_time, 5,10); // [0,59]
|
||||
t.tm_hour = bits(fat_time, 11,15); // [0,23]
|
||||
t.tm_mday = bits(fat_date, 0,4); // [1,31]
|
||||
t.tm_mon = bits(fat_date, 5,8) - 1; // [0,11]
|
||||
t.tm_year = bits(fat_date, 9,15) + 80; // since 1900
|
||||
t.tm_isdst = -1; // unknown - let libc determine
|
||||
|
||||
// otherwise: totally bogus, and at the limit of 32-bit time_t
|
||||
debug_assert(t.tm_year < 138);
|
||||
|
||||
time_t ret = mktime(&t);
|
||||
debug_assert(ret != (time_t)-1); // mktime shouldn't fail
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
u32 FAT_from_time_t(time_t time)
|
||||
{
|
||||
// (values are adjusted for DST)
|
||||
struct tm* t = localtime(&time);
|
||||
|
||||
const u16 fat_time = u16(
|
||||
(t->tm_sec/2) | // 5
|
||||
(u16(t->tm_min) << 5) | // 6
|
||||
(u16(t->tm_hour) << 11) // 5
|
||||
);
|
||||
|
||||
const u16 fat_date = u16(
|
||||
(t->tm_mday) | // 5
|
||||
(u16(t->tm_mon+1) << 5) | // 4
|
||||
(u16(t->tm_year-80) << 9) // 7
|
||||
);
|
||||
|
||||
u32 fat_timedate = u32_from_u16(fat_date, fat_time);
|
||||
return fat_timedate;
|
||||
}
|
@ -1,29 +0,0 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_FAT_TIME
|
||||
#define INCLUDED_FAT_TIME
|
||||
|
||||
extern time_t time_t_from_FAT(u32 fat_timedate);
|
||||
extern u32 FAT_from_time_t(time_t time);
|
||||
|
||||
#endif // INCLUDED_FAT_TIME
|
@ -33,7 +33,6 @@
|
||||
#include "lib/utf8.h"
|
||||
#include "lib/bits.h"
|
||||
#include "lib/byte_order.h"
|
||||
#include "lib/fat_time.h"
|
||||
#include "lib/allocators/pool.h"
|
||||
#include "lib/sysdep/filesystem.h"
|
||||
#include "lib/file/archive/archive.h"
|
||||
@ -41,8 +40,54 @@
|
||||
#include "lib/file/archive/stream.h"
|
||||
#include "lib/file/file.h"
|
||||
#include "lib/file/io/io.h"
|
||||
#include "lib/file/io/io_align.h" // BLOCK_SIZE
|
||||
#include "lib/file/io/write_buffer.h"
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// timestamp conversion: DOS FAT <-> Unix time_t
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
static time_t time_t_from_FAT(u32 fat_timedate)
|
||||
{
|
||||
const u32 fat_time = bits(fat_timedate, 0, 15);
|
||||
const u32 fat_date = bits(fat_timedate, 16, 31);
|
||||
|
||||
struct tm t; // struct tm format:
|
||||
t.tm_sec = bits(fat_time, 0,4) * 2; // [0,59]
|
||||
t.tm_min = bits(fat_time, 5,10); // [0,59]
|
||||
t.tm_hour = bits(fat_time, 11,15); // [0,23]
|
||||
t.tm_mday = bits(fat_date, 0,4); // [1,31]
|
||||
t.tm_mon = bits(fat_date, 5,8) - 1; // [0,11]
|
||||
t.tm_year = bits(fat_date, 9,15) + 80; // since 1900
|
||||
t.tm_isdst = -1; // unknown - let libc determine
|
||||
|
||||
// otherwise: totally bogus, and at the limit of 32-bit time_t
|
||||
debug_assert(t.tm_year < 138);
|
||||
|
||||
time_t ret = mktime(&t);
|
||||
debug_assert(ret != (time_t)-1); // mktime shouldn't fail
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static u32 FAT_from_time_t(time_t time)
|
||||
{
|
||||
// (values are adjusted for DST)
|
||||
struct tm* t = localtime(&time);
|
||||
|
||||
const u16 fat_time = u16(
|
||||
(t->tm_sec/2) | // 5
|
||||
(u16(t->tm_min) << 5) | // 6
|
||||
(u16(t->tm_hour) << 11) // 5
|
||||
);
|
||||
|
||||
const u16 fat_date = u16(
|
||||
(t->tm_mday) | // 5
|
||||
(u16(t->tm_mon+1) << 5) | // 4
|
||||
(u16(t->tm_year-80) << 9) // 7
|
||||
);
|
||||
|
||||
u32 fat_timedate = u32_from_u16(fat_date, fat_time);
|
||||
return fat_timedate;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
@ -288,7 +333,8 @@ public:
|
||||
|
||||
Stream stream(codec);
|
||||
stream.SetOutputBuffer(buf.get(), size);
|
||||
RETURN_ERR(io_Scan(m_file, m_ofs, m_csize, FeedStream, (uintptr_t)&stream));
|
||||
io::Operation op(*m_file.get(), 0, m_csize, m_ofs);
|
||||
RETURN_ERR(io::Run(op, io::Parameters(), std::bind(&Stream::Feed, &stream, std::placeholders::_1, std::placeholders::_2)));
|
||||
RETURN_ERR(stream.Finish());
|
||||
#if CODEC_COMPUTE_CHECKSUM
|
||||
debug_assert(m_checksum == stream.Checksum());
|
||||
@ -315,29 +361,32 @@ private:
|
||||
|
||||
struct LFH_Copier
|
||||
{
|
||||
u8* lfh_dst;
|
||||
size_t lfh_bytes_remaining;
|
||||
LFH_Copier(u8* lfh_dst, size_t lfh_bytes_remaining)
|
||||
: lfh_dst(lfh_dst), lfh_bytes_remaining(lfh_bytes_remaining)
|
||||
{
|
||||
}
|
||||
|
||||
// this code grabs an LFH struct from file block(s) that are
|
||||
// passed to the callback. usually, one call copies the whole thing,
|
||||
// but the LFH may straddle a block boundary.
|
||||
//
|
||||
// rationale: this allows using temp buffers for zip_fixup_lfh,
|
||||
// which avoids involving the file buffer manager and thus
|
||||
// avoids cluttering the trace and cache contents.
|
||||
LibError operator()(const u8* block, size_t size) const
|
||||
{
|
||||
debug_assert(size <= lfh_bytes_remaining);
|
||||
memcpy(lfh_dst, block, size);
|
||||
lfh_dst += size;
|
||||
lfh_bytes_remaining -= size;
|
||||
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
|
||||
mutable u8* lfh_dst;
|
||||
mutable size_t lfh_bytes_remaining;
|
||||
};
|
||||
|
||||
// this code grabs an LFH struct from file block(s) that are
|
||||
// passed to the callback. usually, one call copies the whole thing,
|
||||
// but the LFH may straddle a block boundary.
|
||||
//
|
||||
// rationale: this allows using temp buffers for zip_fixup_lfh,
|
||||
// which avoids involving the file buffer manager and thus
|
||||
// avoids cluttering the trace and cache contents.
|
||||
static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size)
|
||||
{
|
||||
LFH_Copier* p = (LFH_Copier*)cbData;
|
||||
|
||||
debug_assert(size <= p->lfh_bytes_remaining);
|
||||
memcpy(p->lfh_dst, block, size);
|
||||
p->lfh_dst += size;
|
||||
p->lfh_bytes_remaining -= size;
|
||||
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
|
||||
/**
|
||||
* fix up m_ofs (adjust it to point to cdata instead of the LFH).
|
||||
*
|
||||
@ -358,8 +407,8 @@ private:
|
||||
// only in the block cache if the file starts in the same block as a
|
||||
// previously read file (i.e. both are small).
|
||||
LFH lfh;
|
||||
LFH_Copier params = { (u8*)&lfh, sizeof(LFH) };
|
||||
if(io_Scan(m_file, m_ofs, sizeof(LFH), lfh_copier_cb, (uintptr_t)¶ms) == INFO::OK)
|
||||
io::Operation op(*m_file.get(), 0, sizeof(LFH), m_ofs);
|
||||
if(io::Run(op, io::Parameters(), LFH_Copier((u8*)&lfh, sizeof(LFH))) == INFO::OK)
|
||||
m_ofs += (off_t)lfh.Size();
|
||||
}
|
||||
|
||||
@ -382,7 +431,7 @@ class ArchiveReader_Zip : public IArchiveReader
|
||||
{
|
||||
public:
|
||||
ArchiveReader_Zip(const OsPath& pathname)
|
||||
: m_file(new File(pathname, 'r'))
|
||||
: m_file(new File(pathname, LIO_READ))
|
||||
{
|
||||
FileInfo fileInfo;
|
||||
GetFileInfo(pathname, &fileInfo);
|
||||
@ -398,16 +447,17 @@ public:
|
||||
size_t cd_numEntries = 0;
|
||||
size_t cd_size = 0;
|
||||
RETURN_ERR(LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size));
|
||||
shared_ptr<u8> buf = io_Allocate(cd_size, cd_ofs);
|
||||
u8* cd;
|
||||
RETURN_ERR(io_Read(m_file, cd_ofs, buf.get(), cd_size, cd));
|
||||
UniqueRange buf(io::Allocate(cd_size));
|
||||
|
||||
io::Operation op(*m_file.get(), buf.get(), cd_size, cd_ofs);
|
||||
RETURN_ERR(io::Run(op));
|
||||
|
||||
// iterate over Central Directory
|
||||
const u8* pos = cd;
|
||||
const u8* pos = (const u8*)buf.get();
|
||||
for(size_t i = 0; i < cd_numEntries; i++)
|
||||
{
|
||||
// scan for next CDFH
|
||||
CDFH* cdfh = (CDFH*)FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH));
|
||||
CDFH* cdfh = (CDFH*)FindRecord((const u8*)buf.get(), cd_size, pos, cdfh_magic, sizeof(CDFH));
|
||||
if(!cdfh)
|
||||
WARN_RETURN(ERR::CORRUPTED);
|
||||
|
||||
@ -467,11 +517,11 @@ private:
|
||||
|
||||
// read desired chunk of file into memory
|
||||
const off_t ofs = fileSize - off_t(scanSize);
|
||||
u8* data;
|
||||
RETURN_ERR(io_Read(file, ofs, buf, scanSize, data));
|
||||
io::Operation op(*file.get(), buf, scanSize, ofs);
|
||||
RETURN_ERR(io::Run(op));
|
||||
|
||||
// look for ECDR in buffer
|
||||
const ECDR* ecdr = (const ECDR*)FindRecord(data, scanSize, data, ecdr_magic, sizeof(ECDR));
|
||||
const ECDR* ecdr = (const ECDR*)FindRecord(buf, scanSize, buf, ecdr_magic, sizeof(ECDR));
|
||||
if(!ecdr)
|
||||
return INFO::CANNOT_HANDLE;
|
||||
|
||||
@ -482,19 +532,20 @@ private:
|
||||
static LibError LocateCentralDirectory(const PFile& file, off_t fileSize, off_t& cd_ofs, size_t& cd_numEntries, size_t& cd_size)
|
||||
{
|
||||
const size_t maxScanSize = 66000u; // see below
|
||||
shared_ptr<u8> buf = io_Allocate(maxScanSize, BLOCK_SIZE-1); // assume worst-case for alignment
|
||||
UniqueRange buf(io::Allocate(maxScanSize));
|
||||
|
||||
// expected case: ECDR at EOF; no file comment
|
||||
LibError ret = ScanForEcdr(file, fileSize, const_cast<u8*>(buf.get()), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size);
|
||||
LibError ret = ScanForEcdr(file, fileSize, (u8*)buf.get(), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size);
|
||||
if(ret == INFO::OK)
|
||||
return INFO::OK;
|
||||
// worst case: ECDR precedes 64 KiB of file comment
|
||||
ret = ScanForEcdr(file, fileSize, const_cast<u8*>(buf.get()), maxScanSize, cd_numEntries, cd_ofs, cd_size);
|
||||
ret = ScanForEcdr(file, fileSize, (u8*)buf.get(), maxScanSize, cd_numEntries, cd_ofs, cd_size);
|
||||
if(ret == INFO::OK)
|
||||
return INFO::OK;
|
||||
|
||||
// both ECDR scans failed - this is not a valid Zip file.
|
||||
RETURN_ERR(io_ReadAligned(file, 0, const_cast<u8*>(buf.get()), sizeof(LFH)));
|
||||
io::Operation op(*file.get(), buf.get(), sizeof(LFH));
|
||||
RETURN_ERR(io::Run(op));
|
||||
// the Zip file has an LFH but lacks an ECDR. this can happen if
|
||||
// the user hard-exits while an archive is being written.
|
||||
// notes:
|
||||
@ -503,7 +554,7 @@ private:
|
||||
// because it'd be slow.
|
||||
// - do not warn - the corrupt archive will be deleted on next
|
||||
// successful archive builder run anyway.
|
||||
if(FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH)))
|
||||
if(FindRecord((const u8*)buf.get(), sizeof(LFH), (const u8*)buf.get(), lfh_magic, sizeof(LFH)))
|
||||
return ERR::CORRUPTED; // NOWARN
|
||||
// totally bogus
|
||||
else
|
||||
@ -528,8 +579,7 @@ class ArchiveWriter_Zip : public IArchiveWriter
|
||||
{
|
||||
public:
|
||||
ArchiveWriter_Zip(const OsPath& archivePathname, bool noDeflate)
|
||||
: m_file(new File(archivePathname, 'w')), m_fileSize(0)
|
||||
, m_unalignedWriter(new UnalignedWriter(m_file, 0))
|
||||
: m_file(new File(archivePathname, LIO_WRITE)), m_fileSize(0)
|
||||
, m_numEntries(0), m_noDeflate(noDeflate)
|
||||
{
|
||||
THROW_ERR(pool_create(&m_cdfhPool, 10*MiB, 0));
|
||||
@ -546,19 +596,9 @@ public:
|
||||
const off_t cd_ofs = m_fileSize;
|
||||
ecdr->Init(m_numEntries, cd_ofs, cd_size);
|
||||
|
||||
m_unalignedWriter->Append(m_cdfhPool.da.base, cd_size+sizeof(ECDR));
|
||||
m_unalignedWriter->Flush();
|
||||
m_unalignedWriter.reset();
|
||||
write(m_file->Descriptor(), m_cdfhPool.da.base, cd_size+sizeof(ECDR));
|
||||
|
||||
(void)pool_destroy(&m_cdfhPool);
|
||||
|
||||
const OsPath pathname = m_file->Pathname(); // (must be retrieved before resetting m_file)
|
||||
m_file.reset();
|
||||
|
||||
m_fileSize += off_t(cd_size+sizeof(ECDR));
|
||||
|
||||
// remove padding added by UnalignedWriter
|
||||
wtruncate(pathname, m_fileSize);
|
||||
}
|
||||
|
||||
LibError AddFile(const OsPath& pathname, const OsPath& pathnameInArchive)
|
||||
@ -578,7 +618,7 @@ public:
|
||||
return INFO::SKIPPED;
|
||||
|
||||
PFile file(new File);
|
||||
RETURN_ERR(file->Open(pathname, 'r'));
|
||||
RETURN_ERR(file->Open(pathname, LIO_READ));
|
||||
|
||||
const size_t pathnameLength = pathnameInArchive.string().length();
|
||||
|
||||
@ -598,7 +638,7 @@ public:
|
||||
|
||||
// allocate memory
|
||||
const size_t csizeMax = codec->MaxOutputSize(size_t(usize));
|
||||
shared_ptr<u8> buf = io_Allocate(sizeof(LFH) + pathnameLength + csizeMax);
|
||||
UniqueRange buf(io::Allocate(sizeof(LFH) + pathnameLength + csizeMax));
|
||||
|
||||
// read and compress file contents
|
||||
size_t csize; u32 checksum;
|
||||
@ -606,7 +646,8 @@ public:
|
||||
u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength;
|
||||
Stream stream(codec);
|
||||
stream.SetOutputBuffer(cdata, csizeMax);
|
||||
RETURN_ERR(io_Scan(file, 0, usize, FeedStream, (uintptr_t)&stream));
|
||||
io::Operation op(*file.get(), 0, usize);
|
||||
RETURN_ERR(io::Run(op, io::Parameters(), std::bind(&Stream::Feed, &stream, std::placeholders::_1, std::placeholders::_2)));
|
||||
RETURN_ERR(stream.Finish());
|
||||
csize = stream.OutSize();
|
||||
checksum = stream.Checksum();
|
||||
@ -631,7 +672,8 @@ public:
|
||||
|
||||
// write LFH, pathname and cdata to file
|
||||
const size_t packageSize = sizeof(LFH) + pathnameLength + csize;
|
||||
RETURN_ERR(m_unalignedWriter->Append(buf.get(), packageSize));
|
||||
if(write(m_file->Descriptor(), buf.get(), packageSize) < 0)
|
||||
WARN_RETURN(ERR::IO);
|
||||
m_fileSize += (off_t)packageSize;
|
||||
|
||||
return INFO::OK;
|
||||
@ -661,7 +703,6 @@ private:
|
||||
|
||||
PFile m_file;
|
||||
off_t m_fileSize;
|
||||
PUnalignedWriter m_unalignedWriter;
|
||||
|
||||
Pool m_cdfhPool;
|
||||
size_t m_numEntries;
|
||||
|
@ -136,12 +136,3 @@ LibError Stream::Finish()
|
||||
m_outProduced += outProduced;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize)
|
||||
{
|
||||
// TIMER_ACCRUE(tc_stream);
|
||||
|
||||
Stream& stream = *(Stream*)cbData;
|
||||
return stream.Feed(in, inSize);
|
||||
}
|
||||
|
@ -110,6 +110,4 @@ private:
|
||||
u32 m_checksum;
|
||||
};
|
||||
|
||||
extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize);
|
||||
|
||||
#endif // #ifndef INCLUDED_STREAM
|
||||
|
@ -189,23 +189,13 @@ ScopedIoMonitor::~ScopedIoMonitor()
|
||||
timer_reset(&m_startTime);
|
||||
}
|
||||
|
||||
void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, wchar_t mode, off_t size)
|
||||
void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, int opcode, off_t size)
|
||||
{
|
||||
debug_assert(fi < FI_MAX_IDX);
|
||||
debug_assert(mode == 'r' || mode == 'w');
|
||||
const FileOp op = (mode == 'r')? FO_READ : FO_WRITE;
|
||||
debug_assert(opcode == LIO_READ || opcode == LIO_WRITE);
|
||||
|
||||
io_actual_size_total[fi][op] += size;
|
||||
io_elapsed_time[fi][op] += timer_reset(&m_startTime);
|
||||
}
|
||||
|
||||
void stats_io_check_seek(BlockId& blockId)
|
||||
{
|
||||
static BlockId lastBlockId;
|
||||
|
||||
if(blockId != lastBlockId)
|
||||
io_seeks++;
|
||||
lastBlockId = blockId;
|
||||
io_actual_size_total[fi][opcode == LIO_WRITE] += size;
|
||||
io_elapsed_time[fi][opcode == LIO_WRITE] += timer_reset(&m_startTime);
|
||||
}
|
||||
|
||||
|
||||
@ -320,11 +310,11 @@ void file_stats_dump()
|
||||
L"Average size = %g KB; seeks: %lu; total callback time: %g ms\n"
|
||||
L"Total data actually read from disk = %g MB\n",
|
||||
(unsigned long)user_ios, user_io_size_total/MB,
|
||||
#define THROUGHPUT(impl, op) (io_elapsed_time[impl][op] == 0.0)? 0.0 : (io_actual_size_total[impl][op] / io_elapsed_time[impl][op] / MB)
|
||||
THROUGHPUT(FI_LOWIO, FO_READ), THROUGHPUT(FI_LOWIO, FO_WRITE),
|
||||
THROUGHPUT(FI_AIO , FO_READ), THROUGHPUT(FI_AIO , FO_WRITE),
|
||||
#define THROUGHPUT(impl, opcode) (io_elapsed_time[impl][opcode == LIO_WRITE] == 0.0)? 0.0 : (io_actual_size_total[impl][opcode == LIO_WRITE] / io_elapsed_time[impl][opcode == LIO_WRITE] / MB)
|
||||
THROUGHPUT(FI_LOWIO, LIO_READ), THROUGHPUT(FI_LOWIO, LIO_WRITE),
|
||||
THROUGHPUT(FI_AIO , LIO_READ), THROUGHPUT(FI_AIO , LIO_WRITE),
|
||||
user_io_size_total/user_ios/KB, (unsigned long)io_seeks, io_process_time_total/ms,
|
||||
(io_actual_size_total[FI_LOWIO][FO_READ]+io_actual_size_total[FI_AIO][FO_READ])/MB
|
||||
(io_actual_size_total[FI_LOWIO][0]+io_actual_size_total[FI_AIO][0])/MB
|
||||
);
|
||||
|
||||
debug_printf(
|
||||
|
@ -27,15 +27,14 @@
|
||||
#ifndef INCLUDED_FILE_STATS
|
||||
#define INCLUDED_FILE_STATS
|
||||
|
||||
#include "lib/posix/posix_aio.h" // LIO_READ, LIO_WRITE
|
||||
|
||||
#define FILE_STATS_ENABLED 0
|
||||
|
||||
|
||||
enum FileIOImplentation { FI_LOWIO, FI_AIO, FI_BCACHE, FI_MAX_IDX };
|
||||
enum FileOp { FO_READ, FO_WRITE };
|
||||
enum CacheRet { CR_HIT, CR_MISS };
|
||||
|
||||
#include "lib/file/io/block_cache.h" // BlockId
|
||||
|
||||
#if FILE_STATS_ENABLED
|
||||
|
||||
// vfs
|
||||
@ -69,13 +68,12 @@ class ScopedIoMonitor
|
||||
public:
|
||||
ScopedIoMonitor();
|
||||
~ScopedIoMonitor();
|
||||
void NotifyOfSuccess(FileIOImplentation fi, wchar_t mode, off_t size);
|
||||
void NotifyOfSuccess(FileIOImplentation fi, int opcode, off_t size);
|
||||
|
||||
private:
|
||||
double m_startTime;
|
||||
};
|
||||
|
||||
extern void stats_io_check_seek(BlockId& blockId);
|
||||
extern void stats_cb_start();
|
||||
extern void stats_cb_finish();
|
||||
|
||||
@ -106,9 +104,8 @@ class ScopedIoMonitor
|
||||
public:
|
||||
ScopedIoMonitor() {}
|
||||
~ScopedIoMonitor() {}
|
||||
void NotifyOfSuccess(FileIOImplentation UNUSED(fi), wchar_t UNUSED(mode), off_t UNUSED(size)) {}
|
||||
void NotifyOfSuccess(FileIOImplentation UNUSED(fi), int UNUSED(opcode), off_t UNUSED(size)) {}
|
||||
};
|
||||
#define stats_io_check_seek(blockId)
|
||||
#define stats_cb_start()
|
||||
#define stats_cb_finish()
|
||||
#define stats_cache(cr, size)
|
||||
|
@ -48,33 +48,13 @@ RealDirectory::RealDirectory(const OsPath& path, size_t priority, size_t flags)
|
||||
|
||||
/*virtual*/ LibError RealDirectory::Load(const OsPath& name, const shared_ptr<u8>& buf, size_t size) const
|
||||
{
|
||||
const OsPath pathname = m_path / name;
|
||||
|
||||
PFile file(new File);
|
||||
RETURN_ERR(file->Open(pathname, 'r'));
|
||||
|
||||
RETURN_ERR(io_ReadAligned(file, 0, buf.get(), size));
|
||||
return INFO::OK;
|
||||
return io::Load(m_path / name, buf.get(), size);
|
||||
}
|
||||
|
||||
|
||||
LibError RealDirectory::Store(const OsPath& name, const shared_ptr<u8>& fileContents, size_t size)
|
||||
{
|
||||
const OsPath pathname = m_path / name;
|
||||
|
||||
{
|
||||
PFile file(new File);
|
||||
RETURN_ERR(file->Open(pathname, 'w'));
|
||||
RETURN_ERR(io_WriteAligned(file, 0, fileContents.get(), size));
|
||||
}
|
||||
|
||||
// io_WriteAligned pads the file; we need to truncate it to the actual
|
||||
// length. ftruncate can't be used because Windows' FILE_FLAG_NO_BUFFERING
|
||||
// only allows resizing to sector boundaries, so the file must first
|
||||
// be closed.
|
||||
wtruncate(pathname, size);
|
||||
|
||||
return INFO::OK;
|
||||
return io::Store(m_path / name, fileContents.get(), size);
|
||||
}
|
||||
|
||||
|
||||
|
@ -31,7 +31,6 @@
|
||||
#include <sstream>
|
||||
|
||||
#include "lib/allocators/pool.h"
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "lib/timer.h" // timer_Time
|
||||
#include "lib/sysdep/sysdep.h" // sys_OpenFile
|
||||
|
||||
|
@ -27,34 +27,27 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/file/file.h"
|
||||
|
||||
#include "lib/config2.h"
|
||||
#include "lib/sysdep/filesystem.h" // O_*, S_*
|
||||
#include "lib/posix/posix_aio.h"
|
||||
#include "lib/file/common/file_stats.h"
|
||||
|
||||
|
||||
ERROR_ASSOCIATE(ERR::FILE_ACCESS, L"Insufficient access rights to open file", EACCES);
|
||||
ERROR_ASSOCIATE(ERR::IO, L"Error during IO", EIO);
|
||||
|
||||
|
||||
namespace FileImpl {
|
||||
|
||||
LibError Open(const OsPath& pathname, wchar_t accessType, int& fd)
|
||||
LibError FileOpen(const OsPath& pathname, int opcode, int& fd)
|
||||
{
|
||||
int oflag = 0;
|
||||
switch(accessType)
|
||||
switch(opcode)
|
||||
{
|
||||
case 'r':
|
||||
case LIO_READ:
|
||||
oflag = O_RDONLY;
|
||||
break;
|
||||
case 'w':
|
||||
case LIO_WRITE:
|
||||
oflag = O_WRONLY|O_CREAT|O_TRUNC;
|
||||
break;
|
||||
case '+':
|
||||
oflag = O_RDWR;
|
||||
break;
|
||||
default:
|
||||
debug_assert(0);
|
||||
break;
|
||||
}
|
||||
#if OS_WIN
|
||||
oflag |= O_BINARY_NP;
|
||||
@ -71,7 +64,7 @@ LibError Open(const OsPath& pathname, wchar_t accessType, int& fd)
|
||||
}
|
||||
|
||||
|
||||
void Close(int& fd)
|
||||
void FileClose(int& fd)
|
||||
{
|
||||
if(fd >= 0)
|
||||
{
|
||||
@ -79,88 +72,3 @@ void Close(int& fd)
|
||||
fd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
LibError IO(int fd, wchar_t accessType, off_t ofs, u8* buf, size_t size)
|
||||
{
|
||||
debug_assert(accessType == 'r' || accessType == 'w');
|
||||
|
||||
ScopedIoMonitor monitor;
|
||||
|
||||
lseek(fd, ofs, SEEK_SET);
|
||||
|
||||
errno = 0;
|
||||
const ssize_t ret = (accessType == 'w')? write(fd, buf, size) : read(fd, buf, size);
|
||||
if(ret < 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
const size_t totalTransferred = (size_t)ret;
|
||||
#if CONFIG2_FILE_ENABLE_AIO
|
||||
// we won't be called from Issue, i.e. size is always the exact
|
||||
// value without padding and can be checked.
|
||||
if(totalTransferred != size)
|
||||
WARN_RETURN(ERR::IO);
|
||||
#endif
|
||||
|
||||
monitor.NotifyOfSuccess(FI_LOWIO, accessType, totalTransferred);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError Issue(aiocb& req, int fd, wchar_t accessType, off_t alignedOfs, u8* alignedBuf, size_t alignedSize)
|
||||
{
|
||||
memset(&req, 0, sizeof(req));
|
||||
req.aio_lio_opcode = (accessType == 'w')? LIO_WRITE : LIO_READ;
|
||||
req.aio_buf = (volatile void*)alignedBuf;
|
||||
req.aio_fildes = fd;
|
||||
req.aio_offset = alignedOfs;
|
||||
req.aio_nbytes = alignedSize;
|
||||
#if CONFIG2_FILE_ENABLE_AIO
|
||||
struct sigevent* sig = 0; // no notification signal
|
||||
aiocb* const reqs = &req;
|
||||
if(lio_listio(LIO_NOWAIT, &reqs, 1, sig) != 0)
|
||||
return LibError_from_errno();
|
||||
return INFO::OK;
|
||||
#else
|
||||
// quick and dirty workaround (see CONFIG2_FILE_ENABLE_AIO)
|
||||
return IO(fd, accessType, alignedOfs, alignedBuf, alignedSize);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize)
|
||||
{
|
||||
#if CONFIG2_FILE_ENABLE_AIO
|
||||
const int err = aio_error(&req);
|
||||
if(err == EINPROGRESS)
|
||||
{
|
||||
SUSPEND_AGAIN:
|
||||
aiocb* const reqs = &req;
|
||||
errno = 0;
|
||||
const int ret = aio_suspend(&reqs, 1, (timespec*)0); // no timeout
|
||||
if(ret != 0)
|
||||
{
|
||||
if(errno == EINTR) // interrupted by signal
|
||||
goto SUSPEND_AGAIN;
|
||||
return LibError_from_errno();
|
||||
}
|
||||
}
|
||||
else if(err != 0)
|
||||
{
|
||||
errno = err;
|
||||
return LibError_from_errno();
|
||||
}
|
||||
|
||||
const ssize_t bytesTransferred = aio_return(&req);
|
||||
if(bytesTransferred == -1) // transfer failed
|
||||
WARN_RETURN(ERR::IO);
|
||||
|
||||
alignedSize = bytesTransferred;
|
||||
#else
|
||||
alignedSize = req.aio_nbytes;
|
||||
#endif
|
||||
alignedBuf = (u8*)req.aio_buf; // cast from volatile void*
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
} // namespace FileImpl
|
||||
|
@ -27,50 +27,28 @@
|
||||
#ifndef INCLUDED_FILE
|
||||
#define INCLUDED_FILE
|
||||
|
||||
struct aiocb;
|
||||
|
||||
#include "lib/os_path.h"
|
||||
#include "lib/posix/posix_aio.h" // opcode: LIO_READ or LIO_WRITE
|
||||
|
||||
namespace ERR
|
||||
{
|
||||
const LibError FILE_ACCESS = -110300;
|
||||
const LibError IO = -110301;
|
||||
}
|
||||
|
||||
namespace FileImpl
|
||||
{
|
||||
LIB_API LibError Open(const OsPath& pathname, wchar_t mode, int& fd);
|
||||
LIB_API void Close(int& fd);
|
||||
LIB_API LibError IO(int fd, wchar_t mode, off_t ofs, u8* buf, size_t size);
|
||||
LIB_API LibError Issue(aiocb& req, int fd, wchar_t mode, off_t alignedOfs, u8* alignedBuf, size_t alignedSize);
|
||||
LIB_API LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize);
|
||||
}
|
||||
|
||||
LIB_API LibError FileOpen(const OsPath& pathname, int opcode, int& fd);
|
||||
LIB_API void FileClose(int& fd);
|
||||
|
||||
class File
|
||||
{
|
||||
public:
|
||||
File()
|
||||
: m_pathname(), m_fd(0)
|
||||
: pathname(), fd(-1)
|
||||
{
|
||||
}
|
||||
|
||||
LibError Open(const OsPath& pathname, wchar_t mode)
|
||||
File(const OsPath& pathname, int opcode)
|
||||
{
|
||||
RETURN_ERR(FileImpl::Open(pathname, mode, m_fd));
|
||||
m_pathname = pathname;
|
||||
m_mode = mode;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
void Close()
|
||||
{
|
||||
FileImpl::Close(m_fd);
|
||||
}
|
||||
|
||||
File(const OsPath& pathname, wchar_t mode)
|
||||
{
|
||||
(void)Open(pathname, mode);
|
||||
(void)Open(pathname, opcode);
|
||||
}
|
||||
|
||||
~File()
|
||||
@ -78,35 +56,38 @@ public:
|
||||
Close();
|
||||
}
|
||||
|
||||
LibError Open(const OsPath& pathname, int opcode)
|
||||
{
|
||||
RETURN_ERR(FileOpen(pathname, opcode, fd));
|
||||
this->pathname = pathname;
|
||||
this->opcode = opcode;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
void Close()
|
||||
{
|
||||
FileClose(fd);
|
||||
}
|
||||
|
||||
const OsPath& Pathname() const
|
||||
{
|
||||
return m_pathname;
|
||||
return pathname;
|
||||
}
|
||||
|
||||
wchar_t Mode() const
|
||||
int Descriptor() const
|
||||
{
|
||||
return m_mode;
|
||||
return fd;
|
||||
}
|
||||
|
||||
LibError Issue(aiocb& req, wchar_t mode, off_t alignedOfs, u8* alignedBuf, size_t alignedSize) const
|
||||
int Opcode() const
|
||||
{
|
||||
return FileImpl::Issue(req, m_fd, mode, alignedOfs, alignedBuf, alignedSize);
|
||||
}
|
||||
|
||||
LibError Write(off_t ofs, const u8* buf, size_t size)
|
||||
{
|
||||
return FileImpl::IO(m_fd, 'w', ofs, const_cast<u8*>(buf), size);
|
||||
}
|
||||
|
||||
LibError Read(off_t ofs, u8* buf, size_t size) const
|
||||
{
|
||||
return FileImpl::IO(m_fd, 'r', ofs, buf, size);
|
||||
return opcode;
|
||||
}
|
||||
|
||||
private:
|
||||
OsPath m_pathname;
|
||||
int m_fd;
|
||||
wchar_t m_mode;
|
||||
OsPath pathname;
|
||||
int fd;
|
||||
int opcode;
|
||||
};
|
||||
|
||||
typedef shared_ptr<File> PFile;
|
||||
|
@ -1,163 +0,0 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* cache for aligned I/O blocks.
|
||||
*/
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "lib/file/io/block_cache.h"
|
||||
|
||||
#include "lib/config2.h" // CONFIG2_CACHE_READ_ONLY
|
||||
#include "lib/posix/posix_mman.h" // mprotect
|
||||
#include "lib/file/common/file_stats.h"
|
||||
#include "lib/lockfree.h"
|
||||
#include "lib/allocators/pool.h"
|
||||
#include "lib/fnv_hash.h"
|
||||
#include "lib/file/io/io_align.h"
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
BlockId::BlockId()
|
||||
: m_id(0)
|
||||
{
|
||||
}
|
||||
|
||||
BlockId::BlockId(const OsPath& pathname, off_t ofs)
|
||||
{
|
||||
const Path::String& string = pathname.string();
|
||||
m_id = fnv_hash64(string.c_str(), string.length()*sizeof(string[0]));
|
||||
const size_t indexBits = 16;
|
||||
m_id <<= indexBits;
|
||||
const off_t blockIndex = off_t(ofs / BLOCK_SIZE);
|
||||
debug_assert(blockIndex < off_t(1) << indexBits);
|
||||
m_id |= blockIndex;
|
||||
}
|
||||
|
||||
bool BlockId::operator==(const BlockId& rhs) const
|
||||
{
|
||||
return m_id == rhs.m_id;
|
||||
}
|
||||
|
||||
bool BlockId::operator!=(const BlockId& rhs) const
|
||||
{
|
||||
return !operator==(rhs);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
struct Block
|
||||
{
|
||||
Block(BlockId id, const shared_ptr<u8>& buf)
|
||||
{
|
||||
this->id = id;
|
||||
this->buf = buf;
|
||||
}
|
||||
|
||||
// block is "valid" and can satisfy Retrieve() requests if a
|
||||
// (non-default-constructed) ID has been assigned.
|
||||
BlockId id;
|
||||
|
||||
// this block is "in use" if use_count != 1.
|
||||
shared_ptr<u8> buf;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class BlockCache::Impl
|
||||
{
|
||||
public:
|
||||
Impl(size_t numBlocks)
|
||||
: m_maxBlocks(numBlocks)
|
||||
{
|
||||
}
|
||||
|
||||
void Add(BlockId id, const shared_ptr<u8>& buf)
|
||||
{
|
||||
if(m_blocks.size() > m_maxBlocks)
|
||||
{
|
||||
#if CONFIG2_CACHE_READ_ONLY
|
||||
mprotect((void*)m_blocks.front().buf.get(), BLOCK_SIZE, PROT_READ);
|
||||
#endif
|
||||
m_blocks.pop_front(); // evict oldest block
|
||||
}
|
||||
|
||||
#if CONFIG2_CACHE_READ_ONLY
|
||||
mprotect((void*)buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ);
|
||||
#endif
|
||||
m_blocks.push_back(Block(id, buf));
|
||||
}
|
||||
|
||||
bool Retrieve(BlockId id, shared_ptr<u8>& buf)
|
||||
{
|
||||
// (linear search is ok since we only expect to manage a few blocks)
|
||||
for(size_t i = 0; i < m_blocks.size(); i++)
|
||||
{
|
||||
Block& block = m_blocks[i];
|
||||
if(block.id == id)
|
||||
{
|
||||
buf = block.buf;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void InvalidateAll()
|
||||
{
|
||||
// note: don't check whether any references are held etc. because
|
||||
// this should only be called at the end of the (test) program.
|
||||
m_blocks.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
size_t m_maxBlocks;
|
||||
typedef std::deque<Block> Blocks;
|
||||
Blocks m_blocks;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
BlockCache::BlockCache(size_t numBlocks)
|
||||
: impl(new Impl(numBlocks))
|
||||
{
|
||||
}
|
||||
|
||||
void BlockCache::Add(BlockId id, const shared_ptr<u8>& buf)
|
||||
{
|
||||
impl->Add(id, buf);
|
||||
}
|
||||
|
||||
bool BlockCache::Retrieve(BlockId id, shared_ptr<u8>& buf)
|
||||
{
|
||||
return impl->Retrieve(id, buf);
|
||||
}
|
||||
|
||||
void BlockCache::InvalidateAll()
|
||||
{
|
||||
return impl->InvalidateAll();
|
||||
}
|
@ -1,106 +0,0 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* cache for aligned I/O blocks.
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_BLOCK_CACHE
|
||||
#define INCLUDED_BLOCK_CACHE
|
||||
|
||||
#include "lib/os_path.h"
|
||||
|
||||
/**
|
||||
* ID that uniquely identifies a block within a file
|
||||
**/
|
||||
class BlockId
|
||||
{
|
||||
public:
|
||||
BlockId();
|
||||
BlockId(const OsPath& pathname, off_t ofs);
|
||||
bool operator==(const BlockId& rhs) const;
|
||||
bool operator!=(const BlockId& rhs) const;
|
||||
|
||||
private:
|
||||
u64 m_id;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* cache of (aligned) file blocks with support for zero-copy IO.
|
||||
* absorbs the overhead of rounding up archive IOs to the nearest block
|
||||
* boundaries by keeping the last few blocks in memory.
|
||||
*
|
||||
* the interface is somewhat similar to FileCache; see the note there.
|
||||
*
|
||||
* not thread-safe (each thread is intended to have its own cache).
|
||||
**/
|
||||
class BlockCache
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* @param numBlocks (the default value is enough to support temp buffers
|
||||
* and absorb the cost of unaligned reads from archives.)
|
||||
**/
|
||||
BlockCache(size_t numBlocks = 16);
|
||||
|
||||
/**
|
||||
* Add a block to the cache.
|
||||
*
|
||||
* @param id Key that will be used to Retrieve the block.
|
||||
* @param buf
|
||||
*
|
||||
* Call this when the block's IO has completed; its data will
|
||||
* satisfy subsequent Retrieve calls for the same id.
|
||||
* If CONFIG2_CACHE_READ_ONLY, the memory is made read-only.
|
||||
**/
|
||||
void Add(BlockId id, const shared_ptr<u8>& buf);
|
||||
|
||||
/**
|
||||
* Attempt to retrieve a block's contents.
|
||||
*
|
||||
* @return whether the block is in cache.
|
||||
*
|
||||
* if successful, a shared pointer to the contents is returned.
|
||||
* they remain valid until all references are removed and the block
|
||||
* is evicted.
|
||||
**/
|
||||
bool Retrieve(BlockId id, shared_ptr<u8>& buf);
|
||||
|
||||
/**
|
||||
* Invalidate the contents of the cache.
|
||||
*
|
||||
* this effectively discards the contents of existing blocks
|
||||
* (more specifically: prevents them from satisfying Retrieve calls
|
||||
* until a subsequent Add with the same id).
|
||||
*
|
||||
* useful for self-tests: multiple independent IO tests run in the same
|
||||
* process and must not influence each other via the cache.
|
||||
**/
|
||||
void InvalidateAll();
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
shared_ptr<Impl> impl;
|
||||
};
|
||||
|
||||
#endif // #ifndef INCLUDED_BLOCK_CACHE
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
/* Copyright (c) 2011 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
@ -23,19 +23,14 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/file/io/io.h"
|
||||
|
||||
#include "lib/posix/posix_aio.h"
|
||||
#include "lib/allocators/allocators.h" // AllocatorChecker
|
||||
#include "lib/file/file.h"
|
||||
#include "lib/file/common/file_stats.h"
|
||||
#include "lib/file/io/block_cache.h"
|
||||
#include "lib/file/io/io_align.h"
|
||||
#include "lib/sysdep/rtl.h"
|
||||
|
||||
static const size_t ioDepth = 16;
|
||||
ERROR_ASSOCIATE(ERR::IO, L"Error during IO", EIO);
|
||||
|
||||
namespace io {
|
||||
|
||||
// the underlying aio implementation likes buffer and offset to be
|
||||
// sector-aligned; if not, the transfer goes through an align buffer,
|
||||
// and requires an extra memcpy.
|
||||
// the Windows aio implementation requires buffer and offset to be
|
||||
// sector-aligned.
|
||||
//
|
||||
// if the user specifies an unaligned buffer, there's not much we can
|
||||
// do - we can't assume the buffer contains padding. therefore,
|
||||
@ -51,14 +46,14 @@ static const size_t ioDepth = 16;
|
||||
// AIO wrapper. rationale:
|
||||
// - aligning the transfer isn't possible here since we have no control
|
||||
// over the buffer, i.e. we cannot read more data than requested.
|
||||
// instead, this is done in io_manager.
|
||||
// instead, this is done in manager.
|
||||
// - transfer sizes here are arbitrary (i.e. not block-aligned);
|
||||
// that means the cache would have to handle this or also split them up
|
||||
// into blocks, which would duplicate the abovementioned work.
|
||||
// into blocks, which would duplicate the above mentioned work.
|
||||
// - if caching here, we'd also have to handle "forwarding" (i.e.
|
||||
// desired block has been issued but isn't yet complete). again, it
|
||||
// is easier to let the synchronous io_manager handle this.
|
||||
// - finally, io_manager knows more about whether the block should be cached
|
||||
// is easier to let the synchronous manager handle this.
|
||||
// - finally, manager knows more about whether the block should be cached
|
||||
// (e.g. whether another block request will follow), but we don't
|
||||
// currently make use of this.
|
||||
//
|
||||
@ -71,271 +66,73 @@ static const size_t ioDepth = 16;
|
||||
// this is a bit more complicated than just using the cache as storage.
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// allocator
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
#ifndef NDEBUG
|
||||
static AllocatorChecker allocatorChecker;
|
||||
#endif
|
||||
|
||||
class IoDeleter
|
||||
UniqueRange Allocate(size_t size, size_t alignment)
|
||||
{
|
||||
public:
|
||||
IoDeleter(size_t paddedSize)
|
||||
: m_paddedSize(paddedSize)
|
||||
{
|
||||
}
|
||||
debug_assert(is_pow2(alignment));
|
||||
if(alignment <= idxDeleterBits)
|
||||
alignment = idxDeleterBits+1;
|
||||
|
||||
void operator()(u8* mem)
|
||||
{
|
||||
debug_assert(m_paddedSize != 0);
|
||||
#ifndef NDEBUG
|
||||
allocatorChecker.OnDeallocate(mem, m_paddedSize);
|
||||
#endif
|
||||
page_aligned_free(mem, m_paddedSize);
|
||||
m_paddedSize = 0;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t m_paddedSize;
|
||||
};
|
||||
|
||||
|
||||
shared_ptr<u8> io_Allocate(size_t size, off_t ofs)
|
||||
{
|
||||
debug_assert(size != 0);
|
||||
|
||||
const size_t paddedSize = (size_t)PaddedSize(size, ofs);
|
||||
u8* mem = (u8*)page_aligned_alloc(paddedSize);
|
||||
if(!mem)
|
||||
throw std::bad_alloc();
|
||||
|
||||
#ifndef NDEBUG
|
||||
allocatorChecker.OnAllocate(mem, paddedSize);
|
||||
#endif
|
||||
|
||||
return shared_ptr<u8>(mem, IoDeleter(paddedSize));
|
||||
const size_t alignedSize = round_up(size, alignment);
|
||||
const UniqueRange::pointer p = rtl_AllocateAligned(alignedSize, alignment);
|
||||
return UniqueRange(p, size, idxDeleterAligned);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// BlockIo
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class BlockIo
|
||||
LibError Issue(aiocb& cb, size_t queueDepth)
|
||||
{
|
||||
public:
|
||||
LibError Issue(const PFile& file, off_t alignedOfs, u8* alignedBuf)
|
||||
#if CONFIG2_FILE_ENABLE_AIO
|
||||
if(queueDepth > 1)
|
||||
{
|
||||
m_blockId = BlockId(file->Pathname(), alignedOfs);
|
||||
if(file->Mode() == 'r')
|
||||
{
|
||||
if(s_blockCache.Retrieve(m_blockId, m_cachedBlock))
|
||||
{
|
||||
stats_block_cache(CR_HIT);
|
||||
const int ret = (cb.aio_lio_opcode == LIO_WRITE)? aio_write(&cb): aio_read(&cb);
|
||||
RETURN_ERR(LibError_from_posix(ret));
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
debug_assert(lseek(cb.aio_fildes, cb.aio_offset, SEEK_SET) == cb.aio_offset);
|
||||
|
||||
// copy from cache into user buffer
|
||||
if(alignedBuf)
|
||||
{
|
||||
memcpy(alignedBuf, m_cachedBlock.get(), BLOCK_SIZE);
|
||||
m_alignedBuf = alignedBuf;
|
||||
}
|
||||
// return cached block
|
||||
else
|
||||
{
|
||||
m_alignedBuf = const_cast<u8*>(m_cachedBlock.get());
|
||||
}
|
||||
void* buf = (void*)cb.aio_buf; // cast from volatile void*
|
||||
const ssize_t bytesTransferred = (cb.aio_lio_opcode == LIO_WRITE)? write(cb.aio_fildes, buf, cb.aio_nbytes) : read(cb.aio_fildes, buf, cb.aio_nbytes);
|
||||
if(bytesTransferred < 0)
|
||||
return LibError_from_errno();
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
else
|
||||
{
|
||||
stats_block_cache(CR_MISS);
|
||||
// fall through to the actual issue..
|
||||
}
|
||||
}
|
||||
|
||||
stats_io_check_seek(m_blockId);
|
||||
|
||||
// transfer directly to/from user buffer
|
||||
if(alignedBuf)
|
||||
{
|
||||
m_alignedBuf = alignedBuf;
|
||||
}
|
||||
// transfer into newly allocated temporary block
|
||||
else
|
||||
{
|
||||
m_tempBlock = io_Allocate(BLOCK_SIZE);
|
||||
m_alignedBuf = const_cast<u8*>(m_tempBlock.get());
|
||||
}
|
||||
|
||||
return file->Issue(m_req, file->Mode(), alignedOfs, m_alignedBuf, BLOCK_SIZE);
|
||||
cb.aio_nbytes = (size_t)bytesTransferred;
|
||||
}
|
||||
|
||||
LibError WaitUntilComplete(const u8*& block, size_t& blockSize)
|
||||
{
|
||||
if(m_cachedBlock)
|
||||
{
|
||||
block = m_alignedBuf;
|
||||
blockSize = BLOCK_SIZE;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
RETURN_ERR(FileImpl::WaitUntilComplete(m_req, const_cast<u8*&>(block), blockSize));
|
||||
|
||||
if(m_tempBlock)
|
||||
s_blockCache.Add(m_blockId, m_tempBlock);
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
static BlockCache s_blockCache;
|
||||
|
||||
BlockId m_blockId;
|
||||
|
||||
// the address that WaitUntilComplete will return
|
||||
// (cached or temporary block, or user buffer)
|
||||
u8* m_alignedBuf;
|
||||
|
||||
shared_ptr<u8> m_cachedBlock;
|
||||
shared_ptr<u8> m_tempBlock;
|
||||
|
||||
aiocb m_req;
|
||||
};
|
||||
|
||||
BlockCache BlockIo::s_blockCache;
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// IoSplitter
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class IoSplitter
|
||||
{
|
||||
NONCOPYABLE(IoSplitter);
|
||||
public:
|
||||
IoSplitter(off_t ofs, u8* alignedBuf, off_t size)
|
||||
: m_ofs(ofs), m_alignedBuf(alignedBuf), m_size(size)
|
||||
, m_totalIssued(0), m_totalTransferred(0)
|
||||
{
|
||||
m_alignedOfs = AlignedOffset(ofs);
|
||||
m_alignedSize = PaddedSize(size, ofs);
|
||||
m_misalignment = size_t(ofs - m_alignedOfs);
|
||||
}
|
||||
|
||||
LibError Run(const PFile& file, IoCallback cb = 0, uintptr_t cbData = 0)
|
||||
{
|
||||
ScopedIoMonitor monitor;
|
||||
|
||||
// (issue even if cache hit because blocks must be processed in order)
|
||||
std::deque<BlockIo> pendingIos;
|
||||
for(;;)
|
||||
{
|
||||
while(pendingIos.size() < ioDepth && m_totalIssued < m_alignedSize)
|
||||
{
|
||||
pendingIos.push_back(BlockIo());
|
||||
const off_t alignedOfs = m_alignedOfs + m_totalIssued;
|
||||
u8* const alignedBuf = m_alignedBuf? m_alignedBuf+m_totalIssued : 0;
|
||||
RETURN_ERR(pendingIos.back().Issue(file, alignedOfs, alignedBuf));
|
||||
m_totalIssued += BLOCK_SIZE;
|
||||
}
|
||||
|
||||
if(pendingIos.empty())
|
||||
break;
|
||||
|
||||
Process(pendingIos.front(), cb, cbData);
|
||||
pendingIos.pop_front();
|
||||
}
|
||||
|
||||
debug_assert(m_totalIssued >= m_totalTransferred && m_totalTransferred >= m_size);
|
||||
|
||||
monitor.NotifyOfSuccess(FI_AIO, file->Mode(), m_totalTransferred);
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
off_t AlignedOfs() const
|
||||
{
|
||||
return m_alignedOfs;
|
||||
}
|
||||
|
||||
private:
|
||||
LibError Process(BlockIo& blockIo, IoCallback cb, uintptr_t cbData) const
|
||||
{
|
||||
const u8* block; size_t blockSize;
|
||||
RETURN_ERR(blockIo.WaitUntilComplete(block, blockSize));
|
||||
|
||||
// first block: skip past alignment
|
||||
if(m_totalTransferred == 0)
|
||||
{
|
||||
block += m_misalignment;
|
||||
blockSize -= m_misalignment;
|
||||
}
|
||||
|
||||
// last block: don't include trailing padding
|
||||
if(m_totalTransferred + (off_t)blockSize > m_size)
|
||||
blockSize = size_t(m_size - m_totalTransferred);
|
||||
|
||||
m_totalTransferred += (off_t)blockSize;
|
||||
|
||||
if(cb)
|
||||
{
|
||||
stats_cb_start();
|
||||
LibError ret = cb(cbData, block, blockSize);
|
||||
stats_cb_finish();
|
||||
CHECK_ERR(ret);
|
||||
}
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
off_t m_ofs;
|
||||
u8* m_alignedBuf;
|
||||
off_t m_size;
|
||||
|
||||
size_t m_misalignment;
|
||||
off_t m_alignedOfs;
|
||||
off_t m_alignedSize;
|
||||
|
||||
// (useful, raw data: possibly compressed, but doesn't count padding)
|
||||
mutable off_t m_totalIssued;
|
||||
mutable off_t m_totalTransferred;
|
||||
};
|
||||
|
||||
|
||||
LibError io_Scan(const PFile& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData)
|
||||
{
|
||||
u8* alignedBuf = 0; // use temporary block buffers
|
||||
IoSplitter splitter(ofs, alignedBuf, size);
|
||||
return splitter.Run(file, cb, cbData);
|
||||
}
|
||||
|
||||
|
||||
LibError io_Read(const PFile& file, off_t ofs, u8* alignedBuf, off_t size, u8*& data, IoCallback cb, uintptr_t cbData)
|
||||
{
|
||||
IoSplitter splitter(ofs, alignedBuf, size);
|
||||
RETURN_ERR(splitter.Run(file, cb, cbData));
|
||||
data = alignedBuf + ofs - splitter.AlignedOfs();
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError io_WriteAligned(const PFile& file, off_t alignedOfs, const u8* alignedData, off_t size, IoCallback cb, uintptr_t cbData)
|
||||
LibError WaitUntilComplete(aiocb& cb, size_t queueDepth)
|
||||
{
|
||||
debug_assert(IsAligned_Offset(alignedOfs));
|
||||
debug_assert(IsAligned_Data(alignedData));
|
||||
#if CONFIG2_FILE_ENABLE_AIO
|
||||
if(queueDepth > 1)
|
||||
{
|
||||
aiocb* const cbs = &cb;
|
||||
timespec* const timeout = 0; // infinite
|
||||
SUSPEND_AGAIN:
|
||||
errno = 0;
|
||||
const int ret = aio_suspend(&cbs, 1, timeout);
|
||||
if(ret != 0)
|
||||
{
|
||||
if(errno == EINTR) // interrupted by signal
|
||||
goto SUSPEND_AGAIN;
|
||||
return LibError_from_errno();
|
||||
}
|
||||
|
||||
IoSplitter splitter(alignedOfs, const_cast<u8*>(alignedData), size);
|
||||
return splitter.Run(file, cb, cbData);
|
||||
const int err = aio_error(&cb);
|
||||
debug_assert(err != EINPROGRESS); // else aio_return is undefined
|
||||
ssize_t bytesTransferred = aio_return(&cb);
|
||||
if(bytesTransferred == -1) // transfer failed
|
||||
{
|
||||
errno = err;
|
||||
return LibError_from_errno();
|
||||
}
|
||||
cb.aio_nbytes = (size_t)bytesTransferred;
|
||||
}
|
||||
#endif
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError io_ReadAligned(const PFile& file, off_t alignedOfs, u8* alignedBuf, off_t size, IoCallback cb, uintptr_t cbData)
|
||||
{
|
||||
debug_assert(IsAligned_Offset(alignedOfs));
|
||||
debug_assert(IsAligned_Data(alignedBuf));
|
||||
|
||||
IoSplitter splitter(alignedOfs, alignedBuf, size);
|
||||
return splitter.Run(file, cb, cbData);
|
||||
}
|
||||
} // namespace io
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
/* Copyright (c) 2011 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
@ -21,36 +21,314 @@
|
||||
*/
|
||||
|
||||
/*
|
||||
* IO manager: splits requests into asynchronously issued blocks,
|
||||
* thus simplifying caching and enabling periodic callbacks.
|
||||
* provide asynchronous and synchronous I/O with hooks to allow
|
||||
* overlapped processing or progress reporting.
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_IO
|
||||
#define INCLUDED_IO
|
||||
|
||||
#include "lib/config2.h"
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/bits.h"
|
||||
#include "lib/file/file.h"
|
||||
#include "lib/sysdep/filesystem.h" // wtruncate
|
||||
|
||||
// memory will be allocated from the heap, not the (limited) file cache.
|
||||
// this makes sense for write buffers that are never used again,
|
||||
// because we avoid having to displace some other cached items.
|
||||
LIB_API shared_ptr<u8> io_Allocate(size_t size, off_t ofs = 0);
|
||||
#include "lib/allocators/unique_range.h"
|
||||
|
||||
/**
|
||||
* called after a block IO has completed.
|
||||
*
|
||||
* @return INFO::CB_CONTINUE to continue; any other value will cause the
|
||||
* IO splitter to abort immediately and return that.
|
||||
*
|
||||
* this is useful for user progress notification or processing data while
|
||||
* waiting for the next I/O to complete (without the complexity of threads).
|
||||
**/
|
||||
typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t blockSize);
|
||||
namespace ERR
|
||||
{
|
||||
const LibError IO = -110301;
|
||||
}
|
||||
|
||||
LIB_API LibError io_Scan(const PFile& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData);
|
||||
namespace io {
|
||||
|
||||
LIB_API LibError io_Read(const PFile& file, off_t ofs, u8* alignedBuf, off_t size, u8*& data, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
// @return memory suitable for use as an I/O buffer (address is a
|
||||
// multiple of alignment, size is rounded up to a multiple of alignment)
|
||||
//
|
||||
// use this instead of the file cache for write buffers that are
|
||||
// never reused (avoids displacing other items).
|
||||
LIB_API UniqueRange Allocate(size_t size, size_t alignment = maxSectorSize);
|
||||
|
||||
LIB_API LibError io_WriteAligned(const PFile& file, off_t alignedOfs, const u8* alignedData, off_t size, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
LIB_API LibError io_ReadAligned(const PFile& file, off_t alignedOfs, u8* alignedBuf, off_t size, IoCallback cb = 0, uintptr_t cbData = 0);
|
||||
|
||||
#pragma pack(push, 1)
|
||||
|
||||
// required information for any I/O (this is basically the same as aiocb,
|
||||
// but also applies to synchronous I/O and has shorter/nicer names.)
|
||||
struct Operation
|
||||
{
|
||||
// @param buf can be 0, in which case temporary block buffers are allocated.
|
||||
// otherwise, it must be padded to the I/O alignment, e.g. via io::Allocate.
|
||||
Operation(const File& file, void* buf, off_t size, off_t offset = 0)
|
||||
: fd(file.Descriptor()), opcode(file.Opcode())
|
||||
, offset(offset), size(size), buf((void*)buf)
|
||||
{
|
||||
}
|
||||
|
||||
void Validate() const
|
||||
{
|
||||
debug_assert(fd >= 0);
|
||||
debug_assert(opcode == LIO_READ || opcode == LIO_WRITE);
|
||||
|
||||
debug_assert(offset >= 0);
|
||||
debug_assert(size >= 0);
|
||||
// buf can legitimately be 0 (see above)
|
||||
}
|
||||
|
||||
int fd;
|
||||
int opcode;
|
||||
|
||||
off_t offset;
|
||||
off_t size;
|
||||
void* buf;
|
||||
};
|
||||
|
||||
|
||||
// optional information how an Operation is to be carried out
|
||||
struct Parameters
|
||||
{
|
||||
// default to single blocking I/Os
|
||||
Parameters()
|
||||
: alignment(1) // no alignment requirements
|
||||
// use one huge "block" truncated to the requested size.
|
||||
// (this value is a power of two as required by Validate and
|
||||
// avoids overflowing off_t in DivideRoundUp)
|
||||
, blockSize((SIZE_MAX/2)+1)
|
||||
, queueDepth(1) // disable aio
|
||||
{
|
||||
}
|
||||
|
||||
// parameters for asynchronous I/O that maximize throughput on current drives
|
||||
struct OverlappedTag {};
|
||||
Parameters(OverlappedTag)
|
||||
: alignment(maxSectorSize), blockSize(128*KiB), queueDepth(32)
|
||||
{
|
||||
}
|
||||
|
||||
Parameters(size_t blockSize, size_t queueDepth, off_t alignment = maxSectorSize)
|
||||
: alignment(alignment), blockSize(blockSize), queueDepth(queueDepth)
|
||||
{
|
||||
}
|
||||
|
||||
void Validate(const Operation& op) const
|
||||
{
|
||||
debug_assert(is_pow2(alignment));
|
||||
debug_assert(alignment > 0);
|
||||
|
||||
debug_assert(is_pow2(blockSize));
|
||||
debug_assert(pageSize <= blockSize); // no upper limit needed
|
||||
|
||||
debug_assert(1 <= queueDepth && queueDepth <= maxQueueDepth);
|
||||
|
||||
debug_assert(IsAligned(op.offset, alignment));
|
||||
// op.size doesn't need to be aligned
|
||||
debug_assert(IsAligned(op.buf, alignment));
|
||||
}
|
||||
|
||||
// (ATTO only allows 10, which improves upon 8)
|
||||
static const size_t maxQueueDepth = 32;
|
||||
|
||||
off_t alignment;
|
||||
|
||||
size_t blockSize;
|
||||
|
||||
size_t queueDepth;
|
||||
};
|
||||
|
||||
#define IO_OVERLAPPED io::Parameters(io::Parameters::OverlappedTag())
|
||||
|
||||
|
||||
struct DefaultCompletedHook
|
||||
{
|
||||
/**
|
||||
* called after a block I/O has completed.
|
||||
*
|
||||
* @return INFO::CB_CONTINUE to proceed; any other value will
|
||||
* be immediately returned by Run.
|
||||
*
|
||||
* allows progress notification and processing data while waiting for
|
||||
* previous I/Os to complete.
|
||||
**/
|
||||
LibError operator()(const u8* UNUSED(block), size_t UNUSED(blockSize)) const
|
||||
{
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct DefaultIssueHook
|
||||
{
|
||||
/**
|
||||
* called before a block I/O is issued.
|
||||
*
|
||||
* @return INFO::CB_CONTINUE to proceed; any other value will
|
||||
* be immediately returned by Run.
|
||||
*
|
||||
* allows generating the data to write while waiting for
|
||||
* previous I/Os to complete.
|
||||
**/
|
||||
LibError operator()(aiocb& UNUSED(cb)) const
|
||||
{
|
||||
return INFO::CB_CONTINUE;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// ring buffer of partially initialized aiocb that can be passed
|
||||
// directly to aio_write etc. after setting offset and buffer.
|
||||
class ControlBlockRingBuffer
|
||||
{
|
||||
public:
|
||||
ControlBlockRingBuffer(const Operation& op, const Parameters& p)
|
||||
: controlBlocks() // zero-initialize
|
||||
{
|
||||
// (default p.blockSize is "infinity", so clamp to the total size)
|
||||
const size_t blockSize = (size_t)std::min((off_t)p.blockSize, op.size);
|
||||
|
||||
const bool temporaryBuffersRequested = (op.buf == 0);
|
||||
if(temporaryBuffersRequested)
|
||||
buffers = RVALUE(io::Allocate(blockSize * p.queueDepth, p.alignment));
|
||||
|
||||
for(size_t i = 0; i < ARRAY_SIZE(controlBlocks); i++)
|
||||
{
|
||||
aiocb& cb = operator[](i);
|
||||
cb.aio_fildes = op.fd;
|
||||
cb.aio_nbytes = blockSize;
|
||||
cb.aio_lio_opcode = op.opcode;
|
||||
if(temporaryBuffersRequested)
|
||||
cb.aio_buf = (volatile void*)(uintptr_t(buffers.get()) + i * blockSize);
|
||||
}
|
||||
}
|
||||
|
||||
INLINE aiocb& operator[](size_t counter)
|
||||
{
|
||||
return controlBlocks[counter % ARRAY_SIZE(controlBlocks)];
|
||||
}
|
||||
|
||||
private:
|
||||
UniqueRange buffers;
|
||||
aiocb controlBlocks[Parameters::maxQueueDepth];
|
||||
};
|
||||
|
||||
#pragma pack(pop)
|
||||
|
||||
|
||||
LIB_API LibError Issue(aiocb& cb, size_t queueDepth);
|
||||
LIB_API LibError WaitUntilComplete(aiocb& cb, size_t queueDepth);
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Run
|
||||
|
||||
// (hooks must be passed by const reference to allow passing rvalues.
|
||||
// functors with non-const member data can mark them as mutable.)
|
||||
template<class CompletedHook, class IssueHook>
|
||||
static inline LibError Run(const Operation& op, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook())
|
||||
{
|
||||
op.Validate();
|
||||
p.Validate(op);
|
||||
|
||||
ControlBlockRingBuffer controlBlockRingBuffer(op, p);
|
||||
|
||||
const off_t numBlocks = DivideRoundUp(op.size, (off_t)p.blockSize);
|
||||
for(off_t blocksIssued = 0, blocksCompleted = 0; blocksCompleted < numBlocks; blocksCompleted++)
|
||||
{
|
||||
for(; blocksIssued != numBlocks && blocksIssued < blocksCompleted + (off_t)p.queueDepth; blocksIssued++)
|
||||
{
|
||||
aiocb& cb = controlBlockRingBuffer[blocksIssued];
|
||||
cb.aio_offset = op.offset + blocksIssued * p.blockSize;
|
||||
if(op.buf)
|
||||
cb.aio_buf = (volatile void*)(uintptr_t(op.buf) + blocksIssued * p.blockSize);
|
||||
if(blocksIssued == numBlocks-1)
|
||||
cb.aio_nbytes = round_up(size_t(op.size - blocksIssued * p.blockSize), size_t(p.alignment));
|
||||
|
||||
RETURN_IF_NOT_CONTINUE(issueHook(cb));
|
||||
|
||||
RETURN_ERR(Issue(cb, p.queueDepth));
|
||||
}
|
||||
|
||||
aiocb& cb = controlBlockRingBuffer[blocksCompleted];
|
||||
RETURN_ERR(WaitUntilComplete(cb, p.queueDepth));
|
||||
|
||||
RETURN_IF_NOT_CONTINUE(completedHook((u8*)cb.aio_buf, cb.aio_nbytes));
|
||||
}
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
// (overloads allow omitting parameters without requiring a template argument list)
|
||||
template<class CompletedHook>
|
||||
static inline LibError Run(const Operation& op, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook())
|
||||
{
|
||||
return Run(op, p, completedHook, DefaultIssueHook());
|
||||
}
|
||||
|
||||
static inline LibError Run(const Operation& op, const Parameters& p = Parameters())
|
||||
{
|
||||
return Run(op, p, DefaultCompletedHook(), DefaultIssueHook());
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Store
|
||||
|
||||
// efficient writing requires preallocation, and the resulting file is
|
||||
// padded to the sector size and needs to be truncated afterwards.
|
||||
// this function takes care of both.
|
||||
template<class CompletedHook, class IssueHook>
|
||||
static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook())
|
||||
{
|
||||
File file(pathname, LIO_WRITE);
|
||||
io::Operation op(file, (void*)data, size);
|
||||
|
||||
#if OS_WIN && CONFIG2_FILE_ENABLE_AIO
|
||||
(void)waio_Preallocate(op.fd, (off_t)size, p.alignment);
|
||||
#endif
|
||||
|
||||
RETURN_ERR(io::Run(op, p, completedHook, issueHook));
|
||||
|
||||
file.Close(); // (required by wtruncate)
|
||||
|
||||
RETURN_ERR(wtruncate(pathname, size));
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
template<class CompletedHook>
|
||||
static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook())
|
||||
{
|
||||
return Store(pathname, data, size, p, completedHook, DefaultIssueHook());
|
||||
}
|
||||
|
||||
static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters())
|
||||
{
|
||||
return Store(pathname, data, size, p, DefaultCompletedHook(), DefaultIssueHook());
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Load
|
||||
|
||||
// convenience function provided for symmetry with Store
|
||||
template<class CompletedHook, class IssueHook>
|
||||
static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook())
|
||||
{
|
||||
File file(pathname, LIO_READ);
|
||||
io::Operation op(file, buf, size);
|
||||
return io::Run(op, p, completedHook, issueHook);
|
||||
}
|
||||
|
||||
template<class CompletedHook>
|
||||
static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook())
|
||||
{
|
||||
return Load(pathname, buf, size, p, completedHook, DefaultIssueHook());
|
||||
}
|
||||
|
||||
static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters())
|
||||
{
|
||||
return Load(pathname, buf, size, p, DefaultCompletedHook(), DefaultIssueHook());
|
||||
}
|
||||
|
||||
} // namespace io
|
||||
|
||||
#endif // #ifndef INCLUDED_IO
|
||||
|
@ -1,24 +0,0 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "lib/file/io/io_align.h"
|
@ -1,71 +0,0 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_IO_ALIGN
|
||||
#define INCLUDED_IO_ALIGN
|
||||
|
||||
#include "lib/bits.h" // IsAligned, round_up
|
||||
|
||||
/**
|
||||
* block := power-of-two sized chunk of a file.
|
||||
* all transfers are expanded to naturally aligned, whole blocks.
|
||||
* (this makes caching parts of files feasible; it is also much faster
|
||||
* for some aio implementations, e.g. wposix.)
|
||||
* (blocks are also thereby page-aligned, which allows write-protecting
|
||||
* file buffers without worrying about their boundaries.)
|
||||
**/
|
||||
static const size_t BLOCK_SIZE = 1024*KiB;
|
||||
|
||||
// note: *sizes* and *offsets* are aligned to blocks to allow zero-copy block cache.
|
||||
// that the *buffer* need only be sector-aligned (we assume 4kb for simplicity)
|
||||
// (this is a requirement of the underlying Windows OS)
|
||||
static const size_t SECTOR_SIZE = 4*KiB;
|
||||
|
||||
|
||||
template<class T>
|
||||
inline bool IsAligned_Data(T* address)
|
||||
{
|
||||
return IsAligned((uintptr_t)address, SECTOR_SIZE);
|
||||
}
|
||||
|
||||
inline bool IsAligned_Offset(off_t ofs)
|
||||
{
|
||||
return IsAligned(ofs, BLOCK_SIZE);
|
||||
}
|
||||
|
||||
|
||||
inline off_t AlignedOffset(off_t ofs)
|
||||
{
|
||||
return (off_t)round_down<size_t>(size_t(ofs), BLOCK_SIZE);
|
||||
}
|
||||
|
||||
inline off_t AlignedSize(off_t size)
|
||||
{
|
||||
return (off_t)round_up<size_t>(size_t(size), BLOCK_SIZE);
|
||||
}
|
||||
|
||||
inline off_t PaddedSize(off_t size, off_t ofs)
|
||||
{
|
||||
return (off_t)round_up<size_t>(size_t(size + ofs - AlignedOffset(ofs)), BLOCK_SIZE);
|
||||
}
|
||||
|
||||
#endif // #ifndef INCLUDED_IO_ALIGN
|
@ -25,12 +25,15 @@
|
||||
|
||||
#include "lib/bits.h" // IsAligned
|
||||
#include "lib/sysdep/cpu.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "lib/file/io/io.h"
|
||||
#include "lib/file/io/io_align.h"
|
||||
|
||||
|
||||
static const size_t BLOCK_SIZE = 512*KiB;
|
||||
|
||||
|
||||
WriteBuffer::WriteBuffer()
|
||||
: m_capacity(4096), m_data(io_Allocate(m_capacity)), m_size(0)
|
||||
: m_capacity(pageSize), m_data((u8*)rtl_AllocateAligned(m_capacity, maxSectorSize), AlignedDeleter()), m_size(0)
|
||||
{
|
||||
}
|
||||
|
||||
@ -40,7 +43,8 @@ void WriteBuffer::Append(const void* data, size_t size)
|
||||
if(m_size + size > m_capacity)
|
||||
{
|
||||
m_capacity = round_up_to_pow2(m_size + size);
|
||||
shared_ptr<u8> newData = io_Allocate(m_capacity);
|
||||
shared_ptr<u8> newData;
|
||||
AllocateAligned(newData, m_capacity, maxSectorSize);
|
||||
memcpy(newData.get(), m_data.get(), m_size);
|
||||
m_data = newData;
|
||||
}
|
||||
@ -62,12 +66,15 @@ void WriteBuffer::Overwrite(const void* data, size_t size, size_t offset)
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
UnalignedWriter::UnalignedWriter(const PFile& file, off_t ofs)
|
||||
: m_file(file), m_alignedBuf(io_Allocate(BLOCK_SIZE))
|
||||
: m_file(file), m_alignedBuf((u8*)rtl_AllocateAligned(BLOCK_SIZE, maxSectorSize), AlignedDeleter())
|
||||
{
|
||||
m_alignedOfs = AlignedOffset(ofs);
|
||||
m_alignedOfs = round_down(ofs, (off_t)BLOCK_SIZE);
|
||||
const size_t misalignment = (size_t)(ofs - m_alignedOfs);
|
||||
if(misalignment)
|
||||
io_ReadAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE);
|
||||
{
|
||||
io::Operation op(*m_file.get(), m_alignedBuf.get(), BLOCK_SIZE, m_alignedOfs);
|
||||
THROW_ERR(io::Run(op));
|
||||
}
|
||||
m_bytesUsed = misalignment;
|
||||
}
|
||||
|
||||
@ -84,9 +91,10 @@ LibError UnalignedWriter::Append(const u8* data, size_t size) const
|
||||
{
|
||||
// optimization: write directly from the input buffer, if possible
|
||||
const size_t alignedSize = (size / BLOCK_SIZE) * BLOCK_SIZE;
|
||||
if(m_bytesUsed == 0 && IsAligned(data, SECTOR_SIZE) && alignedSize != 0)
|
||||
if(m_bytesUsed == 0 && IsAligned(data, maxSectorSize) && alignedSize != 0)
|
||||
{
|
||||
RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, data, alignedSize));
|
||||
io::Operation op(*m_file.get(), (void*)data, alignedSize, m_alignedOfs);
|
||||
RETURN_ERR(io::Run(op));
|
||||
m_alignedOfs += (off_t)alignedSize;
|
||||
data += alignedSize;
|
||||
size -= alignedSize;
|
||||
@ -118,7 +126,8 @@ void UnalignedWriter::Flush() const
|
||||
|
||||
LibError UnalignedWriter::WriteBlock() const
|
||||
{
|
||||
RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE));
|
||||
io::Operation op(*m_file.get(), m_alignedBuf.get(), BLOCK_SIZE, m_alignedOfs);
|
||||
RETURN_ERR(io::Run(op));
|
||||
m_alignedOfs += BLOCK_SIZE;
|
||||
m_bytesUsed = 0;
|
||||
return INFO::OK;
|
||||
|
@ -30,7 +30,6 @@
|
||||
#include "lib/external_libraries/suppress_boost_warnings.h"
|
||||
|
||||
#include "lib/file/common/file_stats.h"
|
||||
#include "lib/file/io/io_align.h" // BLOCK_SIZE
|
||||
#include "lib/cache_adt.h" // Cache
|
||||
#include "lib/bits.h" // round_up
|
||||
#include "lib/allocators/allocators.h"
|
||||
@ -98,7 +97,7 @@ public:
|
||||
|
||||
shared_ptr<u8> Allocate(size_t size, const PAllocator& pthis)
|
||||
{
|
||||
const size_t alignedSize = round_up(size, BLOCK_SIZE);
|
||||
const size_t alignedSize = Align<maxSectorSize>(size);
|
||||
|
||||
u8* mem = (u8*)m_allocator.Allocate(alignedSize);
|
||||
if(!mem)
|
||||
@ -114,7 +113,7 @@ public:
|
||||
|
||||
void Deallocate(u8* mem, size_t size)
|
||||
{
|
||||
const size_t alignedSize = round_up(size, BLOCK_SIZE);
|
||||
const size_t alignedSize = Align<maxSectorSize>(size);
|
||||
|
||||
// (re)allow writes in case the buffer was made read-only. it would
|
||||
// be nice to unmap the buffer, but this is not possible because
|
||||
|
@ -150,7 +150,7 @@ public:
|
||||
fileContents = DummySharedPtr((u8*)0);
|
||||
else if(size > m_cacheSize)
|
||||
{
|
||||
fileContents = io_Allocate(size);
|
||||
RETURN_ERR(AllocateAligned(fileContents, size, maxSectorSize));
|
||||
RETURN_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size()));
|
||||
}
|
||||
else
|
||||
|
@ -316,6 +316,15 @@ STMT(\
|
||||
}\
|
||||
)
|
||||
|
||||
|
||||
// if expression evaluates to a negative error code, return 0.
|
||||
#define RETURN_IF_NOT_CONTINUE(expression)\
|
||||
STMT(\
|
||||
i64 err64__ = (i64)(expression);\
|
||||
if(err64__ != INFO::CB_CONTINUE)\
|
||||
return err64__;\
|
||||
)
|
||||
|
||||
// return an error and warn about it (replaces debug_warn+return)
|
||||
#define WARN_RETURN(err)\
|
||||
STMT(\
|
||||
@ -375,8 +384,8 @@ STMT(\
|
||||
STMT(\
|
||||
if(!(ok))\
|
||||
{\
|
||||
debug_warn(L"FYI: WARN_RETURN_IF_FALSE reports that a function failed."\
|
||||
L"feel free to ignore or suppress this warning.");\
|
||||
debug_warn(L"FYI: WARN_RETURN_IF_FALSE reports that a function failed. "\
|
||||
L"Feel free to ignore or suppress this warning.");\
|
||||
return ERR::FAIL;\
|
||||
}\
|
||||
)
|
||||
@ -392,8 +401,8 @@ STMT(\
|
||||
#define WARN_IF_FALSE(ok)\
|
||||
STMT(\
|
||||
if(!(ok))\
|
||||
debug_warn(L"FYI: WARN_IF_FALSE reports that a function failed."\
|
||||
L"feel free to ignore or suppress this warning.");\
|
||||
debug_warn(L"FYI: WARN_IF_FALSE reports that a function failed. "\
|
||||
L"Feel free to ignore or suppress this warning.");\
|
||||
)
|
||||
|
||||
|
||||
|
@ -5,7 +5,7 @@
|
||||
#include "lib/external_libraries/vorbis.h"
|
||||
|
||||
#include "lib/byte_order.h"
|
||||
#include "lib/file/file.h"
|
||||
#include "lib/file/io/io.h"
|
||||
#include "lib/file/file_system_util.h"
|
||||
|
||||
|
||||
@ -64,7 +64,8 @@ public:
|
||||
const off_t sizeRemaining = adapter->size - adapter->offset;
|
||||
const size_t sizeToRead = (size_t)std::min(sizeRequested, sizeRemaining);
|
||||
|
||||
if(adapter->file->Read(adapter->offset, (u8*)bufferToFill, sizeToRead) == INFO::OK)
|
||||
io::Operation op(*adapter->file.get(), bufferToFill, sizeToRead, adapter->offset);
|
||||
if(io::Run(op) == INFO::OK)
|
||||
{
|
||||
adapter->offset += sizeToRead;
|
||||
return sizeToRead;
|
||||
|
@ -27,10 +27,6 @@
|
||||
#ifndef INCLUDED_COMPILER
|
||||
#define INCLUDED_COMPILER
|
||||
|
||||
#include "lib/sysdep/arch.h" // ARCH_AMD64
|
||||
#include "lib/config.h" // CONFIG_OMIT_FP
|
||||
|
||||
|
||||
// detect compiler and its version (0 if not present, otherwise
|
||||
// major*100 + minor). note that more than one *_VERSION may be
|
||||
// non-zero due to interoperability (e.g. ICC with MSC).
|
||||
@ -60,18 +56,6 @@
|
||||
#endif
|
||||
|
||||
|
||||
// pass "omit frame pointer" setting on to the compiler
|
||||
#if MSC_VERSION && !ARCH_AMD64
|
||||
# if CONFIG_OMIT_FP
|
||||
# pragma optimize("y", on)
|
||||
# else
|
||||
# pragma optimize("y", off)
|
||||
# endif
|
||||
#elif GCC_VERSION
|
||||
// TODO
|
||||
#endif
|
||||
|
||||
|
||||
// are PreCompiled Headers supported?
|
||||
#if MSC_VERSION
|
||||
# define HAVE_PCH 1
|
||||
@ -82,19 +66,6 @@
|
||||
#endif
|
||||
|
||||
|
||||
// try to define _W64, if not already done
|
||||
// (this is useful for catching pointer size bugs)
|
||||
#ifndef _W64
|
||||
# if MSC_VERSION
|
||||
# define _W64 __w64
|
||||
# elif GCC_VERSION
|
||||
# define _W64 __attribute__((mode (__pointer__)))
|
||||
# else
|
||||
# define _W64
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
// check if compiling in pure C mode (not C++) with support for C99.
|
||||
// (this is more convenient than testing __STDC_VERSION__ directly)
|
||||
//
|
||||
@ -118,7 +89,7 @@
|
||||
#endif
|
||||
|
||||
|
||||
// (at least rudimentary) support for C++0x
|
||||
// do we have (at least rudimentary) support for C++0x?
|
||||
#ifndef HAVE_CPP0X
|
||||
# if defined(__GXX_EXPERIMENTAL_CPP0X__) || MSC_VERSION >= 1600 || ICC_VERSION >= 1200
|
||||
# define HAVE_CPP0X 1
|
||||
@ -127,104 +98,4 @@
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
// C99-like restrict (non-standard in C++, but widely supported in various forms).
|
||||
//
|
||||
// May be used on pointers. May also be used on member functions to indicate
|
||||
// that 'this' is unaliased (e.g. "void C::m() RESTRICT { ... }").
|
||||
// Must not be used on references - GCC supports that but VC doesn't.
|
||||
//
|
||||
// We call this "RESTRICT" to avoid conflicts with VC's __declspec(restrict),
|
||||
// and because it's not really the same as C99's restrict.
|
||||
//
|
||||
// To be safe and satisfy the compilers' stated requirements: an object accessed
|
||||
// by a restricted pointer must not be accessed by any other pointer within the
|
||||
// lifetime of the restricted pointer, if the object is modified.
|
||||
// To maximise the chance of optimisation, any pointers that could potentially
|
||||
// alias with the restricted one should be marked as restricted too.
|
||||
//
|
||||
// It would probably be a good idea to write test cases for any code that uses
|
||||
// this in an even very slightly unclear way, in case it causes obscure problems
|
||||
// in a rare compiler due to differing semantics.
|
||||
//
|
||||
// .. GCC
|
||||
#if GCC_VERSION
|
||||
# define RESTRICT __restrict__
|
||||
// .. VC8 provides __restrict
|
||||
#elif MSC_VERSION >= 1400
|
||||
# define RESTRICT __restrict
|
||||
// .. ICC supports the keyword 'restrict' when run with the /Qrestrict option,
|
||||
// but it always also supports __restrict__ or __restrict to be compatible
|
||||
// with GCC/MSVC, so we'll use the underscored version. One of {GCC,MSC}_VERSION
|
||||
// should have been defined in addition to ICC_VERSION, so we should be using
|
||||
// one of the above cases (unless it's an old VS7.1-emulating ICC).
|
||||
#elif ICC_VERSION
|
||||
# error ICC_VERSION defined without either GCC_VERSION or an adequate MSC_VERSION
|
||||
// .. unsupported; remove it from code
|
||||
#else
|
||||
# define RESTRICT
|
||||
#endif
|
||||
|
||||
|
||||
// C99-style __func__
|
||||
// .. newer GCC already have it
|
||||
#if GCC_VERSION >= 300
|
||||
// nothing need be done
|
||||
// .. old GCC and MSVC have __FUNCTION__
|
||||
#elif GCC_VERSION >= 200 || MSC_VERSION
|
||||
# define __func__ __FUNCTION__
|
||||
// .. unsupported
|
||||
#else
|
||||
# define __func__ "(unknown)"
|
||||
#endif
|
||||
|
||||
|
||||
// tell the compiler that the code at/following this macro invocation is
|
||||
// unreachable. this can improve optimization and avoid warnings.
|
||||
//
|
||||
// this macro should not generate any fallback code; it is merely the
|
||||
// compiler-specific backend for lib.h's UNREACHABLE.
|
||||
// #define it to nothing if the compiler doesn't support such a hint.
|
||||
#define HAVE_ASSUME_UNREACHABLE 1
|
||||
#if MSC_VERSION && !ICC_VERSION // (ICC ignores this)
|
||||
# define ASSUME_UNREACHABLE __assume(0)
|
||||
#elif GCC_VERSION >= 450
|
||||
# define ASSUME_UNREACHABLE __builtin_unreachable()
|
||||
#else
|
||||
# define ASSUME_UNREACHABLE
|
||||
# undef HAVE_ASSUME_UNREACHABLE
|
||||
# define HAVE_ASSUME_UNREACHABLE 0
|
||||
#endif
|
||||
|
||||
|
||||
// extern "C", but does the right thing in pure-C mode
|
||||
#if defined(__cplusplus)
|
||||
# define EXTERN_C extern "C"
|
||||
#else
|
||||
# define EXTERN_C extern
|
||||
#endif
|
||||
|
||||
#if MSC_VERSION
|
||||
# define INLINE __forceinline
|
||||
#else
|
||||
# define INLINE inline
|
||||
#endif
|
||||
|
||||
#if MSC_VERSION
|
||||
# define CALL_CONV __cdecl
|
||||
#else
|
||||
# define CALL_CONV
|
||||
#endif
|
||||
|
||||
#if MSC_VERSION && !ARCH_AMD64
|
||||
# define DECORATED_NAME(name) _##name
|
||||
#else
|
||||
# define DECORATED_NAME(name) name
|
||||
#endif
|
||||
|
||||
// workaround for preprocessor limitation: macro args aren't expanded
|
||||
// before being pasted.
|
||||
#define STRINGIZE2(id) # id
|
||||
#define STRINGIZE(id) STRINGIZE2(id)
|
||||
|
||||
#endif // #ifndef INCLUDED_COMPILER
|
||||
|
@ -86,6 +86,11 @@ extern int wclose(int fd);
|
||||
// unistd.h
|
||||
//
|
||||
|
||||
// waio requires offsets and sizes to be multiples of the sector size.
|
||||
// to allow arbitrarily sized files, we truncate them after I/O.
|
||||
// however, ftruncate cannot be used since it is also subject to the
|
||||
// sector-alignment requirement. instead, the file must be closed and
|
||||
// this function called.
|
||||
LIB_API int wtruncate(const OsPath& pathname, off_t length);
|
||||
|
||||
LIB_API int wunlink(const OsPath& pathname);
|
||||
|
@ -32,6 +32,7 @@
|
||||
#include "lib/sysdep/os/win/win.h"
|
||||
#include "lib/sysdep/os/win/winit.h"
|
||||
#include "lib/sysdep/os/win/wutil.h"
|
||||
#include "lib/sysdep/os/win/wiocp.h"
|
||||
|
||||
|
||||
WINIT_REGISTER_MAIN_INIT(wdir_watch_Init);
|
||||
@ -290,84 +291,22 @@ struct DirWatch
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// CompletionPort
|
||||
|
||||
// this appears to be the best solution for IO notification.
|
||||
// there are three alternatives:
|
||||
// - multiple threads with blocking I/O. this is rather inefficient when
|
||||
// many directories (e.g. mods) are being watched.
|
||||
// - normal overlapped I/O: build a contiguous array of the hEvents
|
||||
// in all OVERLAPPED structures, and WaitForMultipleObjects.
|
||||
// it would be cumbersome to update this array when adding/removing watches.
|
||||
// - callback notification: a notification function is called when the thread
|
||||
// that initiated the I/O (ReadDirectoryChangesW) enters an alertable
|
||||
// wait state. it is desirable for notifications to arrive at a single
|
||||
// known point - see dir_watch_Poll. unfortunately there doesn't appear to
|
||||
// be a reliable and non-blocking means of entering AWS - SleepEx(1) may
|
||||
// wait for 10..15 ms if the system timer granularity is low. even worse,
|
||||
// it was noted in a previous project that APCs are sometimes delivered from
|
||||
// within APIs without having used SleepEx (it seems threads sometimes enter
|
||||
// a semi-AWS when calling the kernel).
|
||||
class CompletionPort
|
||||
{
|
||||
public:
|
||||
CompletionPort()
|
||||
{
|
||||
m_hIOCP = 0; // CreateIoCompletionPort requires 0, not INVALID_HANDLE_VALUE
|
||||
}
|
||||
|
||||
~CompletionPort()
|
||||
{
|
||||
CloseHandle(m_hIOCP);
|
||||
m_hIOCP = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
|
||||
void Attach(HANDLE hFile, uintptr_t key)
|
||||
{
|
||||
WinScopedPreserveLastError s; // CreateIoCompletionPort
|
||||
|
||||
// (when called for the first time, ends up creating m_hIOCP)
|
||||
m_hIOCP = CreateIoCompletionPort(hFile, m_hIOCP, (ULONG_PTR)key, 0);
|
||||
debug_assert(wutil_IsValidHandle(m_hIOCP));
|
||||
}
|
||||
|
||||
LibError Poll(size_t& bytesTransferred, uintptr_t& key, OVERLAPPED*& ovl)
|
||||
{
|
||||
if(m_hIOCP == 0)
|
||||
return ERR::INVALID_HANDLE; // NOWARN (happens if called before the first Attach)
|
||||
for(;;) // don't return abort notifications to caller
|
||||
{
|
||||
DWORD dwBytesTransferred = 0;
|
||||
ULONG_PTR ulKey = 0;
|
||||
ovl = 0;
|
||||
const DWORD timeout = 0;
|
||||
const BOOL gotPacket = GetQueuedCompletionStatus(m_hIOCP, &dwBytesTransferred, &ulKey, &ovl, timeout);
|
||||
bytesTransferred = size_t(dwBytesTransferred);
|
||||
key = uintptr_t(ulKey);
|
||||
if(gotPacket)
|
||||
return INFO::OK;
|
||||
|
||||
if(GetLastError() == WAIT_TIMEOUT)
|
||||
return ERR::AGAIN; // NOWARN (nothing pending)
|
||||
else if(GetLastError() == ERROR_OPERATION_ABORTED)
|
||||
continue; // watch was canceled - ignore
|
||||
else
|
||||
return LibError_from_GLE(); // actual error
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
HANDLE m_hIOCP;
|
||||
};
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// DirWatchManager
|
||||
|
||||
class DirWatchManager
|
||||
{
|
||||
public:
|
||||
DirWatchManager()
|
||||
: hIOCP(0)
|
||||
{
|
||||
}
|
||||
|
||||
~DirWatchManager()
|
||||
{
|
||||
CloseHandle(hIOCP);
|
||||
}
|
||||
|
||||
LibError Add(const OsPath& path, PDirWatch& dirWatch)
|
||||
{
|
||||
debug_assert(path.IsDirectory());
|
||||
@ -386,7 +325,7 @@ public:
|
||||
}
|
||||
|
||||
PDirWatchRequest request(new DirWatchRequest(path));
|
||||
m_completionPort.Attach(request->GetDirHandle(), (uintptr_t)request.get());
|
||||
AttachToCompletionPort(request->GetDirHandle(), hIOCP, (uintptr_t)request.get());
|
||||
RETURN_ERR(request->Issue());
|
||||
dirWatch.reset(new DirWatch(&m_sentinel, request));
|
||||
return INFO::OK;
|
||||
@ -394,8 +333,17 @@ public:
|
||||
|
||||
LibError Poll(DirWatchNotifications& notifications)
|
||||
{
|
||||
size_t bytesTransferred; uintptr_t key; OVERLAPPED* ovl;
|
||||
RETURN_ERR(m_completionPort.Poll(bytesTransferred, key, ovl));
|
||||
DWORD bytesTransferred; ULONG_PTR key; OVERLAPPED* ovl;
|
||||
for(;;) // skip notifications of canceled watches
|
||||
{
|
||||
const LibError ret = PollCompletionPort(hIOCP, 0, bytesTransferred, key, ovl);
|
||||
if(ret == INFO::OK)
|
||||
break;
|
||||
if(GetLastError() == ERROR_OPERATION_ABORTED)
|
||||
continue; // watch was canceled - ignore
|
||||
return ret;
|
||||
}
|
||||
|
||||
DirWatchRequest* request = (DirWatchRequest*)key;
|
||||
request->RetrieveNotifications(notifications);
|
||||
RETURN_ERR(request->Issue()); // re-issue
|
||||
@ -404,7 +352,7 @@ public:
|
||||
|
||||
private:
|
||||
IntrusiveLink m_sentinel;
|
||||
CompletionPort m_completionPort;
|
||||
HANDLE hIOCP;
|
||||
};
|
||||
|
||||
static DirWatchManager* s_dirWatchManager;
|
||||
|
@ -17,7 +17,7 @@ TableIds GetTableIDs(Provider provider)
|
||||
debug_assert(tableIdsSize % sizeof(TableId) == 0);
|
||||
TableIds tableIDs(DivideRoundUp(tableIdsSize, sizeof(TableId)), 0);
|
||||
|
||||
const size_t bytesWritten = pEnumSystemFirmwareTables(provider, &tableIDs[0], tableIdsSize);
|
||||
const size_t bytesWritten = pEnumSystemFirmwareTables(provider, &tableIDs[0], (DWORD)tableIdsSize);
|
||||
debug_assert(bytesWritten == tableIdsSize);
|
||||
|
||||
return tableIDs;
|
||||
@ -39,7 +39,7 @@ Table GetTable(Provider provider, TableId tableId)
|
||||
}
|
||||
|
||||
Table table(tableSize, 0);
|
||||
const size_t bytesWritten = pGetSystemFirmwareTable(provider, tableId, &table[0], tableSize);
|
||||
const size_t bytesWritten = pGetSystemFirmwareTable(provider, tableId, &table[0], (DWORD)tableSize);
|
||||
debug_assert(bytesWritten == tableSize);
|
||||
|
||||
return table;
|
||||
|
@ -27,7 +27,7 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/sysdep/os/win/whrt/counter.h"
|
||||
|
||||
#include "lib/bits.h"
|
||||
#include "lib/alignment.h"
|
||||
#include "lib/sysdep/cpu.h" // cpu_CAS
|
||||
|
||||
#include "lib/sysdep/os/win/whrt/tsc.h"
|
||||
@ -98,7 +98,7 @@ ICounter* CreateCounter(size_t id)
|
||||
|
||||
static const size_t memSize = 200;
|
||||
static u8 mem[memSize];
|
||||
u8* alignedMem = (u8*)round_up((uintptr_t)mem, (uintptr_t)16u);
|
||||
u8* alignedMem = (u8*)Align<16>((uintptr_t)mem);
|
||||
const size_t bytesLeft = mem+memSize - alignedMem;
|
||||
ICounter* counter = ConstructCounterAt(id, alignedMem, bytesLeft);
|
||||
|
||||
|
@ -33,7 +33,7 @@
|
||||
#include "lib/sysdep/os/win/win.h"
|
||||
#include "lib/sysdep/os/win/winit.h"
|
||||
#include "lib/sysdep/acpi.h"
|
||||
#include "lib/adts.h"
|
||||
//#include "lib/adts.h"
|
||||
#include "lib/bits.h"
|
||||
|
||||
#include "lib/sysdep/os/win/whrt/counter.h"
|
||||
|
32
source/lib/sysdep/os/win/wiocp.cpp
Normal file
32
source/lib/sysdep/os/win/wiocp.cpp
Normal file
@ -0,0 +1,32 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/sysdep/os/win/wiocp.h"
|
||||
|
||||
#include "lib/file/file.h" // ERR::IO
|
||||
#include "lib/sysdep/os/win/wutil.h"
|
||||
|
||||
|
||||
void AttachToCompletionPort(HANDLE hFile, HANDLE& hIOCP, ULONG_PTR key, DWORD numConcurrentThreads)
|
||||
{
|
||||
WinScopedPreserveLastError s; // CreateIoCompletionPort
|
||||
|
||||
// (when called for the first time, ends up creating hIOCP)
|
||||
hIOCP = CreateIoCompletionPort(hFile, hIOCP, key, numConcurrentThreads);
|
||||
debug_assert(wutil_IsValidHandle(hIOCP));
|
||||
}
|
||||
|
||||
|
||||
LibError PollCompletionPort(HANDLE hIOCP, DWORD timeout, DWORD& bytesTransferred, ULONG_PTR& key, OVERLAPPED*& ovl)
|
||||
{
|
||||
if(hIOCP == 0)
|
||||
return ERR::INVALID_HANDLE; // NOWARN (happens if called before the first Attach)
|
||||
|
||||
bytesTransferred = 0;
|
||||
key = 0;
|
||||
ovl = 0;
|
||||
if(GetQueuedCompletionStatus(hIOCP, &bytesTransferred, &key, &ovl, timeout))
|
||||
return INFO::OK;
|
||||
if(GetLastError() == WAIT_TIMEOUT)
|
||||
return ERR::AGAIN; // NOWARN (nothing pending)
|
||||
else
|
||||
return ERR::FAIL; // NOWARN (let caller decide what to do)
|
||||
}
|
50
source/lib/sysdep/os/win/wiocp.h
Normal file
50
source/lib/sysdep/os/win/wiocp.h
Normal file
@ -0,0 +1,50 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included
|
||||
* in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* I/O completion port
|
||||
*/
|
||||
|
||||
#ifndef INCLUDED_WIOCP
|
||||
#define INCLUDED_WIOCP
|
||||
|
||||
#include "lib/sysdep/os/win/win.h"
|
||||
|
||||
// this appears to be the best solution for IO notification.
|
||||
// there are three alternatives:
|
||||
// - multiple threads with blocking I/O. this is rather inefficient when
|
||||
// many directories (e.g. mods) are being watched.
|
||||
// - normal overlapped I/O: build a contiguous array of the hEvents
|
||||
// in all OVERLAPPED structures, and WaitForMultipleObjects.
|
||||
// it would be cumbersome to update this array when adding/removing watches.
|
||||
// - callback notification: a notification function is called when the thread
|
||||
// that initiated the I/O (ReadDirectoryChangesW) enters an alertable
|
||||
// wait state. it is desirable for notifications to arrive at a single
|
||||
// known point - see dir_watch_Poll. however, other APIs might also
|
||||
// trigger APC delivery.
|
||||
|
||||
// @param hIOCP 0 to create a new port
|
||||
extern void AttachToCompletionPort(HANDLE hFile, HANDLE& hIOCP, ULONG_PTR key, DWORD numConcurrentThreads = 0);
|
||||
|
||||
extern LibError PollCompletionPort(HANDLE hIOCP, DWORD timeout, DWORD& bytesTransferred, ULONG_PTR& key, OVERLAPPED*& ovl);
|
||||
|
||||
#endif // #ifndef INCLUDED_WIOCP
|
@ -23,7 +23,7 @@
|
||||
#include "precompiled.h"
|
||||
#include "lib/sysdep/numa.h"
|
||||
|
||||
#include "lib/bits.h" // round_up, PopulationCount
|
||||
#include "lib/bits.h" // PopulationCount
|
||||
#include "lib/timer.h"
|
||||
#include "lib/module_init.h"
|
||||
#include "lib/allocators/allocators.h" // page_aligned_alloc
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
/* Copyright (c) 2011 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
@ -24,105 +24,312 @@
|
||||
* emulate POSIX asynchronous I/O on Windows.
|
||||
*/
|
||||
|
||||
// NB: this module is significantly faster than Intel's aio library,
|
||||
// which also returns ERROR_INVALID_PARAMETER from aio_error if the
|
||||
// file is opened with FILE_FLAG_OVERLAPPED. (it looks like they are
|
||||
// using threaded blocking IO)
|
||||
|
||||
#include "precompiled.h"
|
||||
#include "lib/sysdep/os/win/wposix/waio.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "lib/sysdep/os/win/wposix/crt_posix.h" // correct definitions of _open() etc.
|
||||
#include "lib/alignment.h" // IsAligned
|
||||
#include "lib/module_init.h"
|
||||
#include "lib/sysdep/cpu.h" // cpu_AtomicAdd
|
||||
#include "lib/sysdep/filesystem.h" // O_NO_AIO_NP
|
||||
#include "lib/sysdep/os/win/wutil.h" // wutil_SetPrivilege
|
||||
#include "lib/sysdep/os/win/wiocp.h"
|
||||
#include "lib/sysdep/os/win/winit.h"
|
||||
|
||||
#include "lib/bits.h" // IsAligned
|
||||
#include "lib/sysdep/os/win/wposix/wposix_internal.h"
|
||||
#include "lib/sysdep/os/win/wposix/wtime.h" // timespec
|
||||
|
||||
|
||||
WINIT_REGISTER_MAIN_INIT(waio_Init);
|
||||
WINIT_REGISTER_MAIN_SHUTDOWN(waio_Shutdown);
|
||||
|
||||
// note: we assume sector sizes no larger than a page.
|
||||
// (GetDiskFreeSpace allows querying the actual size, but we'd
|
||||
// have to do so for all drives, and that may change depending on whether
|
||||
// there is a DVD in the drive or not)
|
||||
// sector size is relevant because Windows aio requires all IO
|
||||
// buffers, offsets and lengths to be a multiple of it. this requirement
|
||||
// is also carried over into the vfs / file.cpp interfaces for efficiency
|
||||
// (avoids the need for copying to/from align buffers).
|
||||
const uintptr_t sectorSize = 0x1000;
|
||||
// (dynamic linking preserves compatibility with previous Windows versions)
|
||||
static WUTIL_FUNC(pSetFileCompletionNotificationModes, BOOL, (HANDLE, UCHAR));
|
||||
static WUTIL_FUNC(pSetFileIoOverlappedRange, BOOL, (HANDLE, PUCHAR, ULONG));
|
||||
static WUTIL_FUNC(pSetFileValidData, BOOL, (HANDLE, LONGLONG));
|
||||
|
||||
// (there must be one global IOCP because aio_suspend might be called for
|
||||
// requests from different files)
|
||||
static HANDLE hIOCP;
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// OvlAllocator
|
||||
|
||||
// note: the Windows lowio file descriptor limit is currrently 2048.
|
||||
|
||||
/**
|
||||
* association between POSIX file descriptor and Win32 HANDLE.
|
||||
* NB: callers must ensure thread safety.
|
||||
**/
|
||||
class HandleManager
|
||||
// allocator for OVERLAPPED (enables a special optimization, see Associate)
|
||||
struct OvlAllocator // POD
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* associate an aio handle with a file descriptor.
|
||||
**/
|
||||
void Associate(int fd, HANDLE hFile)
|
||||
// freelist entries for (padded) OVERLAPPED from our storage
|
||||
struct Entry
|
||||
{
|
||||
debug_assert(fd > 2);
|
||||
debug_assert(GetFileSize(hFile, 0) != INVALID_FILE_SIZE);
|
||||
std::pair<Map::iterator, bool> ret = m_map.insert(std::make_pair(fd, hFile));
|
||||
debug_assert(ret.second); // fd better not already have been associated
|
||||
SLIST_ENTRY entry;
|
||||
OVERLAPPED ovl;
|
||||
};
|
||||
|
||||
LibError Init()
|
||||
{
|
||||
// the allocation must be naturally aligned to ensure it doesn't
|
||||
// overlap another page, which might prevent SetFileIoOverlappedRange
|
||||
// from pinning the pages if one of them is PAGE_NOACCESS.
|
||||
storage = _mm_malloc(storageSize, storageSize);
|
||||
if(!storage)
|
||||
WARN_RETURN(ERR::NO_MEM);
|
||||
memset(storage, 0, storageSize);
|
||||
|
||||
InitializeSListHead(&freelist);
|
||||
|
||||
// storageSize provides more than enough OVERLAPPED, so we
|
||||
// pad them to the cache line size to maybe avoid a few RFOs.
|
||||
const size_t size = Align<cacheLineSize>(sizeof(Entry));
|
||||
for(uintptr_t offset = 0; offset+size <= storageSize; offset += size)
|
||||
{
|
||||
Entry* entry = (Entry*)(uintptr_t(storage) + offset);
|
||||
debug_assert(IsAligned(entry, MEMORY_ALLOCATION_ALIGNMENT));
|
||||
InterlockedPushEntrySList(&freelist, &entry->entry);
|
||||
}
|
||||
|
||||
extant = 0;
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
void Dissociate(int fd)
|
||||
void Shutdown()
|
||||
{
|
||||
const size_t numRemoved = m_map.erase(fd);
|
||||
debug_assert(numRemoved == 1);
|
||||
debug_assert(extant == 0);
|
||||
|
||||
InterlockedFlushSList(&freelist);
|
||||
|
||||
_mm_free(storage);
|
||||
storage = 0;
|
||||
}
|
||||
|
||||
bool IsAssociated(int fd) const
|
||||
// irrevocably enable a special optimization for all I/Os requests
|
||||
// concerning this file, ending when the file is closed. has no effect
|
||||
// unless Vista+ and SeLockMemoryPrivilege are available.
|
||||
void Associate(HANDLE hFile)
|
||||
{
|
||||
return m_map.find(fd) != m_map.end();
|
||||
debug_assert(extant == 0);
|
||||
|
||||
// pin the page in kernel address space, which means our thread
|
||||
// won't have to be the one to service the I/O, thus avoiding an APC.
|
||||
// ("thread agnostic I/O")
|
||||
if(pSetFileIoOverlappedRange)
|
||||
WARN_IF_FALSE(pSetFileIoOverlappedRange(hFile, (PUCHAR)storage, storageSize));
|
||||
}
|
||||
|
||||
/**
|
||||
* @return aio handle associated with file descriptor or
|
||||
* INVALID_HANDLE_VALUE if there is none.
|
||||
**/
|
||||
HANDLE Get(int fd) const
|
||||
// @return OVERLAPPED initialized for I/O starting at offset,
|
||||
// or 0 if all available structures have already been allocated.
|
||||
OVERLAPPED* Allocate(off_t offset)
|
||||
{
|
||||
Map::const_iterator it = m_map.find(fd);
|
||||
if(it == m_map.end())
|
||||
return INVALID_HANDLE_VALUE;
|
||||
return it->second;
|
||||
Entry* entry = (Entry*)InterlockedPopEntrySList(&freelist);
|
||||
if(!entry)
|
||||
return 0;
|
||||
|
||||
OVERLAPPED& ovl = entry->ovl;
|
||||
ovl.Internal = 0;
|
||||
ovl.InternalHigh = 0;
|
||||
ovl.Offset = u64_lo(offset);
|
||||
ovl.OffsetHigh = u64_hi(offset);
|
||||
ovl.hEvent = 0; // (notification is via IOCP and/or polling)
|
||||
|
||||
cpu_AtomicAdd(&extant, +1);
|
||||
|
||||
return &ovl;
|
||||
}
|
||||
|
||||
private:
|
||||
typedef std::map<int, HANDLE> Map;
|
||||
Map m_map;
|
||||
void Deallocate(OVERLAPPED* ovl)
|
||||
{
|
||||
cpu_AtomicAdd(&extant, -1);
|
||||
|
||||
const uintptr_t address = uintptr_t(ovl);
|
||||
debug_assert(uintptr_t(storage) <= address && address < uintptr_t(storage)+storageSize);
|
||||
InterlockedPushEntrySList(&freelist, (PSLIST_ENTRY)(address - offsetof(Entry, ovl)));
|
||||
}
|
||||
|
||||
// one 4 KiB page is enough for 64 OVERLAPPED per file (i.e. plenty).
|
||||
static const size_t storageSize = pageSize;
|
||||
|
||||
void* storage;
|
||||
|
||||
#if MSC_VERSION
|
||||
# pragma warning(push)
|
||||
# pragma warning(disable:4324) // structure was padded due to __declspec(align())
|
||||
#endif
|
||||
__declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) SLIST_HEADER freelist;
|
||||
#if MSC_VERSION
|
||||
# pragma warning(pop)
|
||||
#endif
|
||||
|
||||
volatile intptr_t extant;
|
||||
};
|
||||
|
||||
static HandleManager* handleManager;
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// FileControlBlock
|
||||
|
||||
// do we want to open a second aio-capable handle?
|
||||
static bool IsAioPossible(int fd, bool is_com_port, int oflag)
|
||||
// (must correspond to static zero-initialization of fd)
|
||||
static const intptr_t FD_AVAILABLE = 0;
|
||||
|
||||
// information required to start asynchronous I/Os from a file
|
||||
// (aiocb stores a pointer to the originating FCB)
|
||||
struct FileControlBlock // POD
|
||||
{
|
||||
// stdin/stdout/stderr
|
||||
if(fd <= 2)
|
||||
return false;
|
||||
// search key, indicates the file descriptor with which this
|
||||
// FCB was associated (or FD_AVAILABLE if none).
|
||||
volatile intptr_t fd;
|
||||
|
||||
// COM port - we don't currently need aio access for those, and
|
||||
// aio_reopen's CreateFileW would fail with "access denied".
|
||||
if(is_com_port)
|
||||
return false;
|
||||
// second aio-enabled handle from waio_reopen
|
||||
HANDLE hFile;
|
||||
|
||||
// caller is requesting we skip it (see open())
|
||||
if(oflag & O_NO_AIO_NP)
|
||||
return false;
|
||||
OvlAllocator ovl;
|
||||
|
||||
return true;
|
||||
LibError Init()
|
||||
{
|
||||
fd = FD_AVAILABLE;
|
||||
hFile = INVALID_HANDLE_VALUE;
|
||||
return ovl.Init();
|
||||
}
|
||||
|
||||
void Shutdown()
|
||||
{
|
||||
debug_assert(fd == FD_AVAILABLE);
|
||||
debug_assert(hFile == INVALID_HANDLE_VALUE);
|
||||
ovl.Shutdown();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// NB: the Windows lowio file descriptor limit is 2048, but
|
||||
// our applications rarely open more than a few files at a time.
|
||||
static FileControlBlock fileControlBlocks[16];
|
||||
|
||||
|
||||
static FileControlBlock* AssociateFileControlBlock(int fd, HANDLE hFile)
|
||||
{
|
||||
for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
|
||||
{
|
||||
FileControlBlock& fcb = fileControlBlocks[i];
|
||||
if(cpu_CAS(&fcb.fd, FD_AVAILABLE, fd)) // the FCB is now ours
|
||||
{
|
||||
fcb.hFile = hFile;
|
||||
fcb.ovl.Associate(hFile);
|
||||
|
||||
AttachToCompletionPort(hFile, hIOCP, (ULONG_PTR)&fcb);
|
||||
|
||||
// minor optimization: avoid posting to IOCP in rare cases
|
||||
// where the I/O completes synchronously
|
||||
if(pSetFileCompletionNotificationModes)
|
||||
{
|
||||
// (for reasons as yet unknown, this fails when the file is
|
||||
// opened for read-only access)
|
||||
(void)pSetFileCompletionNotificationModes(fcb.hFile, FILE_SKIP_COMPLETION_PORT_ON_SUCCESS);
|
||||
}
|
||||
|
||||
return &fcb;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static void DissociateFileControlBlock(FileControlBlock* fcb)
|
||||
{
|
||||
fcb->hFile = INVALID_HANDLE_VALUE;
|
||||
fcb->fd = FD_AVAILABLE;
|
||||
}
|
||||
|
||||
|
||||
static FileControlBlock* FindFileControlBlock(int fd)
|
||||
{
|
||||
debug_assert(fd != FD_AVAILABLE);
|
||||
|
||||
for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
|
||||
{
|
||||
FileControlBlock& fcb = fileControlBlocks[i];
|
||||
if(fcb.fd == fd)
|
||||
return &fcb;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// init/shutdown
|
||||
|
||||
static ModuleInitState waio_initState;
|
||||
|
||||
static LibError waio_Init()
|
||||
{
|
||||
for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
|
||||
fileControlBlocks[i].Init();
|
||||
|
||||
WUTIL_IMPORT_KERNEL32(SetFileCompletionNotificationModes, pSetFileCompletionNotificationModes);
|
||||
|
||||
// NB: using these functions when the privileges are not available would
|
||||
// trigger warnings. since callers have to check the function pointers
|
||||
// anyway, just refrain from setting them in such cases.
|
||||
|
||||
if(wutil_SetPrivilege(L"SeLockMemoryPrivilege", true) == INFO::OK)
|
||||
WUTIL_IMPORT_KERNEL32(SetFileIoOverlappedRange, pSetFileIoOverlappedRange);
|
||||
|
||||
if(wutil_SetPrivilege(L"SeManageVolumePrivilege", true) == INFO::OK)
|
||||
WUTIL_IMPORT_KERNEL32(SetFileValidData, pSetFileValidData);
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
static LibError waio_Shutdown()
|
||||
{
|
||||
if(waio_initState == 0) // we were never initialized
|
||||
return INFO::OK;
|
||||
|
||||
for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
|
||||
fileControlBlocks[i].Shutdown();
|
||||
|
||||
WARN_IF_FALSE(CloseHandle(hIOCP));
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// OpenFile
|
||||
|
||||
static DWORD DesiredAccess(int oflag)
|
||||
{
|
||||
switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR))
|
||||
{
|
||||
case O_RDONLY:
|
||||
// (WinXP x64 requires FILE_WRITE_ATTRIBUTES for SetFileCompletionNotificationModes)
|
||||
return GENERIC_READ|FILE_WRITE_ATTRIBUTES;
|
||||
case O_WRONLY:
|
||||
return GENERIC_WRITE;
|
||||
case O_RDWR:
|
||||
return GENERIC_READ|GENERIC_WRITE;
|
||||
default:
|
||||
DEBUG_WARN_ERR(ERR::INVALID_PARAM);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static DWORD ShareMode(int oflag)
|
||||
{
|
||||
switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR))
|
||||
{
|
||||
case O_RDONLY:
|
||||
return FILE_SHARE_READ;
|
||||
case O_WRONLY:
|
||||
return FILE_SHARE_WRITE;
|
||||
case O_RDWR:
|
||||
return FILE_SHARE_READ|FILE_SHARE_WRITE;
|
||||
default:
|
||||
DEBUG_WARN_ERR(ERR::INVALID_PARAM);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static DWORD CreationDisposition(int oflag)
|
||||
{
|
||||
if(oflag & O_CREAT)
|
||||
@ -134,68 +341,68 @@ static DWORD CreationDisposition(int oflag)
|
||||
return OPEN_EXISTING;
|
||||
}
|
||||
|
||||
|
||||
// (re)open file in asynchronous mode and associate handle with fd.
|
||||
// (this works because the files default to DENY_NONE sharing)
|
||||
LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...)
|
||||
static DWORD FlagsAndAttributes()
|
||||
{
|
||||
WinScopedPreserveLastError s; // CreateFile
|
||||
// - FILE_FLAG_SEQUENTIAL_SCAN is ignored when FILE_FLAG_NO_BUFFERING
|
||||
// is set (c.f. "Windows via C/C++", p. 324)
|
||||
// - FILE_FLAG_WRITE_THROUGH is ~5% slower (diskspd.cpp suggests it
|
||||
// disables hardware caching; the overhead may also be due to the
|
||||
// Windows cache manager)
|
||||
const DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING;
|
||||
const DWORD attributes = FILE_ATTRIBUTE_NORMAL;
|
||||
return flags|attributes;
|
||||
}
|
||||
|
||||
debug_assert(!(oflag & O_APPEND)); // not supported
|
||||
if(!IsAioPossible(fd, false, oflag))
|
||||
return INFO::SKIPPED;
|
||||
static LibError OpenFile(const OsPath& pathname, int oflag, HANDLE& hFile)
|
||||
{
|
||||
WinScopedPreserveLastError s;
|
||||
|
||||
DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_SEQUENTIAL_SCAN;
|
||||
|
||||
// decode file access mode
|
||||
DWORD access, share;
|
||||
switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR))
|
||||
{
|
||||
case O_RDONLY:
|
||||
access = GENERIC_READ;
|
||||
share = FILE_SHARE_READ;
|
||||
break;
|
||||
|
||||
case O_WRONLY:
|
||||
access = GENERIC_WRITE;
|
||||
share = FILE_SHARE_WRITE;
|
||||
flags |= FILE_FLAG_WRITE_THROUGH;
|
||||
break;
|
||||
|
||||
case O_RDWR:
|
||||
access = GENERIC_READ|GENERIC_WRITE;
|
||||
share = FILE_SHARE_READ|FILE_SHARE_WRITE;
|
||||
flags |= FILE_FLAG_WRITE_THROUGH;
|
||||
break;
|
||||
|
||||
default:
|
||||
WARN_RETURN(ERR::INVALID_PARAM);
|
||||
}
|
||||
|
||||
// open file
|
||||
const DWORD access = DesiredAccess(oflag);
|
||||
const DWORD share = ShareMode(oflag);
|
||||
const DWORD create = CreationDisposition(oflag);
|
||||
const HANDLE hFile = CreateFileW(OsString(pathname).c_str(), access, share, 0, create, FILE_ATTRIBUTE_NORMAL|flags, 0);
|
||||
const DWORD flags = FlagsAndAttributes();
|
||||
hFile = CreateFileW(OsString(pathname).c_str(), access, share, 0, create, flags, 0);
|
||||
if(hFile == INVALID_HANDLE_VALUE)
|
||||
return LibError_from_GLE();
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// Windows-only APIs
|
||||
|
||||
LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...)
|
||||
{
|
||||
debug_assert(fd > 2);
|
||||
debug_assert(!(oflag & O_APPEND)); // not supported
|
||||
|
||||
if(oflag & O_NO_AIO_NP)
|
||||
return INFO::SKIPPED;
|
||||
|
||||
RETURN_ERR(ModuleInit(&waio_initState, waio_Init));
|
||||
|
||||
HANDLE hFile;
|
||||
RETURN_ERR(OpenFile(pathname, oflag, hFile));
|
||||
|
||||
if(!AssociateFileControlBlock(fd, hFile))
|
||||
{
|
||||
WinScopedLock lock(WAIO_CS);
|
||||
handleManager->Associate(fd, hFile);
|
||||
CloseHandle(hFile);
|
||||
WARN_RETURN(ERR::LIMIT);
|
||||
}
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
LibError waio_close(int fd)
|
||||
{
|
||||
HANDLE hFile;
|
||||
{
|
||||
WinScopedLock lock(WAIO_CS);
|
||||
if(!handleManager->IsAssociated(fd)) // wasn't opened for aio
|
||||
return INFO::SKIPPED;
|
||||
hFile = handleManager->Get(fd);
|
||||
handleManager->Dissociate(fd);
|
||||
}
|
||||
FileControlBlock* fcb = FindFileControlBlock(fd);
|
||||
if(!fcb)
|
||||
WARN_RETURN(ERR::INVALID_HANDLE);
|
||||
const HANDLE hFile = fcb->hFile;
|
||||
|
||||
DissociateFileControlBlock(fcb);
|
||||
|
||||
if(!CloseHandle(hFile))
|
||||
WARN_RETURN(ERR::INVALID_HANDLE);
|
||||
@ -204,266 +411,106 @@ LibError waio_close(int fd)
|
||||
}
|
||||
|
||||
|
||||
// we don't want to #define read to _read, since that's a fairly common
|
||||
// identifier. therefore, translate from MS CRT names via thunk functions.
|
||||
// efficiency is less important, and the overhead could be optimized away.
|
||||
|
||||
int read(int fd, void* buf, size_t nbytes)
|
||||
LibError waio_Preallocate(int fd, off_t alignedSize, off_t alignment)
|
||||
{
|
||||
return _read(fd, buf, (int)nbytes);
|
||||
}
|
||||
debug_assert(IsAligned(alignedSize, alignment));
|
||||
|
||||
int write(int fd, void* buf, size_t nbytes)
|
||||
{
|
||||
return _write(fd, buf, (int)nbytes);
|
||||
}
|
||||
|
||||
off_t lseek(int fd, off_t ofs, int whence)
|
||||
{
|
||||
return _lseeki64(fd, ofs, whence);
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
class aiocb::Impl
|
||||
{
|
||||
public:
|
||||
Impl()
|
||||
{
|
||||
m_hFile = INVALID_HANDLE_VALUE;
|
||||
|
||||
// (hEvent is initialized below and the rest in Issue(), but clear out
|
||||
// any subsequently added fields)
|
||||
memset(&m_overlapped, 0, sizeof(m_overlapped));
|
||||
|
||||
const BOOL manualReset = TRUE;
|
||||
const BOOL initialState = FALSE;
|
||||
m_overlapped.hEvent = CreateEvent(0, manualReset, initialState, 0);
|
||||
}
|
||||
|
||||
~Impl()
|
||||
{
|
||||
CloseHandle(m_overlapped.hEvent);
|
||||
}
|
||||
|
||||
LibError Issue(HANDLE hFile, off_t ofs, void* buf, size_t size, bool isWrite)
|
||||
{
|
||||
WinScopedPreserveLastError s;
|
||||
|
||||
m_hFile = hFile;
|
||||
|
||||
// note: Read-/WriteFile reset m_overlapped.hEvent, so we don't have to.
|
||||
m_overlapped.Internal = m_overlapped.InternalHigh = 0;
|
||||
m_overlapped.Offset = u64_lo(ofs);
|
||||
m_overlapped.OffsetHigh = u64_hi(ofs);
|
||||
|
||||
DWORD bytesTransferred;
|
||||
BOOL ok;
|
||||
if(isWrite)
|
||||
ok = WriteFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped);
|
||||
else
|
||||
ok = ReadFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped);
|
||||
if(!ok && GetLastError() == ERROR_IO_PENDING) // "pending" isn't an error
|
||||
{
|
||||
ok = TRUE;
|
||||
SetLastError(0);
|
||||
}
|
||||
return LibError_from_win32(ok);
|
||||
}
|
||||
|
||||
bool HasCompleted() const
|
||||
{
|
||||
// NB: .Internal "was originally reserved for system use and its behavior may change".
|
||||
// besides 0 and STATUS_PENDING, I have seen the address of a pointer to a buffer.
|
||||
return HasOverlappedIoCompleted(&m_overlapped);
|
||||
}
|
||||
|
||||
// required for WaitForMultipleObjects
|
||||
HANDLE Event() const
|
||||
{
|
||||
return m_overlapped.hEvent;
|
||||
}
|
||||
|
||||
LibError GetResult(size_t* pBytesTransferred)
|
||||
{
|
||||
DWORD bytesTransferred;
|
||||
const BOOL wait = FALSE; // callers should wait until HasCompleted
|
||||
if(!GetOverlappedResult(m_hFile, &m_overlapped, &bytesTransferred, wait))
|
||||
{
|
||||
*pBytesTransferred = 0;
|
||||
return LibError_from_GLE();
|
||||
}
|
||||
else
|
||||
{
|
||||
*pBytesTransferred = bytesTransferred;
|
||||
return INFO::OK;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
OVERLAPPED m_overlapped;
|
||||
HANDLE m_hFile;
|
||||
};
|
||||
|
||||
|
||||
// called by aio_read, aio_write, and lio_listio.
|
||||
// cb->aio_lio_opcode specifies desired operation.
|
||||
// @return LibError, also setting errno in case of failure.
|
||||
static LibError aio_issue(struct aiocb* cb)
|
||||
{
|
||||
// no-op (probably from lio_listio)
|
||||
if(!cb || cb->aio_lio_opcode == LIO_NOP)
|
||||
return INFO::SKIPPED;
|
||||
|
||||
// extract aiocb fields for convenience
|
||||
const bool isWrite = (cb->aio_lio_opcode == LIO_WRITE);
|
||||
const int fd = cb->aio_fildes;
|
||||
const size_t size = cb->aio_nbytes;
|
||||
const off_t ofs = cb->aio_offset;
|
||||
void* const buf = (void*)cb->aio_buf; // from volatile void*
|
||||
|
||||
// Win32 requires transfers to be sector-aligned.
|
||||
if(!IsAligned(ofs, sectorSize) || !IsAligned(buf, sectorSize) || !IsAligned(size, sectorSize))
|
||||
{
|
||||
errno = EINVAL;
|
||||
WARN_RETURN(ERR::INVALID_PARAM);
|
||||
}
|
||||
|
||||
HANDLE hFile;
|
||||
{
|
||||
WinScopedLock lock(WAIO_CS);
|
||||
hFile = handleManager->Get(fd);
|
||||
}
|
||||
if(hFile == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
errno = EINVAL;
|
||||
FileControlBlock* fcb = FindFileControlBlock(fd);
|
||||
if(!fcb)
|
||||
WARN_RETURN(ERR::INVALID_HANDLE);
|
||||
}
|
||||
const HANDLE hFile = fcb->hFile;
|
||||
|
||||
debug_assert(!cb->impl); // SUSv3 requires that the aiocb not be in use
|
||||
cb->impl.reset(new aiocb::Impl);
|
||||
// allocate all space up front to reduce fragmentation
|
||||
LARGE_INTEGER size64; size64.QuadPart = alignedSize;
|
||||
WARN_IF_FALSE(SetFilePointerEx(hFile, size64, 0, FILE_BEGIN));
|
||||
WARN_IF_FALSE(SetEndOfFile(hFile));
|
||||
|
||||
LibError ret = cb->impl->Issue(hFile, ofs, buf, size, isWrite);
|
||||
if(ret < 0)
|
||||
{
|
||||
LibError_set_errno(ret);
|
||||
return ret;
|
||||
}
|
||||
// avoid synchronous zero-fill (see discussion in header)
|
||||
if(pSetFileValidData)
|
||||
WARN_IF_FALSE(pSetFileValidData(hFile, alignedSize));
|
||||
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
// return status of transfer
|
||||
int aio_error(const struct aiocb* cb)
|
||||
{
|
||||
return cb->impl->HasCompleted()? 0 : EINPROGRESS;
|
||||
}
|
||||
//-----------------------------------------------------------------------------
|
||||
// helper functions
|
||||
|
||||
|
||||
// get bytes transferred. call exactly once for each issued request.
|
||||
ssize_t aio_return(struct aiocb* cb)
|
||||
// called by aio_read, aio_write, and lio_listio.
|
||||
// cb->aio_lio_opcode specifies desired operation.
|
||||
// @return -1 on failure (having also set errno)
|
||||
static int Issue(aiocb* cb)
|
||||
{
|
||||
// SUSv3 says we mustn't be callable before the request has completed
|
||||
debug_assert(cb->impl);
|
||||
debug_assert(cb->impl->HasCompleted());
|
||||
size_t bytesTransferred;
|
||||
LibError ret = cb->impl->GetResult(&bytesTransferred);
|
||||
cb->impl.reset(); // disallow calling again, as required by SUSv3
|
||||
if(ret < 0)
|
||||
debug_assert(IsAligned(cb->aio_offset, maxSectorSize));
|
||||
debug_assert(IsAligned(cb->aio_buf, maxSectorSize));
|
||||
debug_assert(IsAligned(cb->aio_nbytes, maxSectorSize));
|
||||
|
||||
FileControlBlock* fcb = FindFileControlBlock(cb->aio_fildes);
|
||||
if(!fcb || fcb->hFile == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
LibError_set_errno(ret);
|
||||
return (ssize_t)-1;
|
||||
}
|
||||
return (ssize_t)bytesTransferred;
|
||||
}
|
||||
|
||||
|
||||
int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* ts)
|
||||
{
|
||||
if(n <= 0 || n > MAXIMUM_WAIT_OBJECTS)
|
||||
{
|
||||
WARN_ERR(ERR::INVALID_PARAM);
|
||||
DEBUG_WARN_ERR(ERR::INVALID_HANDLE);
|
||||
errno = EINVAL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// build array of event handles
|
||||
HANDLE hEvents[MAXIMUM_WAIT_OBJECTS];
|
||||
size_t numPendingIos = 0;
|
||||
debug_assert(!cb->fcb && !cb->ovl); // SUSv3: aiocb must not be in use
|
||||
cb->fcb = fcb;
|
||||
cb->ovl = fcb->ovl.Allocate(cb->aio_offset);
|
||||
if(!cb->ovl)
|
||||
{
|
||||
DEBUG_WARN_ERR(ERR::LIMIT);
|
||||
errno = EMFILE;
|
||||
return -1;
|
||||
}
|
||||
|
||||
WinScopedPreserveLastError s;
|
||||
|
||||
const HANDLE hFile = fcb->hFile;
|
||||
void* const buf = (void*)cb->aio_buf; // from volatile void*
|
||||
const DWORD size = u64_lo(cb->aio_nbytes);
|
||||
debug_assert(u64_hi(cb->aio_nbytes) == 0);
|
||||
OVERLAPPED* ovl = (OVERLAPPED*)cb->ovl;
|
||||
// (there is no point in using WriteFileGather/ReadFileScatter here
|
||||
// because the IO manager still needs to lock pages and translate them
|
||||
// into an MDL, and we'd just be increasing the number of addresses)
|
||||
const BOOL ok = (cb->aio_lio_opcode == LIO_WRITE)? WriteFile(hFile, buf, size, 0, ovl) : ReadFile(hFile, buf, size, 0, ovl);
|
||||
if(ok || GetLastError() == ERROR_IO_PENDING)
|
||||
return 0; // success
|
||||
|
||||
LibError_set_errno(LibError_from_GLE());
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
static bool AreAnyComplete(const struct aiocb* const cbs[], int n)
|
||||
{
|
||||
for(int i = 0; i < n; i++)
|
||||
{
|
||||
if(!cbs[i]) // SUSv3 says NULL entries are to be ignored
|
||||
if(!cbs[i]) // SUSv3: must ignore NULL entries
|
||||
continue;
|
||||
|
||||
aiocb::Impl* impl = cbs[i]->impl.get();
|
||||
debug_assert(impl);
|
||||
if(!impl->HasCompleted())
|
||||
hEvents[numPendingIos++] = impl->Event();
|
||||
if(HasOverlappedIoCompleted((OVERLAPPED*)cbs[i]->ovl))
|
||||
return true;
|
||||
}
|
||||
if(!numPendingIos) // done, don't need to suspend.
|
||||
return 0;
|
||||
|
||||
const BOOL waitAll = FALSE;
|
||||
// convert timespec to milliseconds (ts == 0 => no timeout)
|
||||
const DWORD timeout = ts? (DWORD)(ts->tv_sec*1000 + ts->tv_nsec/1000000) : INFINITE;
|
||||
const DWORD result = WaitForMultipleObjects((DWORD)numPendingIos, hEvents, waitAll, timeout);
|
||||
|
||||
for(size_t i = 0; i < numPendingIos; i++)
|
||||
ResetEvent(hEvents[i]);
|
||||
|
||||
switch(result)
|
||||
{
|
||||
case WAIT_FAILED:
|
||||
WARN_ERR(ERR::FAIL);
|
||||
errno = EIO;
|
||||
return -1;
|
||||
|
||||
case WAIT_TIMEOUT:
|
||||
errno = EAGAIN;
|
||||
return -1;
|
||||
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
int aio_cancel(int fd, struct aiocb* cb)
|
||||
{
|
||||
// Win32 limitation: can't cancel single transfers -
|
||||
// all pending reads on this file are canceled.
|
||||
UNUSED2(cb);
|
||||
|
||||
HANDLE hFile;
|
||||
{
|
||||
WinScopedLock lock(WAIO_CS);
|
||||
hFile = handleManager->Get(fd);
|
||||
}
|
||||
if(hFile == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
WARN_ERR(ERR::INVALID_HANDLE);
|
||||
errno = EINVAL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
WARN_IF_FALSE(CancelIo(hFile));
|
||||
return AIO_CANCELED;
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// API
|
||||
|
||||
int aio_read(struct aiocb* cb)
|
||||
{
|
||||
cb->aio_lio_opcode = LIO_READ;
|
||||
return (aio_issue(cb) < 0)? 0 : -1;
|
||||
return Issue(cb);
|
||||
}
|
||||
|
||||
|
||||
int aio_write(struct aiocb* cb)
|
||||
{
|
||||
cb->aio_lio_opcode = LIO_WRITE;
|
||||
return (aio_issue(cb) < 0)? 0 : -1;
|
||||
return Issue(cb);
|
||||
}
|
||||
|
||||
|
||||
@ -474,7 +521,10 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se)
|
||||
|
||||
for(int i = 0; i < n; i++)
|
||||
{
|
||||
if(aio_issue(cbs[i]) < 0)
|
||||
if(cbs[i] == 0 || cbs[i]->aio_lio_opcode == LIO_NOP)
|
||||
continue;
|
||||
|
||||
if(Issue(cbs[i]) == -1)
|
||||
return -1;
|
||||
}
|
||||
|
||||
@ -485,24 +535,107 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se)
|
||||
}
|
||||
|
||||
|
||||
int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* timeout)
|
||||
{
|
||||
// consume all pending notifications to prevent them from piling up if
|
||||
// requests are always complete by the time we're called
|
||||
DWORD bytesTransferred; ULONG_PTR key; OVERLAPPED* ovl;
|
||||
while(PollCompletionPort(hIOCP, 0, bytesTransferred, key, ovl) == INFO::OK) {}
|
||||
|
||||
// avoid blocking if already complete (synchronous requests don't post notifications)
|
||||
if(AreAnyComplete(cbs, n))
|
||||
return 0;
|
||||
|
||||
// caller doesn't want to block, and no requests are complete
|
||||
if(timeout && timeout->tv_sec == 0 && timeout->tv_nsec == 0)
|
||||
{
|
||||
errno = EAGAIN;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// reduce CPU usage by blocking until a notification arrives or a
|
||||
// brief timeout elapses (necessary because other threads - or even
|
||||
// the above poll - might have consumed our notification). note that
|
||||
// re-posting notifications that don't concern the respective requests
|
||||
// is not desirable because POSIX doesn't require aio_suspend to be
|
||||
// called, which means notifications might pile up.
|
||||
const DWORD milliseconds = 1; // as short as possible (don't oversleep)
|
||||
const LibError ret = PollCompletionPort(hIOCP, milliseconds, bytesTransferred, key, ovl);
|
||||
if(ret != INFO::OK && ret != ERR::AGAIN) // failed
|
||||
{
|
||||
debug_assert(0);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// scan again (even if we got a notification, it might not concern THESE requests)
|
||||
if(AreAnyComplete(cbs, n))
|
||||
return 0;
|
||||
|
||||
// none completed, must repeat the above steps. provoke being called again by
|
||||
// claiming to have been interrupted by a signal.
|
||||
errno = EINTR;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
int aio_error(const struct aiocb* cb)
|
||||
{
|
||||
const OVERLAPPED* ovl = (const OVERLAPPED*)cb->ovl;
|
||||
if(!ovl) // called after aio_return
|
||||
return EINVAL;
|
||||
if(!HasOverlappedIoCompleted(ovl))
|
||||
return EINPROGRESS;
|
||||
if(ovl->Internal != ERROR_SUCCESS)
|
||||
return EIO;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
ssize_t aio_return(struct aiocb* cb)
|
||||
{
|
||||
FileControlBlock* fcb = (FileControlBlock*)cb->fcb;
|
||||
OVERLAPPED* ovl = (OVERLAPPED*)cb->ovl;
|
||||
if(!fcb || !ovl)
|
||||
{
|
||||
errno = EINVAL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
const ULONG_PTR status = ovl->Internal;
|
||||
const ULONG_PTR bytesTransferred = ovl->InternalHigh;
|
||||
|
||||
cb->ovl = 0; // prevent further calls to aio_error/aio_return
|
||||
COMPILER_FENCE;
|
||||
fcb->ovl.Deallocate(ovl);
|
||||
cb->fcb = 0; // allow reuse
|
||||
|
||||
return (status == ERROR_SUCCESS)? bytesTransferred : -1;
|
||||
}
|
||||
|
||||
|
||||
int aio_cancel(int UNUSED(fd), struct aiocb* cb)
|
||||
{
|
||||
// (faster than calling FindFileControlBlock)
|
||||
const HANDLE hFile = ((const FileControlBlock*)cb->fcb)->hFile;
|
||||
if(hFile == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
WARN_ERR(ERR::INVALID_HANDLE);
|
||||
errno = EINVAL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
// cancel all I/Os this thread issued for the given file
|
||||
// (CancelIoEx can cancel individual operations, but is only
|
||||
// available starting with Vista)
|
||||
WARN_IF_FALSE(CancelIo(hFile));
|
||||
|
||||
return AIO_CANCELED;
|
||||
}
|
||||
|
||||
|
||||
int aio_fsync(int, struct aiocb*)
|
||||
{
|
||||
WARN_ERR(ERR::NOT_IMPLEMENTED);
|
||||
errno = ENOSYS;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
static LibError waio_Init()
|
||||
{
|
||||
handleManager = new HandleManager;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
static LibError waio_Shutdown()
|
||||
{
|
||||
delete handleManager;
|
||||
return INFO::OK;
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
/* Copyright (c) 2010 Wildfire Games
|
||||
/* Copyright (c) 2011 Wildfire Games
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
@ -29,13 +29,12 @@
|
||||
|
||||
#include "lib/lib_errors.h"
|
||||
#include "lib/os_path.h"
|
||||
#include "lib/posix/posix_time.h" // timespec
|
||||
#include "lib/sysdep/os/win/wposix/wposix_types.h"
|
||||
|
||||
#include "lib/sysdep/os/win/wposix/no_crt_posix.h"
|
||||
|
||||
// Note: transfer buffers, offsets, and lengths must be sector-aligned
|
||||
// (we don't bother copying to an align buffer because the file cache
|
||||
// already requires splitting IOs into aligned blocks)
|
||||
// (we don't bother copying to an align buffer because our block cache
|
||||
// already requires splitting IOs into naturally-aligned blocks)
|
||||
|
||||
|
||||
//
|
||||
@ -57,15 +56,6 @@ struct sigevent // unused
|
||||
};
|
||||
|
||||
|
||||
//
|
||||
// <unistd.h>
|
||||
//
|
||||
|
||||
extern int read (int fd, void* buf, size_t nbytes); // thunk
|
||||
extern int write(int fd, void* buf, size_t nbytes); // thunk
|
||||
extern off_t lseek(int fd, off_t ofs, int whence); // thunk
|
||||
|
||||
|
||||
//
|
||||
// <aio.h>
|
||||
//
|
||||
@ -80,8 +70,11 @@ struct aiocb
|
||||
struct sigevent aio_sigevent; // Signal number and value. (unused)
|
||||
int aio_lio_opcode; // Operation to be performed.
|
||||
|
||||
class Impl;
|
||||
shared_ptr<Impl> impl;
|
||||
// internal use only; must be zero-initialized before
|
||||
// calling the first aio_read/aio_write/lio_listio (aio_return also
|
||||
// zero-initializes them)
|
||||
void* fcb;
|
||||
void* ovl;
|
||||
};
|
||||
|
||||
enum
|
||||
@ -101,21 +94,55 @@ enum
|
||||
LIO_WRITE
|
||||
};
|
||||
|
||||
extern int aio_cancel(int, struct aiocb*);
|
||||
extern int aio_error(const struct aiocb*);
|
||||
extern int aio_fsync(int, struct aiocb*);
|
||||
extern int aio_read(struct aiocb*);
|
||||
extern ssize_t aio_return(struct aiocb*);
|
||||
struct timespec;
|
||||
extern int aio_suspend(const struct aiocb* const[], int, const struct timespec*);
|
||||
extern int aio_write(struct aiocb*);
|
||||
extern int lio_listio(int, struct aiocb* const[], int, struct sigevent*);
|
||||
|
||||
// for use by wfilesystem's wopen/wclose:
|
||||
// (if never called, IOCP notifications will pile up.)
|
||||
extern int aio_suspend(const struct aiocb* const[], int, const struct timespec*);
|
||||
|
||||
// (re)open file in asynchronous mode and associate handle with fd.
|
||||
// (this works because the files default to DENY_NONE sharing)
|
||||
// @return status of transfer (0 or an errno)
|
||||
extern int aio_error(const struct aiocb*);
|
||||
|
||||
// @return bytes transferred or -1 on error.
|
||||
// frees internal storage related to the request and MUST be called
|
||||
// exactly once for each aiocb after aio_error != EINPROGRESS.
|
||||
extern ssize_t aio_return(struct aiocb*);
|
||||
|
||||
extern int aio_cancel(int, struct aiocb*);
|
||||
|
||||
extern int aio_fsync(int, struct aiocb*);
|
||||
|
||||
// Windows doesn't allow aio unless the file is opened in asynchronous mode,
|
||||
// which is not possible with _wsopen_s. since we don't want to have to
|
||||
// provide a separate File class for aio-enabled files, our wopen wrapper
|
||||
// will also call this function to open a SECOND handle to the file (works
|
||||
// because CRT open() defaults to DENY_NONE sharing). the CRT's lowio
|
||||
// descriptor table remains unaffected, but our [w]aio_* functions are
|
||||
// notified of the file descriptor, which means e.g. read and aio_read can
|
||||
// both be used. this function must have been called before any
|
||||
// other [w]aio_* functions are used.
|
||||
extern LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...);
|
||||
|
||||
// close our second aio-enabled handle to the file (called from wclose).
|
||||
extern LibError waio_close(int fd);
|
||||
|
||||
// call this before writing a large file to preallocate clusters, thus
|
||||
// reducing fragmentation.
|
||||
//
|
||||
// @param alignedSize must be a multiple of alignment (SetEndOfFile requires
|
||||
// sector alignment; this could be avoided by using the undocumented
|
||||
// NtSetInformationFile or SetFileInformationByHandle on Vista and later).
|
||||
// use wtruncate after I/O is complete to chop off any excess padding.
|
||||
//
|
||||
// NB: writes that extend a file (i.e. ALL WRITES when creating new files)
|
||||
// are synchronous, which prevents overlapping I/O and other work.
|
||||
// (http://support.microsoft.com/default.aspx?scid=kb%3Ben-us%3B156932)
|
||||
// if Windows XP and the SE_MANAGE_VOLUME_NAME privileges are available,
|
||||
// this function sets the valid data length to avoid the synchronous zero-fill.
|
||||
// note that this exposes the previous disk contents (possibly even to
|
||||
// other users since the waio_reopen design cannot deny file sharing) until
|
||||
// the application successfully writes to the file.
|
||||
LIB_API LibError waio_Preallocate(int fd, off_t alignedSize, off_t alignment);
|
||||
|
||||
#endif // #ifndef INCLUDED_WAIO
|
||||
|
@ -373,6 +373,26 @@ int wclose(int fd)
|
||||
// unistd.h
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// we don't want to #define read to _read, since that's a fairly common
|
||||
// identifier. therefore, translate from MS CRT names via thunk functions.
|
||||
// efficiency is less important, and the overhead could be optimized away.
|
||||
|
||||
int read(int fd, void* buf, size_t nbytes)
|
||||
{
|
||||
return _read(fd, buf, (int)nbytes);
|
||||
}
|
||||
|
||||
int write(int fd, void* buf, size_t nbytes)
|
||||
{
|
||||
return _write(fd, buf, (int)nbytes);
|
||||
}
|
||||
|
||||
off_t lseek(int fd, off_t ofs, int whence)
|
||||
{
|
||||
return _lseeki64(fd, ofs, whence);
|
||||
}
|
||||
|
||||
|
||||
int wtruncate(const OsPath& pathname, off_t length)
|
||||
{
|
||||
HANDLE hFile = CreateFileW(OsString(pathname).c_str(), GENERIC_WRITE, 0, 0, OPEN_EXISTING, 0, 0);
|
||||
|
@ -52,4 +52,13 @@ typedef unsigned int mode_t; // defined by MinGW but not VC
|
||||
#define S_ISDIR(m) (m & S_IFDIR)
|
||||
#define S_ISREG(m) (m & S_IFREG)
|
||||
|
||||
|
||||
//
|
||||
// <unistd.h>
|
||||
//
|
||||
|
||||
extern int read (int fd, void* buf, size_t nbytes); // thunk
|
||||
extern int write(int fd, void* buf, size_t nbytes); // thunk
|
||||
extern off_t lseek(int fd, off_t ofs, int whence); // thunk
|
||||
|
||||
#endif // #ifndef INCLUDED_WFILESYSTEM
|
||||
|
@ -66,7 +66,7 @@ std::wstring sys_WideFromArgv(const char* argv_i)
|
||||
const int inputSize = -1; // null-terminated
|
||||
std::vector<wchar_t> buf(strlen(argv_i)+1); // (upper bound on number of characters)
|
||||
// NB: avoid mbstowcs because it may specify another locale
|
||||
const int ret = MultiByteToWideChar(cp, flags, argv_i, inputSize, &buf[0], buf.size());
|
||||
const int ret = MultiByteToWideChar(cp, flags, argv_i, (int)inputSize, &buf[0], (int)buf.size());
|
||||
debug_assert(ret != 0);
|
||||
return std::wstring(&buf[0]);
|
||||
}
|
||||
|
@ -438,6 +438,32 @@ WinScopedDisableWow64Redirection::~WinScopedDisableWow64Redirection()
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
LibError wutil_SetPrivilege(const wchar_t* privilege, bool enable)
|
||||
{
|
||||
WinScopedPreserveLastError s;
|
||||
|
||||
HANDLE hToken;
|
||||
if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES|TOKEN_QUERY, &hToken))
|
||||
return ERR::_1;
|
||||
|
||||
TOKEN_PRIVILEGES tp;
|
||||
if (!LookupPrivilegeValueW(NULL, privilege, &tp.Privileges[0].Luid))
|
||||
return ERR::_2;
|
||||
tp.PrivilegeCount = 1;
|
||||
tp.Privileges[0].Attributes = enable? SE_PRIVILEGE_ENABLED : 0;
|
||||
|
||||
SetLastError(0);
|
||||
const BOOL ok = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, 0, 0);
|
||||
if(!ok || GetLastError() != 0)
|
||||
return ERR::_3;
|
||||
|
||||
WARN_IF_FALSE(CloseHandle(hToken));
|
||||
return INFO::OK;
|
||||
}
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// module handle
|
||||
|
||||
|
@ -77,7 +77,6 @@ extern void wutil_Free(void* p);
|
||||
// critical sections used by win-specific code
|
||||
enum WinLockId
|
||||
{
|
||||
WAIO_CS,
|
||||
WDBG_SYM_CS, // protects (non-reentrant) dbghelp.dll
|
||||
WDIR_WATCH_CS,
|
||||
|
||||
@ -191,6 +190,8 @@ private:
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
LIB_API LibError wutil_SetPrivilege(const wchar_t* privilege, bool enable);
|
||||
|
||||
/**
|
||||
* @return module handle of lib code (that of the main EXE if
|
||||
* linked statically, otherwise the DLL).
|
||||
|
@ -647,7 +647,7 @@ void FieldStringizer::operator()<const char*>(size_t flags, const char*& value,
|
||||
if(lastChar == std::string::npos) // nothing but spaces
|
||||
return;
|
||||
string.resize(lastChar+1); // strip trailing spaces
|
||||
if(string == "To Be Filled By O.E.M.")
|
||||
if(!stricmp(value, "To Be Filled By O.E.M."))
|
||||
return;
|
||||
|
||||
WriteName(name);
|
||||
|
@ -33,6 +33,7 @@
|
||||
|
||||
#include "lib/timer.h"
|
||||
#include "lib/bits.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "lib/sysdep/cpu.h"
|
||||
|
||||
#include "tex_codec.h"
|
||||
@ -255,7 +256,8 @@ static LibError add_mipmaps(Tex* t, size_t w, size_t h, size_t bpp, void* newDat
|
||||
WARN_RETURN(ERR::TEX_INVALID_SIZE);
|
||||
t->flags |= TEX_MIPMAPS; // must come before tex_img_size!
|
||||
const size_t mipmap_size = tex_img_size(t);
|
||||
shared_ptr<u8> mipmapData = io_Allocate(mipmap_size, 0);
|
||||
shared_ptr<u8> mipmapData;
|
||||
AllocateAligned(mipmapData, mipmap_size);
|
||||
CreateLevelData cld = { bpp/8, w, h, (const u8*)newData, data_size };
|
||||
tex_util_foreach_mipmap(w, h, bpp, mipmapData.get(), 0, 1, create_level, &cld);
|
||||
t->data = mipmapData;
|
||||
@ -332,7 +334,8 @@ TIMER_ACCRUE(tc_plain_transform);
|
||||
//
|
||||
// this is necessary even when not flipping because the initial data
|
||||
// is read-only.
|
||||
shared_ptr<u8> newData = io_Allocate(new_data_size);
|
||||
shared_ptr<u8> newData;
|
||||
AllocateAligned(newData, new_data_size);
|
||||
|
||||
// setup row source/destination pointers (simplifies outer loop)
|
||||
u8* dst = (u8*)newData.get();
|
||||
|
@ -27,12 +27,15 @@
|
||||
#include "precompiled.h"
|
||||
|
||||
#include "lib/byte_order.h"
|
||||
#include "tex_codec.h"
|
||||
#include "lib/bits.h"
|
||||
#include "lib/timer.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "tex_codec.h"
|
||||
|
||||
|
||||
// NOTE: the convention is bottom-up for DDS, but there's no way to tell.
|
||||
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
// S3TC decompression
|
||||
//-----------------------------------------------------------------------------
|
||||
@ -279,7 +282,8 @@ static LibError s3tc_decompress(Tex* t)
|
||||
const size_t dxt = t->flags & TEX_DXT;
|
||||
const size_t out_bpp = (dxt != 1)? 32 : 24;
|
||||
const size_t out_size = tex_img_size(t) * out_bpp / t->bpp;
|
||||
shared_ptr<u8> decompressedData = io_Allocate(out_size);
|
||||
shared_ptr<u8> decompressedData;
|
||||
AllocateAligned(decompressedData, out_size, pageSize);
|
||||
|
||||
const size_t s3tc_block_size = (dxt == 3 || dxt == 5)? 16 : 8;
|
||||
S3tcDecompressInfo di = { dxt, s3tc_block_size, out_bpp/8, decompressedData.get() };
|
||||
|
@ -28,7 +28,7 @@
|
||||
#define INCLUDED_TEX_INTERNAL
|
||||
|
||||
#include "lib/allocators/dynarray.h"
|
||||
#include "lib/file/io/io.h" // io_Allocate
|
||||
#include "lib/file/io/io.h" // io::Allocate
|
||||
|
||||
/**
|
||||
* check if the given texture format is acceptable: 8bpp grey,
|
||||
|
@ -26,10 +26,12 @@
|
||||
|
||||
#include "precompiled.h"
|
||||
|
||||
#include <setjmp.h>
|
||||
|
||||
#include "lib/external_libraries/libjpeg.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
|
||||
#include "tex_codec.h"
|
||||
#include <setjmp.h>
|
||||
|
||||
|
||||
// squelch "dtor / setjmp interaction" warnings.
|
||||
@ -478,7 +480,8 @@ static LibError jpg_decode_impl(DynArray* da, jpeg_decompress_struct* cinfo, Tex
|
||||
// alloc destination buffer
|
||||
const size_t pitch = w * bpp / 8;
|
||||
const size_t img_size = pitch * h; // for allow_rows
|
||||
shared_ptr<u8> data = io_Allocate(img_size);
|
||||
shared_ptr<u8> data;
|
||||
AllocateAligned(data, img_size, pageSize);
|
||||
|
||||
// read rows
|
||||
shared_ptr<RowPtr> rows = tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0);
|
||||
|
@ -30,6 +30,7 @@
|
||||
|
||||
#include "lib/byte_order.h"
|
||||
#include "tex_codec.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "lib/timer.h"
|
||||
|
||||
#if MSC_VERSION
|
||||
@ -111,7 +112,8 @@ static LibError png_decode_impl(DynArray* da, png_structp png_ptr, png_infop inf
|
||||
WARN_RETURN(ERR::TEX_INVALID_COLOR_TYPE);
|
||||
|
||||
const size_t img_size = pitch * h;
|
||||
shared_ptr<u8> data = io_Allocate(img_size);
|
||||
shared_ptr<u8> data;
|
||||
AllocateAligned(data, img_size, pageSize);
|
||||
|
||||
shared_ptr<RowPtr> rows = tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0);
|
||||
png_read_image(png_ptr, (png_bytepp)rows.get());
|
||||
|
@ -17,17 +17,18 @@
|
||||
|
||||
#include "precompiled.h"
|
||||
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
#include "Pyrogenesis.h"
|
||||
#include "Parser.h"
|
||||
#include "ConfigDB.h"
|
||||
#include "CLogger.h"
|
||||
#include "Filesystem.h"
|
||||
#include "scripting/ScriptingHost.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
|
||||
#include "scriptinterface/ScriptInterface.h"
|
||||
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
typedef std::map <CStr, CConfigValueSet> TConfigMap;
|
||||
TConfigMap CConfigDB::m_Map[CFG_LAST];
|
||||
VfsPath CConfigDB::m_ConfigFile[CFG_LAST];
|
||||
@ -425,7 +426,8 @@ bool CConfigDB::WriteFile(EConfigNamespace ns, const VfsPath& path)
|
||||
return false;
|
||||
}
|
||||
|
||||
shared_ptr<u8> buf = io_Allocate(1*MiB);
|
||||
shared_ptr<u8> buf;
|
||||
AllocateAligned(buf, 1*MiB, maxSectorSize);
|
||||
char* pos = (char*)buf.get();
|
||||
TConfigMap &map=m_Map[ns];
|
||||
for(TConfigMap::const_iterator it = map.begin(); it != map.end(); ++it)
|
||||
|
@ -33,7 +33,6 @@
|
||||
#include "lib/sysdep/arch/x86_x64/topology.h"
|
||||
#include "lib/sysdep/smbios.h"
|
||||
#include "lib/tex/tex.h"
|
||||
#include "lib/file/io/io_align.h" // BLOCK_SIZE
|
||||
|
||||
#include "ps/GameSetup/Config.h"
|
||||
#include "ps/GameSetup/GameSetup.h"
|
||||
@ -207,7 +206,6 @@ LibError tex_write(Tex* t, const VfsPath& filename)
|
||||
// write to disk
|
||||
LibError ret = INFO::OK;
|
||||
{
|
||||
(void)da_set_size(&da, round_up(da.cur_size, BLOCK_SIZE));
|
||||
shared_ptr<u8> file = DummySharedPtr(da.base);
|
||||
const ssize_t bytes_written = g_VFS->CreateFile(filename, file, da.pos);
|
||||
if(bytes_written > 0)
|
||||
@ -254,7 +252,8 @@ void WriteScreenshot(const VfsPath& extension)
|
||||
|
||||
const size_t img_size = w * h * bpp/8;
|
||||
const size_t hdr_size = tex_hdr_size(filename);
|
||||
shared_ptr<u8> buf = io_Allocate(hdr_size+img_size);
|
||||
shared_ptr<u8> buf;
|
||||
AllocateAligned(buf, hdr_size+img_size, maxSectorSize);
|
||||
GLvoid* img = buf.get() + hdr_size;
|
||||
Tex t;
|
||||
if(tex_wrap(w, h, bpp, flags, buf, hdr_size, &t) < 0)
|
||||
@ -311,7 +310,8 @@ void WriteBigScreenshot(const VfsPath& extension, int tiles)
|
||||
void* tile_data = malloc(tile_size);
|
||||
if(!tile_data)
|
||||
WARN_ERR_RETURN(ERR::NO_MEM);
|
||||
shared_ptr<u8> img_buf = io_Allocate(hdr_size+img_size);
|
||||
shared_ptr<u8> img_buf;
|
||||
AllocateAligned(img_buf, hdr_size+img_size, maxSectorSize);
|
||||
|
||||
Tex t;
|
||||
GLvoid* img = img_buf.get() + hdr_size;
|
||||
|
@ -23,6 +23,7 @@
|
||||
#include "ps/Filesystem.h"
|
||||
#include "ps/XML/Xeromyces.h"
|
||||
#include "lib/utf8.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "lib/sysdep/cpu.h"
|
||||
#include "maths/Fixed.h"
|
||||
|
||||
@ -95,7 +96,8 @@ bool XMLWriter_File::StoreVFS(const PIVFS& vfs, const VfsPath& pathname)
|
||||
if (m_LastElement) debug_warn(L"ERROR: Saving XML while an element is still open");
|
||||
|
||||
const size_t size = m_Data.length();
|
||||
shared_ptr<u8> data = io_Allocate(size);
|
||||
shared_ptr<u8> data;
|
||||
AllocateAligned(data, size, maxSectorSize);
|
||||
memcpy(data.get(), m_Data.data(), size);
|
||||
LibError ret = vfs->CreateFile(pathname, data, size);
|
||||
if (ret < 0)
|
||||
|
@ -32,6 +32,7 @@
|
||||
|
||||
#include "lib/bits.h" // is_pow2
|
||||
#include "lib/res/graphics/ogl_tex.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "maths/Matrix3D.h"
|
||||
#include "maths/MathUtil.h"
|
||||
#include "ps/CLogger.h"
|
||||
@ -1773,7 +1774,8 @@ int CRenderer::LoadAlphaMaps()
|
||||
size_t tile_w = 2+base+2; // 2 pixel border (avoids bilinear filtering artifacts)
|
||||
size_t total_w = round_up_to_pow2(tile_w * NumAlphaMaps);
|
||||
size_t total_h = base; debug_assert(is_pow2(total_h));
|
||||
shared_ptr<u8> data = io_Allocate(total_w*total_h*3);
|
||||
shared_ptr<u8> data;
|
||||
AllocateAligned(data, total_w*total_h*3, maxSectorSize);
|
||||
// for each tile on row
|
||||
for(size_t i=0;i<NumAlphaMaps;i++)
|
||||
{
|
||||
|
@ -25,6 +25,7 @@
|
||||
#include "graphics/Terrain.h"
|
||||
#include "lib/timer.h"
|
||||
#include "lib/tex/tex.h"
|
||||
#include "lib/allocators/shared_ptr.h"
|
||||
#include "ps/CLogger.h"
|
||||
#include "ps/Filesystem.h"
|
||||
#include "ps/Util.h"
|
||||
@ -130,7 +131,8 @@ private:
|
||||
|
||||
const size_t img_size = w * h * bpp/8;
|
||||
const size_t hdr_size = tex_hdr_size(filename);
|
||||
shared_ptr<u8> buf = io_Allocate(hdr_size+img_size);
|
||||
shared_ptr<u8> buf;
|
||||
AllocateAligned(buf, hdr_size+img_size, maxSectorSize);
|
||||
Tex t;
|
||||
if (tex_wrap(w, h, bpp, flags, buf, hdr_size, &t) < 0)
|
||||
return;
|
||||
|
Loading…
Reference in New Issue
Block a user