major refactor of file/io and alignment code. requires update-workspaces

. completely rewrite waio - use IOCP, add several hardcore
optimizations. now outperforms the
  AS SSD and ATTO benchmarks when writing
. refactor file interface - use LIO_READ instead of 'r', allow access to
file descriptor.
. completely rewrite the IO wrapper. now much more simple, less CPU
overhead, adds
  support for pre-issue/post-completion hooks and preallocation.
  io::Run defaults to simple synchronous IO; use io::Parameters to get
asynchronous.
. add alignment.h with constants and Align() function template (more
efficient than
  round_up for compile-time constants)
. add UniqueRange - similar to C++0x unique_ptr (emulated for C++03),
plus a
  built-in size. avoids expensive thread-safe reference counting in
shared_ptr.

cleanup:
- move fat_time functions into archive_zip
- remove no longer needed io_align and block_cache
- reduce dependencies in sysdep/compiler (move parts to
code_annotation.h)
- move IOCP into separate file (reused by waio)

This was SVN commit r9350.

source/graphics/TextureConverter.cpp

@@ -458,7 +458,8 @@ bool CTextureConverter::Poll(CTexturePtr& texture, VfsPath& dest, bool& ok)
 
 	// Move output into a correctly-aligned buffer
 	size_t size = result->output.buffer.size();
-	shared_ptr<u8> file = io_Allocate(size);
+	shared_ptr<u8> file;
+	AllocateAligned(file, size, maxSectorSize);
 	memcpy(file.get(), &result->output.buffer[0], size);
 	if (m_VFS->CreateFile(result->dest, file, size) < 0)
 	{

source/graphics/tests/test_MeshManager.h

@@ -20,6 +20,7 @@
 #include "lib/file/file_system_util.h"
 #include "lib/file/vfs/vfs.h"
 #include "lib/file/io/io.h"
+#include "lib/allocators/shared_ptr.h"
 
 #include "graphics/ColladaManager.h"
 #include "graphics/MeshManager.h"
@@ -116,7 +117,8 @@ public:
 	{
 		copyFile(srcDAE, testDAE);
 		//buildArchive();
-		shared_ptr<u8> buf = io_Allocate(100);
+		shared_ptr<u8> buf;
+		AllocateAligned(buf, 100, maxSectorSize);
 		strcpy_s((char*)buf.get(), 5, "Test");
 		g_VFS->CreateFile(testDAE, buf, 4);
 	}
@@ -178,7 +180,8 @@ public:
 		TestLogger logger;
 
 		copyFile(srcDAE, testDAE);
-		shared_ptr<u8> buf = io_Allocate(100);
+		shared_ptr<u8> buf;
+		AllocateAligned(buf, 100, maxSectorSize);
 		strcpy_s((char*)buf.get(), 100, "Not valid XML");
 		g_VFS->CreateFile(testSkeletonDefs, buf, 13);
 
@@ -192,7 +195,8 @@ public:
 		TestLogger logger;
 
 		copyFile(srcSkeletonDefs, testSkeletonDefs);
-		shared_ptr<u8> buf = io_Allocate(100);
+		shared_ptr<u8> buf;
+		AllocateAligned(buf, 100, maxSectorSize);
 		strcpy_s((char*)buf.get(), 100, "Not valid XML");
 		g_VFS->CreateFile(testDAE, buf, 13);
 

source/lib/alignment.h

@@ -0,0 +1,105 @@
+#ifndef INCLUDED_ALIGNMENT
+#define INCLUDED_ALIGNMENT
+
+#include "lib/sysdep/compiler.h"	// MSC_VERSION
+
+template<typename T>
+inline bool IsAligned(T t, uintptr_t multiple)
+{
+	return (uintptr_t(t) % multiple) == 0;
+}
+
+template<size_t multiple>
+inline size_t Align(size_t n)
+{
+	cassert(multiple != 0 && ((multiple & (multiple-1)) == 0));	// is power of 2
+	return (n + multiple-1) & ~(multiple-1);
+}
+
+
+//
+// SIMD vector
+//
+
+static const size_t vectorSize = 16;
+
+#define VERIFY_VECTOR_MULTIPLE(size)\
+	VERIFY(IsAligned(size, vectorSize))
+
+#define VERIFY_VECTOR_ALIGNED(pointer)\
+	VERIFY_VECTOR_MULTIPLE(pointer);\
+	ASSUME_ALIGNED(pointer, vectorSize)
+
+
+//
+// CPU cache
+//
+
+static const size_t cacheLineSize = 64;	// (L2)
+
+#if MSC_VERSION
+#define CACHE_ALIGNED __declspec(align(64))	// align() requires a literal; keep in sync with cacheLineSize
+#endif
+
+
+//
+// MMU pages
+//
+
+static const size_t pageSize = 0x1000;	// 4 KB
+static const size_t largePageSize = 0x200000;	// 2 MB
+
+
+// waio opens files with FILE_FLAG_NO_BUFFERING, so Windows requires
+// file offsets / buffers and sizes to be sector-aligned. querying the
+// actual sector size via GetDiskFreeSpace is inconvenient and slow.
+// we always request large blocks anyway, so just check whether inputs
+// are aligned to a `maximum' sector size. this catches common mistakes
+// before they cause scary "IO failed" errors. if the value turns out
+// to be too low, the Windows APIs will still complain.
+static const uintptr_t maxSectorSize = 0x1000;
+
+#endif	// #ifndef INCLUDED_ALIGNMENT
+#ifndef INCLUDED_ALIGNMENT
+#define INCLUDED_ALIGNMENT
+
+template<typename T>
+inline bool IsAligned(T t, uintptr_t multiple)
+{
+	return (uintptr_t(t) % multiple) == 0;
+}
+
+
+//
+// SIMD vector
+//
+
+static const size_t vectorSize = 16;
+
+#define VERIFY_VECTOR_MULTIPLE(size)\
+	VERIFY(IsAligned(size, vectorSize))
+
+#define VERIFY_VECTOR_ALIGNED(pointer)\
+	VERIFY_VECTOR_MULTIPLE(pointer);\
+	ASSUME_ALIGNED(pointer, vectorSize)
+
+
+//
+// CPU cache
+//
+
+static const size_t cacheLineSize = 64;	// (L2)
+
+#if MSC_VERSION
+#define CACHE_ALIGNED __declspec(align(64))	// align() requires a literal; keep in sync with cacheLineSize
+#endif
+
+
+//
+// MMU pages
+//
+
+static const size_t pageSize = 0x1000;	// 4 KB
+static const size_t largePageSize = 0x200000;	// 2 MB
+
+#endif	// #ifndef INCLUDED_ALIGNMENT

source/lib/allocators/allocators.cpp

@@ -27,9 +27,8 @@
 #include "precompiled.h"
 #include "lib/allocators/allocators.h"
 
+#include "lib/alignment.h"
 #include "lib/sysdep/cpu.h"	// cpu_CAS
-#include "lib/bits.h"
-
 #include "lib/allocators/mem_util.h"
 
 
@@ -157,7 +156,7 @@ void single_free(void* storage, volatile intptr_t* in_use_flag, void* p)
 
 void* static_calloc(StaticStorage* ss, size_t size)
 {
-	void* p = (void*)round_up((uintptr_t)ss->pos, (uintptr_t)16u);
+	void* p = (void*)Align<16>((uintptr_t)ss->pos);
 	ss->pos = (u8*)p+size;
 	debug_assert(ss->pos <= ss->end);
 	return p;

source/lib/allocators/mem_util.cpp

@@ -28,6 +28,7 @@
 #include "lib/allocators/mem_util.h"
 
 #include "lib/bits.h"				// round_up
+#include "lib/alignment.h"
 #include "lib/posix/posix_mman.h"
 #include "lib/sysdep/os_cpu.h"			// os_cpu_PageSize
 
@@ -45,8 +46,7 @@ size_t mem_RoundUpToPage(size_t size)
 size_t mem_RoundUpToAlignment(size_t size)
 {
 	// all allocators should align to at least this many bytes:
-	const size_t alignment = 8;
-	return round_up(size, alignment);
+	return Align<8>(size);
 }
 
 

source/lib/allocators/pool.cpp

@@ -29,6 +29,10 @@
 
 #include "lib/allocators/mem_util.h"
 
+#include "lib/timer.h"
+
+TIMER_ADD_CLIENT(tc_pool_alloc);
+
 
 LibError pool_create(Pool* p, size_t max_size, size_t el_size)
 {
@@ -66,6 +70,7 @@ bool pool_contains(const Pool* p, void* el)
 
 void* pool_alloc(Pool* p, size_t size)
 {
+	TIMER_ACCRUE(tc_pool_alloc);
 	// if pool allows variable sizes, go with the size parameter,
 	// otherwise the pool el_size setting.
 	const size_t el_size = p->el_size? p->el_size : mem_RoundUpToAlignment(size);

source/lib/allocators/shared_ptr.h

@@ -23,7 +23,7 @@
 #ifndef INCLUDED_SHARED_PTR
 #define INCLUDED_SHARED_PTR
 
-#include "lib/sysdep/arch/x86_x64/cache.h"
+#include "lib/alignment.h"
 #include "lib/sysdep/rtl.h" // rtl_AllocateAligned
 
 struct DummyDeleter
@@ -61,6 +61,7 @@ struct FreeDeleter
 // (note: uses CheckedArrayDeleter)
 LIB_API shared_ptr<u8> Allocate(size_t size);
 
+
 struct AlignedDeleter
 {
 	template<class T>
@@ -71,9 +72,13 @@ struct AlignedDeleter
 };
 
 template<class T>
-inline shared_ptr<T> AllocateAligned(size_t size)
+static inline LibError AllocateAligned(shared_ptr<T>& p, size_t size, size_t alignment = cacheLineSize)
 {
-	return shared_ptr<T>((T*)rtl_AllocateAligned(size, x86_x64_Caches(L2D)->entrySize), AlignedDeleter());
+	void* mem = rtl_AllocateAligned(size, alignment);
+	if(!mem)
+		WARN_RETURN(ERR::NO_MEM);
+	p.reset((T*)mem, AlignedDeleter());
+	return INFO::OK;
 }
 
 #endif	// #ifndef INCLUDED_SHARED_PTR

source/lib/allocators/unique_range.cpp

@@ -0,0 +1,40 @@
+#include "precompiled.h"
+#include "lib/allocators/unique_range.h"
+
+#include "lib/sysdep/cpu.h"	// cpu_AtomicAdd
+#include "lib/sysdep/rtl.h"	// rtl_FreeAligned
+
+
+static void UniqueRangeDeleterNone(void* UNUSED(pointer), size_t UNUSED(size))
+{
+	// (introducing this do-nothing function avoids having to check whether deleter != 0)
+}
+
+static void UniqueRangeDeleterAligned(void* pointer, size_t UNUSED(size))
+{
+	return rtl_FreeAligned(pointer);
+}
+
+
+static UniqueRangeDeleter deleters[idxDeleterBits+1] = { UniqueRangeDeleterNone, UniqueRangeDeleterAligned };
+
+static IdxDeleter numDeleters = 2;
+
+
+IdxDeleter AddUniqueRangeDeleter(UniqueRangeDeleter deleter)
+{
+	debug_assert(deleter);
+	IdxDeleter idxDeleter = cpu_AtomicAdd(&numDeleters, 1);
+	debug_assert(idxDeleter < (IdxDeleter)ARRAY_SIZE(deleters));
+	deleters[idxDeleter] = deleter;
+	return idxDeleter;
+}
+
+
+void CallUniqueRangeDeleter(void* pointer, size_t size, IdxDeleter idxDeleter) throw()
+{
+	ASSERT(idxDeleter < numDeleters);
+	// (some deleters do not tolerate null pointers)
+	if(pointer)
+		deleters[idxDeleter](pointer, size);
+}

source/lib/allocators/unique_range.h

@@ -0,0 +1,185 @@
+#ifndef INCLUDED_UNIQUE_RANGE
+#define INCLUDED_UNIQUE_RANGE
+
+
+#define ASSERT debug_assert
+
+
+#include "lib/lib_api.h"
+
+// we usually don't hold multiple references to allocations, so unique_ptr
+// can be used instead of the more complex (ICC generated incorrect code on
+// 2 occasions) and expensive shared_ptr.
+// a custom deleter is required because allocators such as ReserveAddressSpace need to
+// pass the size to their deleter. we want to mix pointers from various allocators, but
+// unique_ptr's deleter is fixed at compile-time, so it would need to be general enough
+// to handle all allocators.
+// storing the size and a function pointer would be one such solution, with the added
+// bonus of no longer requiring a complete type at the invocation of ~unique_ptr.
+// however, this inflates the pointer size to 3 words. if only a few allocator types
+// are needed, we can replace the function pointer with an index stashed into the
+// lower bits of the pointer (safe because allocations are always aligned to the
+// word size).
+typedef intptr_t IdxDeleter;
+
+// no-op deleter (use when returning part of an existing allocation)
+// must be zero because reset() sets address (which includes idxDeleter) to zero.
+static const IdxDeleter idxDeleterNone = 0;
+
+static const IdxDeleter idxDeleterAligned = 1;
+
+// (temporary value to prevent concurrent calls to AddUniqueRangeDeleter)
+static const IdxDeleter idxDeleterBusy = -IdxDeleter(1);
+
+// governs the maximum number of IdxDeleter and each pointer's alignment requirements
+static const IdxDeleter idxDeleterBits = 0x7;
+
+typedef void (*UniqueRangeDeleter)(void* pointer, size_t size);
+
+/**
+ * @return the next available IdxDeleter and associate it with the deleter.
+ * halts the program if the idxDeleterBits limit has been reached.
+ *
+ * thread-safe, but no attempt is made to detect whether the deleter has already been
+ * registered (would require a mutex). each allocator must ensure they only call this once.
+ **/
+LIB_API IdxDeleter AddUniqueRangeDeleter(UniqueRangeDeleter deleter);
+
+LIB_API void CallUniqueRangeDeleter(void* pointer, size_t size, IdxDeleter idxDeleter) throw();
+
+
+// unfortunately, unique_ptr allows constructing without a custom deleter. to ensure callers can
+// rely upon pointers being associated with a size, we introduce a `UniqueRange' replacement.
+// its interface is identical to unique_ptr except for the constructors, the addition of
+// size() and the removal of operator bool (which avoids implicit casts to int).
+class UniqueRange
+{
+public:
+	typedef void* pointer;
+	typedef void element_type;
+
+	UniqueRange()
+	{
+		Set(0, 0, idxDeleterNone);
+	}
+
+	UniqueRange(pointer p, size_t size, IdxDeleter deleter)
+	{
+		Set(p, size, deleter);
+	}
+
+	UniqueRange(RVREF(UniqueRange) rvref)
+	{
+		UniqueRange& rhs = LVALUE(rvref);
+		address_ = rhs.address_;
+		size_ = rhs.size_;
+		rhs.address_ = 0;
+	}
+
+	UniqueRange& operator=(RVREF(UniqueRange) rvref)
+	{
+		UniqueRange& rhs = LVALUE(rvref);
+		if(this != &rhs)
+		{
+			Delete();
+			address_ = rhs.address_;
+			size_ = rhs.size_;
+			rhs.address_ = 0;
+		}
+		return *this;
+	}
+
+	~UniqueRange()
+	{
+		Delete();
+	}
+
+	pointer get() const
+	{
+		return pointer(address_ & ~idxDeleterBits);
+	}
+
+	IdxDeleter get_deleter() const
+	{
+		return IdxDeleter(address_ & idxDeleterBits);
+	}
+
+	size_t size() const
+	{
+		return size_;
+	}
+
+	// side effect: subsequent get_deleter will return idxDeleterNone
+	pointer release()	// relinquish ownership
+	{
+		pointer ret = get();
+		address_ = 0;
+		return ret;
+	}
+
+	void reset()
+	{
+		Delete();
+		address_ = 0;
+	}
+
+	void reset(pointer p, size_t size, IdxDeleter deleter)
+	{
+		Delete();
+		Set(p, size, deleter);
+	}
+
+	void swap(UniqueRange& rhs)
+	{
+		std::swap(address_, rhs.address_);
+		std::swap(size_, rhs.size_);
+	}
+
+private:
+	void Set(pointer p, size_t size, IdxDeleter deleter)
+	{
+		ASSERT((uintptr_t(p) & idxDeleterBits) == 0);
+		ASSERT(deleter <= idxDeleterBits);
+
+		address_ = uintptr_t(p) | deleter;
+		size_ = size;
+
+		ASSERT(get() == p);
+		ASSERT(get_deleter() == deleter);
+		ASSERT(this->size() == size);
+	}
+
+	void Delete()
+	{
+		CallUniqueRangeDeleter(get(), size(), get_deleter());
+	}
+
+	// disallow construction and assignment from lvalue
+	UniqueRange(const UniqueRange&);
+	UniqueRange& operator=(const UniqueRange&);
+
+	// (IdxDeleter is stored in the lower bits of address since size might not even be a multiple of 4.)
+	uintptr_t address_;
+	size_t size_;
+};
+
+namespace std {
+
+static inline void swap(UniqueRange& p1, UniqueRange& p2)
+{
+	p1.swap(p2);
+}
+
+static inline void swap(RVREF(UniqueRange) p1, UniqueRange& p2)
+{
+	p2.swap(LVALUE(p1));
+}
+
+static inline void swap(UniqueRange& p1, RVREF(UniqueRange) p2)
+{
+	p1.swap(LVALUE(p2));
+}
+
+}
+
+#endif	// #ifndef INCLUDED_UNIQUE_RANGE

source/lib/bits.h

@@ -230,13 +230,6 @@ inline T round_down(T n, T multiple)
 }
 
 
-template<typename T>
-inline bool IsAligned(T t, uintptr_t multiple)
-{
-	return ((uintptr_t)t % multiple) == 0;
-}
-
-
 template<typename T>
 inline T MaxPowerOfTwoDivisor(T value)
 {

source/lib/code_annotation.h

@@ -28,6 +28,7 @@
 #define INCLUDED_CODE_ANNOTATION
 
 #include "lib/sysdep/compiler.h"
+#include "lib/sysdep/arch.h"	// ARCH_AMD64
 
 /**
  * mark a function local variable or parameter as unused and avoid
@@ -78,6 +79,20 @@
 #define UNREACHABLE	// actually defined below.. this is for
 # undef UNREACHABLE	// CppDoc's benefit only.
 
+// this macro should not generate any fallback code; it is merely the
+// compiler-specific backend for UNREACHABLE.
+// #define it to nothing if the compiler doesn't support such a hint.
+#define HAVE_ASSUME_UNREACHABLE 1
+#if MSC_VERSION && !ICC_VERSION // (ICC ignores this)
+# define ASSUME_UNREACHABLE __assume(0)
+#elif GCC_VERSION >= 450
+# define ASSUME_UNREACHABLE __builtin_unreachable()
+#else
+# define ASSUME_UNREACHABLE
+# undef HAVE_ASSUME_UNREACHABLE
+# define HAVE_ASSUME_UNREACHABLE 0
+#endif
+
 // compiler supports ASSUME_UNREACHABLE => allow it to assume the code is
 // never reached (improves optimization at the cost of undefined behavior
 // if the annotation turns out to be incorrect).
@@ -214,4 +229,150 @@ private:\
 # define COMPILER_FENCE
 #endif
 
+
+// try to define _W64, if not already done
+// (this is useful for catching pointer size bugs)
+#ifndef _W64
+# if MSC_VERSION
+#  define _W64 __w64
+# elif GCC_VERSION
+#  define _W64 __attribute__((mode (__pointer__)))
+# else
+#  define _W64
+# endif
+#endif
+
+
+// C99-like restrict (non-standard in C++, but widely supported in various forms).
+//
+// May be used on pointers. May also be used on member functions to indicate
+// that 'this' is unaliased (e.g. "void C::m() RESTRICT { ... }").
+// Must not be used on references - GCC supports that but VC doesn't.
+//
+// We call this "RESTRICT" to avoid conflicts with VC's __declspec(restrict),
+// and because it's not really the same as C99's restrict.
+//
+// To be safe and satisfy the compilers' stated requirements: an object accessed
+// by a restricted pointer must not be accessed by any other pointer within the
+// lifetime of the restricted pointer, if the object is modified.
+// To maximise the chance of optimisation, any pointers that could potentially
+// alias with the restricted one should be marked as restricted too.
+//
+// It would probably be a good idea to write test cases for any code that uses
+// this in an even very slightly unclear way, in case it causes obscure problems
+// in a rare compiler due to differing semantics.
+//
+// .. GCC
+#if GCC_VERSION
+# define RESTRICT __restrict__
+// .. VC8 provides __restrict
+#elif MSC_VERSION >= 1400
+# define RESTRICT __restrict
+// .. ICC supports the keyword 'restrict' when run with the /Qrestrict option,
+//    but it always also supports __restrict__ or __restrict to be compatible
+//    with GCC/MSVC, so we'll use the underscored version. One of {GCC,MSC}_VERSION
+//    should have been defined in addition to ICC_VERSION, so we should be using
+//    one of the above cases (unless it's an old VS7.1-emulating ICC).
+#elif ICC_VERSION
+# error ICC_VERSION defined without either GCC_VERSION or an adequate MSC_VERSION
+// .. unsupported; remove it from code
+#else
+# define RESTRICT
+#endif
+
+
+// C99-style __func__
+// .. newer GCC already have it
+#if GCC_VERSION >= 300
+// nothing need be done
+// .. old GCC and MSVC have __FUNCTION__
+#elif GCC_VERSION >= 200 || MSC_VERSION
+# define __func__ __FUNCTION__
+// .. unsupported
+#else
+# define __func__ "(unknown)"
+#endif
+
+
+// extern "C", but does the right thing in pure-C mode
+#if defined(__cplusplus)
+# define EXTERN_C extern "C"
+#else
+# define EXTERN_C extern
+#endif
+
+
+#if MSC_VERSION
+# define INLINE __forceinline
+#else
+# define INLINE inline
+#endif
+
+
+#if MSC_VERSION
+# define CALL_CONV __cdecl
+#else
+# define CALL_CONV
+#endif
+
+
+#if MSC_VERSION && !ARCH_AMD64
+# define DECORATED_NAME(name) _##name
+#else
+# define DECORATED_NAME(name) name
+#endif
+
+
+// workaround for preprocessor limitation: macro args aren't expanded
+// before being pasted.
+#define STRINGIZE2(id) # id
+#define STRINGIZE(id) STRINGIZE2(id)
+
+
+//-----------------------------------------------------------------------------
+// partial emulation of C++0x rvalue references (required for UniqueRange)
+
+#if HAVE_CPP0X
+
+#define RVREF(T) T&&	// the type of an rvalue reference
+#define LVALUE(rvalue) rvalue	// (a named rvalue reference is an lvalue)
+#define RVALUE(lvalue) std::move(lvalue)
+#define RVALUE_FROM_R(rvalue) RVALUE(rvalue)	// (see above)
+
+#else
+
+// RVALUE wraps an lvalue reference in this class for later use by a
+// "move ctor" that takes an RValue.
+template<typename T>
+class RValue
+{
+public:
+	explicit RValue(T& lvalue): lvalue(lvalue) {}
+	T& LValue() const { return lvalue; }
+
+private:
+	T& lvalue;
+};
+
+// from rvalue or const lvalue
+template<class T>
+static inline RValue<T> ToRValue(const T& t)
+{
+	return RValue<T>((T&)t);
+}
+
+// from lvalue
+template<class T>
+static inline RValue<T> ToRValue(T& t)
+{
+	return RValue<T>(t);
+}
+
+#define RVREF(T) const RValue<T>&	// the type of an rvalue reference
+#define LVALUE(rvalue) rvalue.LValue()
+#define RVALUE(lvalue) ToRValue(lvalue)
+#define RVALUE_FROM_R(rvalue) rvalue
+
+#endif	// #if !HAVE_CPP0X
+
 #endif	// #ifndef INCLUDED_CODE_ANNOTATION

source/lib/fat_time.cpp

@@ -1,77 +0,0 @@
-/* Copyright (c) 2010 Wildfire Games
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * timestamp conversion: DOS FAT <-> Unix time_t
- */
-
-#include "precompiled.h"
-#include "lib/fat_time.h"
-
-#include <ctime>
-
-#include "lib/bits.h"
-
-
-time_t time_t_from_FAT(u32 fat_timedate)
-{
-	const u32 fat_time = bits(fat_timedate, 0, 15);
-	const u32 fat_date = bits(fat_timedate, 16, 31);
-
-	struct tm t;							// struct tm format:
-	t.tm_sec   = bits(fat_time, 0,4) * 2;	// [0,59]
-	t.tm_min   = bits(fat_time, 5,10);		// [0,59]
-	t.tm_hour  = bits(fat_time, 11,15);		// [0,23]
-	t.tm_mday  = bits(fat_date, 0,4);		// [1,31]
-	t.tm_mon   = bits(fat_date, 5,8) - 1;	// [0,11]
-	t.tm_year  = bits(fat_date, 9,15) + 80;	// since 1900
-	t.tm_isdst = -1;	// unknown - let libc determine
-
-	// otherwise: totally bogus, and at the limit of 32-bit time_t
-	debug_assert(t.tm_year < 138);
-
-	time_t ret = mktime(&t);
-	debug_assert(ret != (time_t)-1);	// mktime shouldn't fail
-	return ret;
-}
-
-
-u32 FAT_from_time_t(time_t time)
-{
-	// (values are adjusted for DST)
-	struct tm* t = localtime(&time);
-
-	const u16 fat_time = u16(
-		(t->tm_sec/2) |		    // 5
-		(u16(t->tm_min) << 5) | // 6
-		(u16(t->tm_hour) << 11)	// 5
-	);
-
-	const u16 fat_date = u16(
-		(t->tm_mday) |            // 5
-		(u16(t->tm_mon+1) << 5) | // 4
-		(u16(t->tm_year-80) << 9) // 7
-	);
-
-	u32 fat_timedate = u32_from_u16(fat_date, fat_time);
-	return fat_timedate;
-}

source/lib/fat_time.h

@@ -1,29 +0,0 @@
-/* Copyright (c) 2010 Wildfire Games
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef INCLUDED_FAT_TIME
-#define INCLUDED_FAT_TIME
-
-extern time_t time_t_from_FAT(u32 fat_timedate);
-extern u32 FAT_from_time_t(time_t time);
-
-#endif // INCLUDED_FAT_TIME

source/lib/file/archive/archive_zip.cpp

@@ -33,7 +33,6 @@
 #include "lib/utf8.h"
 #include "lib/bits.h"
 #include "lib/byte_order.h"
-#include "lib/fat_time.h"
 #include "lib/allocators/pool.h"
 #include "lib/sysdep/filesystem.h"
 #include "lib/file/archive/archive.h"
@@ -41,8 +40,54 @@
 #include "lib/file/archive/stream.h"
 #include "lib/file/file.h"
 #include "lib/file/io/io.h"
-#include "lib/file/io/io_align.h"	// BLOCK_SIZE
-#include "lib/file/io/write_buffer.h"
+
+//-----------------------------------------------------------------------------
+// timestamp conversion: DOS FAT <-> Unix time_t
+//-----------------------------------------------------------------------------
+
+static time_t time_t_from_FAT(u32 fat_timedate)
+{
+	const u32 fat_time = bits(fat_timedate, 0, 15);
+	const u32 fat_date = bits(fat_timedate, 16, 31);
+
+	struct tm t;							// struct tm format:
+	t.tm_sec   = bits(fat_time, 0,4) * 2;	// [0,59]
+	t.tm_min   = bits(fat_time, 5,10);		// [0,59]
+	t.tm_hour  = bits(fat_time, 11,15);		// [0,23]
+	t.tm_mday  = bits(fat_date, 0,4);		// [1,31]
+	t.tm_mon   = bits(fat_date, 5,8) - 1;	// [0,11]
+	t.tm_year  = bits(fat_date, 9,15) + 80;	// since 1900
+	t.tm_isdst = -1;	// unknown - let libc determine
+
+	// otherwise: totally bogus, and at the limit of 32-bit time_t
+	debug_assert(t.tm_year < 138);
+
+	time_t ret = mktime(&t);
+	debug_assert(ret != (time_t)-1);	// mktime shouldn't fail
+	return ret;
+}
+
+
+static u32 FAT_from_time_t(time_t time)
+{
+	// (values are adjusted for DST)
+	struct tm* t = localtime(&time);
+
+	const u16 fat_time = u16(
+		(t->tm_sec/2) |		    // 5
+		(u16(t->tm_min) << 5) | // 6
+		(u16(t->tm_hour) << 11)	// 5
+		);
+
+	const u16 fat_date = u16(
+		(t->tm_mday) |            // 5
+		(u16(t->tm_mon+1) << 5) | // 4
+		(u16(t->tm_year-80) << 9) // 7
+		);
+
+	u32 fat_timedate = u32_from_u16(fat_date, fat_time);
+	return fat_timedate;
+}
 
 
 //-----------------------------------------------------------------------------
@@ -288,7 +333,8 @@ public:
 
 		Stream stream(codec);
 		stream.SetOutputBuffer(buf.get(), size);
-		RETURN_ERR(io_Scan(m_file, m_ofs, m_csize, FeedStream, (uintptr_t)&stream));
+		io::Operation op(*m_file.get(), 0, m_csize, m_ofs);
+		RETURN_ERR(io::Run(op, io::Parameters(), std::bind(&Stream::Feed, &stream, std::placeholders::_1, std::placeholders::_2)));
 		RETURN_ERR(stream.Finish());
 #if CODEC_COMPUTE_CHECKSUM
 		debug_assert(m_checksum == stream.Checksum());
@@ -315,29 +361,32 @@ private:
 
 	struct LFH_Copier
 	{
-		u8* lfh_dst;
-		size_t lfh_bytes_remaining;
+		LFH_Copier(u8* lfh_dst, size_t lfh_bytes_remaining)
+			: lfh_dst(lfh_dst), lfh_bytes_remaining(lfh_bytes_remaining)
+		{
+		}
+
+		// this code grabs an LFH struct from file block(s) that are
+		// passed to the callback. usually, one call copies the whole thing,
+		// but the LFH may straddle a block boundary.
+		//
+		// rationale: this allows using temp buffers for zip_fixup_lfh,
+		// which avoids involving the file buffer manager and thus
+		// avoids cluttering the trace and cache contents.
+		LibError operator()(const u8* block, size_t size) const
+		{
+			debug_assert(size <= lfh_bytes_remaining);
+			memcpy(lfh_dst, block, size);
+			lfh_dst += size;
+			lfh_bytes_remaining -= size;
+
+			return INFO::CB_CONTINUE;
+		}
+
+		mutable u8* lfh_dst;
+		mutable size_t lfh_bytes_remaining;
 	};
 
-	// this code grabs an LFH struct from file block(s) that are
-	// passed to the callback. usually, one call copies the whole thing,
-	// but the LFH may straddle a block boundary.
-	//
-	// rationale: this allows using temp buffers for zip_fixup_lfh,
-	// which avoids involving the file buffer manager and thus
-	// avoids cluttering the trace and cache contents.
-	static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size)
-	{
-		LFH_Copier* p = (LFH_Copier*)cbData;
-
-		debug_assert(size <= p->lfh_bytes_remaining);
-		memcpy(p->lfh_dst, block, size);
-		p->lfh_dst += size;
-		p->lfh_bytes_remaining -= size;
-
-		return INFO::CB_CONTINUE;
-	}
-
 	/**
 	 * fix up m_ofs (adjust it to point to cdata instead of the LFH).
 	 *
@@ -358,8 +407,8 @@ private:
 		// only in the block cache if the file starts in the same block as a
 		// previously read file (i.e. both are small).
 		LFH lfh;
-		LFH_Copier params = { (u8*)&lfh, sizeof(LFH) };
-		if(io_Scan(m_file, m_ofs, sizeof(LFH), lfh_copier_cb, (uintptr_t)&params) == INFO::OK)
+		io::Operation op(*m_file.get(), 0, sizeof(LFH), m_ofs);
+		if(io::Run(op, io::Parameters(), LFH_Copier((u8*)&lfh, sizeof(LFH))) == INFO::OK)
 			m_ofs += (off_t)lfh.Size();
 	}
 
@@ -382,7 +431,7 @@ class ArchiveReader_Zip : public IArchiveReader
 {
 public:
 	ArchiveReader_Zip(const OsPath& pathname)
-		: m_file(new File(pathname, 'r'))
+		: m_file(new File(pathname, LIO_READ))
 	{
 		FileInfo fileInfo;
 		GetFileInfo(pathname, &fileInfo);
@@ -398,16 +447,17 @@ public:
 		size_t cd_numEntries = 0;
 		size_t cd_size = 0;
 		RETURN_ERR(LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size));
-		shared_ptr<u8> buf = io_Allocate(cd_size, cd_ofs);
-		u8* cd;
-		RETURN_ERR(io_Read(m_file, cd_ofs, buf.get(), cd_size, cd));
+		UniqueRange buf(io::Allocate(cd_size));
+
+		io::Operation op(*m_file.get(), buf.get(), cd_size, cd_ofs);
+		RETURN_ERR(io::Run(op));
 
 		// iterate over Central Directory
-		const u8* pos = cd;
+		const u8* pos = (const u8*)buf.get();
 		for(size_t i = 0; i < cd_numEntries; i++)
 		{
 			// scan for next CDFH
-			CDFH* cdfh = (CDFH*)FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH));
+			CDFH* cdfh = (CDFH*)FindRecord((const u8*)buf.get(), cd_size, pos, cdfh_magic, sizeof(CDFH));
 			if(!cdfh)
 				WARN_RETURN(ERR::CORRUPTED);
 
@@ -467,11 +517,11 @@ private:
 
 		// read desired chunk of file into memory
 		const off_t ofs = fileSize - off_t(scanSize);
-		u8* data;
-		RETURN_ERR(io_Read(file, ofs, buf, scanSize, data));
+		io::Operation op(*file.get(), buf, scanSize, ofs);
+		RETURN_ERR(io::Run(op));
 
 		// look for ECDR in buffer
-		const ECDR* ecdr = (const ECDR*)FindRecord(data, scanSize, data, ecdr_magic, sizeof(ECDR));
+		const ECDR* ecdr = (const ECDR*)FindRecord(buf, scanSize, buf, ecdr_magic, sizeof(ECDR));
 		if(!ecdr)
 			return INFO::CANNOT_HANDLE;
 
@@ -482,19 +532,20 @@ private:
 	static LibError LocateCentralDirectory(const PFile& file, off_t fileSize, off_t& cd_ofs, size_t& cd_numEntries, size_t& cd_size)
 	{
 		const size_t maxScanSize = 66000u;	// see below
-		shared_ptr<u8> buf = io_Allocate(maxScanSize, BLOCK_SIZE-1);	// assume worst-case for alignment
+		UniqueRange buf(io::Allocate(maxScanSize));
 
 		// expected case: ECDR at EOF; no file comment
-		LibError ret = ScanForEcdr(file, fileSize, const_cast<u8*>(buf.get()), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size);
+		LibError ret = ScanForEcdr(file, fileSize, (u8*)buf.get(), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size);
 		if(ret == INFO::OK)
 			return INFO::OK;
 		// worst case: ECDR precedes 64 KiB of file comment
-		ret = ScanForEcdr(file, fileSize, const_cast<u8*>(buf.get()), maxScanSize, cd_numEntries, cd_ofs, cd_size);
+		ret = ScanForEcdr(file, fileSize, (u8*)buf.get(), maxScanSize, cd_numEntries, cd_ofs, cd_size);
 		if(ret == INFO::OK)
 			return INFO::OK;
 
 		// both ECDR scans failed - this is not a valid Zip file.
-		RETURN_ERR(io_ReadAligned(file, 0, const_cast<u8*>(buf.get()), sizeof(LFH)));
+		io::Operation op(*file.get(), buf.get(), sizeof(LFH));
+		RETURN_ERR(io::Run(op));
 		// the Zip file has an LFH but lacks an ECDR. this can happen if
 		// the user hard-exits while an archive is being written.
 		// notes:
@@ -503,7 +554,7 @@ private:
 		//   because it'd be slow.
 		// - do not warn - the corrupt archive will be deleted on next
 		//   successful archive builder run anyway.
-		if(FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH)))
+		if(FindRecord((const u8*)buf.get(), sizeof(LFH), (const u8*)buf.get(), lfh_magic, sizeof(LFH)))
 			return ERR::CORRUPTED;	// NOWARN
 		// totally bogus
 		else
@@ -528,8 +579,7 @@ class ArchiveWriter_Zip : public IArchiveWriter
 {
 public:
 	ArchiveWriter_Zip(const OsPath& archivePathname, bool noDeflate)
-		: m_file(new File(archivePathname, 'w')), m_fileSize(0)
-		, m_unalignedWriter(new UnalignedWriter(m_file, 0))
+		: m_file(new File(archivePathname, LIO_WRITE)), m_fileSize(0)
 		, m_numEntries(0), m_noDeflate(noDeflate)
 	{
 		THROW_ERR(pool_create(&m_cdfhPool, 10*MiB, 0));
@@ -546,19 +596,9 @@ public:
 		const off_t cd_ofs = m_fileSize;
 		ecdr->Init(m_numEntries, cd_ofs, cd_size);
 
-		m_unalignedWriter->Append(m_cdfhPool.da.base, cd_size+sizeof(ECDR));
-		m_unalignedWriter->Flush();
-		m_unalignedWriter.reset();
+		write(m_file->Descriptor(), m_cdfhPool.da.base, cd_size+sizeof(ECDR));
 
 		(void)pool_destroy(&m_cdfhPool);
-
-		const OsPath pathname = m_file->Pathname();	// (must be retrieved before resetting m_file)
-		m_file.reset();
-
-		m_fileSize += off_t(cd_size+sizeof(ECDR));
-
-		// remove padding added by UnalignedWriter
-		wtruncate(pathname, m_fileSize);
 	}
 
 	LibError AddFile(const OsPath& pathname, const OsPath& pathnameInArchive)
@@ -578,7 +618,7 @@ public:
 			return INFO::SKIPPED;
 
 		PFile file(new File);
-		RETURN_ERR(file->Open(pathname, 'r'));
+		RETURN_ERR(file->Open(pathname, LIO_READ));
 
 		const size_t pathnameLength = pathnameInArchive.string().length();
 
@@ -598,7 +638,7 @@ public:
 
 		// allocate memory
 		const size_t csizeMax = codec->MaxOutputSize(size_t(usize));
-		shared_ptr<u8> buf = io_Allocate(sizeof(LFH) + pathnameLength + csizeMax);
+		UniqueRange buf(io::Allocate(sizeof(LFH) + pathnameLength + csizeMax));
 
 		// read and compress file contents
 		size_t csize; u32 checksum;
@@ -606,7 +646,8 @@ public:
 			u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength;
 			Stream stream(codec);
 			stream.SetOutputBuffer(cdata, csizeMax);
-			RETURN_ERR(io_Scan(file, 0, usize, FeedStream, (uintptr_t)&stream));
+			io::Operation op(*file.get(), 0, usize);
+			RETURN_ERR(io::Run(op, io::Parameters(), std::bind(&Stream::Feed, &stream, std::placeholders::_1, std::placeholders::_2)));
 			RETURN_ERR(stream.Finish());
 			csize = stream.OutSize();
 			checksum = stream.Checksum();
@@ -631,7 +672,8 @@ public:
 
 		// write LFH, pathname and cdata to file
 		const size_t packageSize = sizeof(LFH) + pathnameLength + csize;
-		RETURN_ERR(m_unalignedWriter->Append(buf.get(), packageSize));
+		if(write(m_file->Descriptor(), buf.get(), packageSize) < 0)
+			WARN_RETURN(ERR::IO);
 		m_fileSize += (off_t)packageSize;
 
 		return INFO::OK;
@@ -661,7 +703,6 @@ private:
 
 	PFile m_file;
 	off_t m_fileSize;
-	PUnalignedWriter m_unalignedWriter;
 
 	Pool m_cdfhPool;
 	size_t m_numEntries;

source/lib/file/archive/stream.cpp

@@ -136,12 +136,3 @@ LibError Stream::Finish()
 	m_outProduced += outProduced;
 	return INFO::OK;
 }
-
-
-LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize)
-{
-//	TIMER_ACCRUE(tc_stream);
-
-	Stream& stream = *(Stream*)cbData;
-	return stream.Feed(in, inSize);
-}

source/lib/file/archive/stream.h

@@ -110,6 +110,4 @@ private:
 	u32 m_checksum;
 };
 
-extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize);
-
 #endif	// #ifndef INCLUDED_STREAM

source/lib/file/common/file_stats.cpp

@@ -189,23 +189,13 @@ ScopedIoMonitor::~ScopedIoMonitor()
 	timer_reset(&m_startTime);
 }
 
-void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, wchar_t mode, off_t size)
+void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, int opcode, off_t size)
 {
 	debug_assert(fi < FI_MAX_IDX);
-	debug_assert(mode == 'r' || mode == 'w');
-	const FileOp op = (mode == 'r')? FO_READ : FO_WRITE;
+	debug_assert(opcode == LIO_READ || opcode == LIO_WRITE);
 
-	io_actual_size_total[fi][op] += size;
-	io_elapsed_time[fi][op] += timer_reset(&m_startTime);
-}
-
-void stats_io_check_seek(BlockId& blockId)
-{
-	static BlockId lastBlockId;
-
-	if(blockId != lastBlockId)
-		io_seeks++;
-	lastBlockId = blockId;
+	io_actual_size_total[fi][opcode == LIO_WRITE] += size;
+	io_elapsed_time[fi][opcode == LIO_WRITE] += timer_reset(&m_startTime);
 }
 
 
@@ -320,11 +310,11 @@ void file_stats_dump()
 		L"Average size = %g KB; seeks: %lu; total callback time: %g ms\n"
 		L"Total data actually read from disk = %g MB\n",
 		(unsigned long)user_ios, user_io_size_total/MB,
-#define THROUGHPUT(impl, op) (io_elapsed_time[impl][op] == 0.0)? 0.0 : (io_actual_size_total[impl][op] / io_elapsed_time[impl][op] / MB)
-		THROUGHPUT(FI_LOWIO, FO_READ), THROUGHPUT(FI_LOWIO, FO_WRITE),
-		THROUGHPUT(FI_AIO  , FO_READ), THROUGHPUT(FI_AIO  , FO_WRITE),
+#define THROUGHPUT(impl, opcode) (io_elapsed_time[impl][opcode == LIO_WRITE] == 0.0)? 0.0 : (io_actual_size_total[impl][opcode == LIO_WRITE] / io_elapsed_time[impl][opcode == LIO_WRITE] / MB)
+		THROUGHPUT(FI_LOWIO, LIO_READ), THROUGHPUT(FI_LOWIO, LIO_WRITE),
+		THROUGHPUT(FI_AIO  , LIO_READ), THROUGHPUT(FI_AIO  , LIO_WRITE),
 		user_io_size_total/user_ios/KB, (unsigned long)io_seeks, io_process_time_total/ms,
-		(io_actual_size_total[FI_LOWIO][FO_READ]+io_actual_size_total[FI_AIO][FO_READ])/MB
+		(io_actual_size_total[FI_LOWIO][0]+io_actual_size_total[FI_AIO][0])/MB
 	);
 
 	debug_printf(

source/lib/file/common/file_stats.h

@@ -27,15 +27,14 @@
 #ifndef INCLUDED_FILE_STATS
 #define INCLUDED_FILE_STATS
 
+#include "lib/posix/posix_aio.h"	// LIO_READ, LIO_WRITE
+
 #define FILE_STATS_ENABLED 0
 
 
 enum FileIOImplentation { FI_LOWIO, FI_AIO, FI_BCACHE, FI_MAX_IDX };
-enum FileOp { FO_READ, FO_WRITE };
 enum CacheRet { CR_HIT, CR_MISS };
 
-#include "lib/file/io/block_cache.h"	// BlockId
-
 #if FILE_STATS_ENABLED
 
 // vfs
@@ -69,13 +68,12 @@ class ScopedIoMonitor
 public:
 	ScopedIoMonitor();
 	~ScopedIoMonitor();
-	void NotifyOfSuccess(FileIOImplentation fi, wchar_t mode, off_t size);
+	void NotifyOfSuccess(FileIOImplentation fi, int opcode, off_t size);
 
 private:
 	double m_startTime;
 };
 
-extern void stats_io_check_seek(BlockId& blockId);
 extern void stats_cb_start();
 extern void stats_cb_finish();
 
@@ -106,9 +104,8 @@ class ScopedIoMonitor
 public:
 	ScopedIoMonitor() {}
 	~ScopedIoMonitor() {}
-	void NotifyOfSuccess(FileIOImplentation UNUSED(fi), wchar_t UNUSED(mode), off_t UNUSED(size)) {}
+	void NotifyOfSuccess(FileIOImplentation UNUSED(fi), int UNUSED(opcode), off_t UNUSED(size)) {}
 };
-#define stats_io_check_seek(blockId)
 #define stats_cb_start()
 #define stats_cb_finish()
 #define stats_cache(cr, size)

source/lib/file/common/real_directory.cpp

@@ -48,33 +48,13 @@ RealDirectory::RealDirectory(const OsPath& path, size_t priority, size_t flags)
 
 /*virtual*/ LibError RealDirectory::Load(const OsPath& name, const shared_ptr<u8>& buf, size_t size) const
 {
-	const OsPath pathname = m_path / name;
-
-	PFile file(new File);
-	RETURN_ERR(file->Open(pathname, 'r'));
-
-	RETURN_ERR(io_ReadAligned(file, 0, buf.get(), size));
-	return INFO::OK;
+	return io::Load(m_path / name, buf.get(), size);
 }
 
 
 LibError RealDirectory::Store(const OsPath& name, const shared_ptr<u8>& fileContents, size_t size)
 {
-	const OsPath pathname = m_path / name;
-
-	{
-		PFile file(new File);
-		RETURN_ERR(file->Open(pathname, 'w'));
-		RETURN_ERR(io_WriteAligned(file, 0, fileContents.get(), size));
-	}
-
-	// io_WriteAligned pads the file; we need to truncate it to the actual
-	// length. ftruncate can't be used because Windows' FILE_FLAG_NO_BUFFERING
-	// only allows resizing to sector boundaries, so the file must first
-	// be closed.
-	wtruncate(pathname, size);
-
-	return INFO::OK;
+	return io::Store(m_path / name, fileContents.get(), size);
 }
 
 

source/lib/file/common/trace.cpp

@@ -31,7 +31,6 @@
 #include <sstream>
 
 #include "lib/allocators/pool.h"
-#include "lib/bits.h"	// round_up
 #include "lib/timer.h"	// timer_Time
 #include "lib/sysdep/sysdep.h"	// sys_OpenFile
 

source/lib/file/file.cpp

@@ -27,34 +27,27 @@
 #include "precompiled.h"
 #include "lib/file/file.h"
 
-#include "lib/config2.h"
 #include "lib/sysdep/filesystem.h"	// O_*, S_*
-#include "lib/posix/posix_aio.h"
 #include "lib/file/common/file_stats.h"
 
 
 ERROR_ASSOCIATE(ERR::FILE_ACCESS, L"Insufficient access rights to open file", EACCES);
-ERROR_ASSOCIATE(ERR::IO, L"Error during IO", EIO);
 
 
-namespace FileImpl {
-
-LibError Open(const OsPath& pathname, wchar_t accessType, int& fd)
+LibError FileOpen(const OsPath& pathname, int opcode, int& fd)
 {
 	int oflag = 0;
-	switch(accessType)
+	switch(opcode)
 	{
-	case 'r':
+	case LIO_READ:
 		oflag = O_RDONLY;
 		break;
-	case 'w':
+	case LIO_WRITE:
 		oflag = O_WRONLY|O_CREAT|O_TRUNC;
 		break;
-	case '+':
-		oflag = O_RDWR;
-		break;
 	default:
 		debug_assert(0);
+		break;
 	}
 #if OS_WIN
 	oflag |= O_BINARY_NP;
@@ -71,7 +64,7 @@ LibError Open(const OsPath& pathname, wchar_t accessType, int& fd)
 }
 
 
-void Close(int& fd)
+void FileClose(int& fd)
 {
 	if(fd >= 0)
 	{
@@ -79,88 +72,3 @@ void Close(int& fd)
 		fd = -1;
 	}
 }
-
-
-LibError IO(int fd, wchar_t accessType, off_t ofs, u8* buf, size_t size)
-{
-	debug_assert(accessType == 'r' || accessType == 'w');
-
-	ScopedIoMonitor monitor;
-
-	lseek(fd, ofs, SEEK_SET);
-
-	errno = 0;
-	const ssize_t ret = (accessType == 'w')? write(fd, buf, size) : read(fd, buf, size);
-	if(ret < 0)
-		return LibError_from_errno();
-
-	const size_t totalTransferred = (size_t)ret;
-#if CONFIG2_FILE_ENABLE_AIO
-	// we won't be called from Issue, i.e. size is always the exact
-	// value without padding and can be checked.
-	if(totalTransferred != size)
-		WARN_RETURN(ERR::IO);
-#endif
-
-	monitor.NotifyOfSuccess(FI_LOWIO, accessType, totalTransferred);
-	return INFO::OK;
-}
-
-
-LibError Issue(aiocb& req, int fd, wchar_t accessType, off_t alignedOfs, u8* alignedBuf, size_t alignedSize)
-{
-	memset(&req, 0, sizeof(req));
-	req.aio_lio_opcode = (accessType == 'w')? LIO_WRITE : LIO_READ;
-	req.aio_buf        = (volatile void*)alignedBuf;
-	req.aio_fildes     = fd;
-	req.aio_offset     = alignedOfs;
-	req.aio_nbytes     = alignedSize;
-#if CONFIG2_FILE_ENABLE_AIO
-	struct sigevent* sig = 0;	// no notification signal
-	aiocb* const reqs = &req;
-	if(lio_listio(LIO_NOWAIT, &reqs, 1, sig) != 0)
-		return LibError_from_errno();
-	return INFO::OK;
-#else
-	// quick and dirty workaround (see CONFIG2_FILE_ENABLE_AIO)
-	return IO(fd, accessType, alignedOfs, alignedBuf, alignedSize);
-#endif
-}
-
-
-LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize)
-{
-#if CONFIG2_FILE_ENABLE_AIO
-	const int err = aio_error(&req);
-	if(err == EINPROGRESS)
-	{
-SUSPEND_AGAIN:
-		aiocb* const reqs = &req;
-		errno = 0;
-		const int ret = aio_suspend(&reqs, 1, (timespec*)0);	// no timeout
-		if(ret != 0)
-		{
-			if(errno == EINTR) // interrupted by signal
-				goto SUSPEND_AGAIN;
-			return LibError_from_errno();
-		}
-	}
-	else if(err != 0)
-	{
-		errno = err;
-		return LibError_from_errno();
-	}
-
-	const ssize_t bytesTransferred = aio_return(&req);
-	if(bytesTransferred == -1)	// transfer failed
-		WARN_RETURN(ERR::IO);
-
-	alignedSize = bytesTransferred;
-#else
-	alignedSize = req.aio_nbytes;
-#endif
-	alignedBuf = (u8*)req.aio_buf;	// cast from volatile void*
-	return INFO::OK;
-}
-
-}	// namespace FileImpl

source/lib/file/file.h

@@ -27,50 +27,28 @@
 #ifndef INCLUDED_FILE
 #define INCLUDED_FILE
 
-struct aiocb;
-
 #include "lib/os_path.h"
+#include "lib/posix/posix_aio.h"	// opcode: LIO_READ or LIO_WRITE
 
 namespace ERR
 {
 	const LibError FILE_ACCESS = -110300;
-	const LibError IO          = -110301;
-}
-
-namespace FileImpl
-{
-	LIB_API LibError Open(const OsPath& pathname, wchar_t mode, int& fd);
-	LIB_API void Close(int& fd);
-	LIB_API LibError IO(int fd, wchar_t mode, off_t ofs, u8* buf, size_t size);
-	LIB_API LibError Issue(aiocb& req, int fd, wchar_t mode, off_t alignedOfs, u8* alignedBuf, size_t alignedSize);
-	LIB_API LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize);
 }
 
+LIB_API LibError FileOpen(const OsPath& pathname, int opcode, int& fd);
+LIB_API void FileClose(int& fd);
 
 class File
 {
 public:
 	File()
-		: m_pathname(), m_fd(0)
+		: pathname(), fd(-1)
 	{
 	}
 
-	LibError Open(const OsPath& pathname, wchar_t mode)
+	File(const OsPath& pathname, int opcode)
 	{
-		RETURN_ERR(FileImpl::Open(pathname, mode, m_fd));
-		m_pathname = pathname;
-		m_mode = mode;
-		return INFO::OK;
-	}
-
-	void Close()
-	{
-		FileImpl::Close(m_fd);
-	}
-
-	File(const OsPath& pathname, wchar_t mode)
-	{
-		(void)Open(pathname, mode);
+		(void)Open(pathname, opcode);
 	}
 
 	~File()
@@ -78,35 +56,38 @@ public:
 		Close();
 	}
 
+	LibError Open(const OsPath& pathname, int opcode)
+	{
+		RETURN_ERR(FileOpen(pathname, opcode, fd));
+		this->pathname = pathname;
+		this->opcode = opcode;
+		return INFO::OK;
+	}
+
+	void Close()
+	{
+		FileClose(fd);
+	}
+
 	const OsPath& Pathname() const
 	{
-		return m_pathname;
+		return pathname;
 	}
 
-	wchar_t Mode() const
+	int Descriptor() const
 	{
-		return m_mode;
+		return fd;
 	}
 
-	LibError Issue(aiocb& req, wchar_t mode, off_t alignedOfs, u8* alignedBuf, size_t alignedSize) const
+	int Opcode() const
 	{
-		return FileImpl::Issue(req, m_fd, mode, alignedOfs, alignedBuf, alignedSize);
-	}
-
-	LibError Write(off_t ofs, const u8* buf, size_t size)
-	{
-		return FileImpl::IO(m_fd, 'w', ofs, const_cast<u8*>(buf), size);
-	}
-
-	LibError Read(off_t ofs, u8* buf, size_t size) const
-	{
-		return FileImpl::IO(m_fd, 'r', ofs, buf, size);
+		return opcode;
 	}
 
 private:
-	OsPath m_pathname;
-	int m_fd;
-	wchar_t m_mode;
+	OsPath pathname;
+	int fd;
+	int opcode;
 };
 
 typedef shared_ptr<File> PFile;

source/lib/file/io/block_cache.cpp

@@ -1,163 +0,0 @@
-/* Copyright (c) 2010 Wildfire Games
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * cache for aligned I/O blocks.
- */
-
-#include "precompiled.h"
-#include "lib/file/io/block_cache.h"
-
-#include "lib/config2.h"	// CONFIG2_CACHE_READ_ONLY
-#include "lib/posix/posix_mman.h"	// mprotect
-#include "lib/file/common/file_stats.h"
-#include "lib/lockfree.h"
-#include "lib/allocators/pool.h"
-#include "lib/fnv_hash.h"
-#include "lib/file/io/io_align.h"
-
-
-//-----------------------------------------------------------------------------
-
-BlockId::BlockId()
-	: m_id(0)
-{
-}
-
-BlockId::BlockId(const OsPath& pathname, off_t ofs)
-{
-	const Path::String& string = pathname.string();
-	m_id = fnv_hash64(string.c_str(), string.length()*sizeof(string[0]));
-	const size_t indexBits = 16;
-	m_id <<= indexBits;
-	const off_t blockIndex = off_t(ofs / BLOCK_SIZE);
-	debug_assert(blockIndex < off_t(1) << indexBits);
-	m_id |= blockIndex;
-}
-
-bool BlockId::operator==(const BlockId& rhs) const
-{
-	return m_id == rhs.m_id;
-}
-
-bool BlockId::operator!=(const BlockId& rhs) const
-{
-	return !operator==(rhs);
-}
-
-
-//-----------------------------------------------------------------------------
-
-struct Block
-{
-	Block(BlockId id, const shared_ptr<u8>& buf)
-	{
-		this->id = id;
-		this->buf = buf;
-	}
-
-	// block is "valid" and can satisfy Retrieve() requests if a
-	// (non-default-constructed) ID has been assigned.
-	BlockId id;
-
-	// this block is "in use" if use_count != 1.
-	shared_ptr<u8> buf;
-};
-
-
-//-----------------------------------------------------------------------------
-
-class BlockCache::Impl
-{
-public:
-	Impl(size_t numBlocks)
-		: m_maxBlocks(numBlocks)
-	{
-	}
-
-	void Add(BlockId id, const shared_ptr<u8>& buf)
-	{
-		if(m_blocks.size() > m_maxBlocks)
-		{
-#if CONFIG2_CACHE_READ_ONLY
-			mprotect((void*)m_blocks.front().buf.get(), BLOCK_SIZE, PROT_READ);
-#endif
-			m_blocks.pop_front();	// evict oldest block
-		}
-
-#if CONFIG2_CACHE_READ_ONLY
-		mprotect((void*)buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ);
-#endif
-		m_blocks.push_back(Block(id, buf));
-	}
-
-	bool Retrieve(BlockId id, shared_ptr<u8>& buf)
-	{
-		// (linear search is ok since we only expect to manage a few blocks)
-		for(size_t i = 0; i < m_blocks.size(); i++)
-		{
-			Block& block = m_blocks[i];
-			if(block.id == id)
-			{
-				buf = block.buf;
-				return true;
-			}
-		}
-
-		return false;
-	}
-
-	void InvalidateAll()
-	{
-		// note: don't check whether any references are held etc. because
-		// this should only be called at the end of the (test) program.
-		m_blocks.clear();
-	}
-
-private:
-	size_t m_maxBlocks;
-	typedef std::deque<Block> Blocks;
-	Blocks m_blocks;
-};
-
-
-//-----------------------------------------------------------------------------
-
-BlockCache::BlockCache(size_t numBlocks)
-	: impl(new Impl(numBlocks))
-{
-}
-
-void BlockCache::Add(BlockId id, const shared_ptr<u8>& buf)
-{
-	impl->Add(id, buf);
-}
-
-bool BlockCache::Retrieve(BlockId id, shared_ptr<u8>& buf)
-{
-	return impl->Retrieve(id, buf);
-}
-
-void BlockCache::InvalidateAll()
-{
-	return impl->InvalidateAll();
-}

source/lib/file/io/block_cache.h

@@ -1,106 +0,0 @@
-/* Copyright (c) 2010 Wildfire Games
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * cache for aligned I/O blocks.
- */
-
-#ifndef INCLUDED_BLOCK_CACHE
-#define INCLUDED_BLOCK_CACHE
-
-#include "lib/os_path.h"
-
-/**
- * ID that uniquely identifies a block within a file
- **/
-class BlockId
-{
-public:
-	BlockId();
-	BlockId(const OsPath& pathname, off_t ofs);
-	bool operator==(const BlockId& rhs) const;
-	bool operator!=(const BlockId& rhs) const;
-
-private:
-	u64 m_id;
-};
-
-
-/**
- * cache of (aligned) file blocks with support for zero-copy IO.
- * absorbs the overhead of rounding up archive IOs to the nearest block
- * boundaries by keeping the last few blocks in memory.
- *
- * the interface is somewhat similar to FileCache; see the note there.
- *
- * not thread-safe (each thread is intended to have its own cache).
- **/
-class BlockCache
-{
-public:
-	/**
-	 * @param numBlocks (the default value is enough to support temp buffers
-	 * and absorb the cost of unaligned reads from archives.)
-	 **/
-	BlockCache(size_t numBlocks = 16);
-
-	/**
-	 * Add a block to the cache.
-	 *
-	 * @param id Key that will be used to Retrieve the block.
-	 * @param buf
-	 *
-	 * Call this when the block's IO has completed; its data will
-	 * satisfy subsequent Retrieve calls for the same id.
-	 * If CONFIG2_CACHE_READ_ONLY, the memory is made read-only.
-	 **/
-	void Add(BlockId id, const shared_ptr<u8>& buf);
-
-	/**
-	 * Attempt to retrieve a block's contents.
-	 *
-	 * @return whether the block is in cache.
-	 *
-	 * if successful, a shared pointer to the contents is returned.
-	 * they remain valid until all references are removed and the block
-	 * is evicted.
-	 **/
-	bool Retrieve(BlockId id, shared_ptr<u8>& buf);
-
-	/**
-	 * Invalidate the contents of the cache.
-	 *
-	 * this effectively discards the contents of existing blocks
-	 * (more specifically: prevents them from satisfying Retrieve calls
-	 * until a subsequent Add with the same id).
-	 *
-	 * useful for self-tests: multiple independent IO tests run in the same
-	 * process and must not influence each other via the cache.
-	 **/
-	void InvalidateAll();
-
-private:
-	class Impl;
-	shared_ptr<Impl> impl;
-};
-
-#endif	// #ifndef INCLUDED_BLOCK_CACHE

source/lib/file/io/io.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -23,19 +23,14 @@
 #include "precompiled.h"
 #include "lib/file/io/io.h"
 
-#include "lib/posix/posix_aio.h"
-#include "lib/allocators/allocators.h"	// AllocatorChecker
-#include "lib/file/file.h"
-#include "lib/file/common/file_stats.h"
-#include "lib/file/io/block_cache.h"
-#include "lib/file/io/io_align.h"
+#include "lib/sysdep/rtl.h"
 
-static const size_t ioDepth = 16;
+ERROR_ASSOCIATE(ERR::IO, L"Error during IO", EIO);
 
+namespace io {
 
-// the underlying aio implementation likes buffer and offset to be
-// sector-aligned; if not, the transfer goes through an align buffer,
-// and requires an extra memcpy.
+// the Windows aio implementation requires buffer and offset to be
+// sector-aligned.
 //
 // if the user specifies an unaligned buffer, there's not much we can
 // do - we can't assume the buffer contains padding. therefore,
@@ -51,14 +46,14 @@ static const size_t ioDepth = 16;
 // AIO wrapper. rationale:
 // - aligning the transfer isn't possible here since we have no control
 //   over the buffer, i.e. we cannot read more data than requested.
-//   instead, this is done in io_manager.
+//   instead, this is done in manager.
 // - transfer sizes here are arbitrary (i.e. not block-aligned);
 //   that means the cache would have to handle this or also split them up
-//   into blocks, which would duplicate the abovementioned work.
+//   into blocks, which would duplicate the above mentioned work.
 // - if caching here, we'd also have to handle "forwarding" (i.e.
 //   desired block has been issued but isn't yet complete). again, it
-//   is easier to let the synchronous io_manager handle this.
-// - finally, io_manager knows more about whether the block should be cached
+//   is easier to let the synchronous manager handle this.
+// - finally, manager knows more about whether the block should be cached
 //   (e.g. whether another block request will follow), but we don't
 //   currently make use of this.
 //
@@ -71,271 +66,73 @@ static const size_t ioDepth = 16;
 //   this is a bit more complicated than just using the cache as storage.
 
 
-//-----------------------------------------------------------------------------
-// allocator
-//-----------------------------------------------------------------------------
-
-#ifndef NDEBUG
-static AllocatorChecker allocatorChecker;
-#endif
-
-class IoDeleter
+UniqueRange Allocate(size_t size, size_t alignment)
 {
-public:
-	IoDeleter(size_t paddedSize)
-		: m_paddedSize(paddedSize)
-	{
-	}
+	debug_assert(is_pow2(alignment));
+	if(alignment <= idxDeleterBits)
+		alignment = idxDeleterBits+1;
 
-	void operator()(u8* mem)
-	{
-		debug_assert(m_paddedSize != 0);
-#ifndef NDEBUG
-		allocatorChecker.OnDeallocate(mem, m_paddedSize);
-#endif
-		page_aligned_free(mem, m_paddedSize);
-		m_paddedSize = 0;
-	}
-
-private:
-	size_t m_paddedSize;
-};
-
-
-shared_ptr<u8> io_Allocate(size_t size, off_t ofs)
-{
-	debug_assert(size != 0);
-
-	const size_t paddedSize = (size_t)PaddedSize(size, ofs);
-	u8* mem = (u8*)page_aligned_alloc(paddedSize);
-	if(!mem)
-		throw std::bad_alloc();
-
-#ifndef NDEBUG
-	allocatorChecker.OnAllocate(mem, paddedSize);
-#endif
-
-	return shared_ptr<u8>(mem, IoDeleter(paddedSize));
+	const size_t alignedSize = round_up(size, alignment);
+	const UniqueRange::pointer p = rtl_AllocateAligned(alignedSize, alignment);
+	return UniqueRange(p, size, idxDeleterAligned);
 }
 
 
-//-----------------------------------------------------------------------------
-// BlockIo
-//-----------------------------------------------------------------------------
-
-class BlockIo
+LibError Issue(aiocb& cb, size_t queueDepth)
 {
-public:
-	LibError Issue(const PFile& file, off_t alignedOfs, u8* alignedBuf)
+#if CONFIG2_FILE_ENABLE_AIO
+	if(queueDepth > 1)
 	{
-		m_blockId = BlockId(file->Pathname(), alignedOfs);
-		if(file->Mode() == 'r')
-		{
-			if(s_blockCache.Retrieve(m_blockId, m_cachedBlock))
-			{
-				stats_block_cache(CR_HIT);
+		const int ret = (cb.aio_lio_opcode == LIO_WRITE)? aio_write(&cb): aio_read(&cb);
+		RETURN_ERR(LibError_from_posix(ret));
+	}
+	else
+#endif
+	{
+		debug_assert(lseek(cb.aio_fildes, cb.aio_offset, SEEK_SET) == cb.aio_offset);
 
-				// copy from cache into user buffer
-				if(alignedBuf)
-				{
-					memcpy(alignedBuf, m_cachedBlock.get(), BLOCK_SIZE);
-					m_alignedBuf = alignedBuf;
-				}
-				// return cached block
-				else
-				{
-					m_alignedBuf = const_cast<u8*>(m_cachedBlock.get());
-				}
+		void* buf = (void*)cb.aio_buf;	// cast from volatile void*
+		const ssize_t bytesTransferred = (cb.aio_lio_opcode == LIO_WRITE)? write(cb.aio_fildes, buf, cb.aio_nbytes) : read(cb.aio_fildes, buf, cb.aio_nbytes);
+		if(bytesTransferred < 0)
+			return LibError_from_errno();
 
-				return INFO::OK;
-			}
-			else
-			{
-				stats_block_cache(CR_MISS);
-				// fall through to the actual issue..
-			}
-		}
-
-		stats_io_check_seek(m_blockId);
-
-		// transfer directly to/from user buffer
-		if(alignedBuf)
-		{
-			m_alignedBuf = alignedBuf;
-		}
-		// transfer into newly allocated temporary block
-		else
-		{
-			m_tempBlock = io_Allocate(BLOCK_SIZE);
-			m_alignedBuf = const_cast<u8*>(m_tempBlock.get());
-		}
-
-		return file->Issue(m_req, file->Mode(), alignedOfs, m_alignedBuf, BLOCK_SIZE);
+		cb.aio_nbytes = (size_t)bytesTransferred;
 	}
 
-	LibError WaitUntilComplete(const u8*& block, size_t& blockSize)
-	{
-		if(m_cachedBlock)
-		{
-			block = m_alignedBuf;
-			blockSize = BLOCK_SIZE;
-			return INFO::OK;
-		}
-
-		RETURN_ERR(FileImpl::WaitUntilComplete(m_req, const_cast<u8*&>(block), blockSize));
-
-		if(m_tempBlock)
-			s_blockCache.Add(m_blockId, m_tempBlock);
-
-		return INFO::OK;
-	}
-
-private:
-	static BlockCache s_blockCache;
-
-	BlockId m_blockId;
-
-	// the address that WaitUntilComplete will return
-	// (cached or temporary block, or user buffer)
-	u8* m_alignedBuf;
-
-	shared_ptr<u8> m_cachedBlock;
-	shared_ptr<u8> m_tempBlock;
-
-	aiocb m_req;
-};
-
-BlockCache BlockIo::s_blockCache;
-
-
-//-----------------------------------------------------------------------------
-// IoSplitter
-//-----------------------------------------------------------------------------
-
-class IoSplitter
-{
-	NONCOPYABLE(IoSplitter);
-public:
-	IoSplitter(off_t ofs, u8* alignedBuf, off_t size)
-		: m_ofs(ofs), m_alignedBuf(alignedBuf), m_size(size)
-		, m_totalIssued(0), m_totalTransferred(0)
-	{
-		m_alignedOfs = AlignedOffset(ofs);
-		m_alignedSize = PaddedSize(size, ofs);
-		m_misalignment = size_t(ofs - m_alignedOfs);
-	}
-
-	LibError Run(const PFile& file, IoCallback cb = 0, uintptr_t cbData = 0)
-	{
-		ScopedIoMonitor monitor;
-
-		// (issue even if cache hit because blocks must be processed in order)
-		std::deque<BlockIo> pendingIos;
-		for(;;)
-		{
-			while(pendingIos.size() < ioDepth && m_totalIssued < m_alignedSize)
-			{
-				pendingIos.push_back(BlockIo());
-				const off_t alignedOfs = m_alignedOfs + m_totalIssued;
-				u8* const alignedBuf = m_alignedBuf? m_alignedBuf+m_totalIssued : 0;
-				RETURN_ERR(pendingIos.back().Issue(file, alignedOfs, alignedBuf));
-				m_totalIssued += BLOCK_SIZE;
-			}
-
-			if(pendingIos.empty())
-				break;
-
-			Process(pendingIos.front(), cb, cbData);
-			pendingIos.pop_front();
-		}
-
-		debug_assert(m_totalIssued >= m_totalTransferred && m_totalTransferred >= m_size);
-
-		monitor.NotifyOfSuccess(FI_AIO, file->Mode(), m_totalTransferred);
-		return INFO::OK;
-	}
-
-	off_t AlignedOfs() const
-	{
-		return m_alignedOfs;
-	}
-
-private:
-	LibError Process(BlockIo& blockIo, IoCallback cb, uintptr_t cbData) const
-	{
-		const u8* block; size_t blockSize;
-		RETURN_ERR(blockIo.WaitUntilComplete(block, blockSize));
-
-		// first block: skip past alignment
-		if(m_totalTransferred == 0)
-		{
-			block += m_misalignment;
-			blockSize -= m_misalignment;
-		}
-
-		// last block: don't include trailing padding
-		if(m_totalTransferred + (off_t)blockSize > m_size)
-			blockSize = size_t(m_size - m_totalTransferred);
-
-		m_totalTransferred += (off_t)blockSize;
-
-		if(cb)
-		{
-			stats_cb_start();
-			LibError ret = cb(cbData, block, blockSize);
-			stats_cb_finish();
-			CHECK_ERR(ret);
-		}
-
-		return INFO::OK;
-	}
-
-	off_t m_ofs;
-	u8* m_alignedBuf;
-	off_t m_size;
-
-	size_t m_misalignment;
-	off_t m_alignedOfs;
-	off_t m_alignedSize;
-
-	// (useful, raw data: possibly compressed, but doesn't count padding)
-	mutable off_t m_totalIssued;
-	mutable off_t m_totalTransferred;
-};
-
-
-LibError io_Scan(const PFile& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData)
-{
-	u8* alignedBuf = 0;	// use temporary block buffers
-	IoSplitter splitter(ofs, alignedBuf, size);
-	return splitter.Run(file, cb, cbData);
-}
-
-
-LibError io_Read(const PFile& file, off_t ofs, u8* alignedBuf, off_t size, u8*& data, IoCallback cb, uintptr_t cbData)
-{
-	IoSplitter splitter(ofs, alignedBuf, size);
-	RETURN_ERR(splitter.Run(file, cb, cbData));
-	data = alignedBuf + ofs - splitter.AlignedOfs();
 	return INFO::OK;
 }
 
 
-LibError io_WriteAligned(const PFile& file, off_t alignedOfs, const u8* alignedData, off_t size, IoCallback cb, uintptr_t cbData)
+LibError WaitUntilComplete(aiocb& cb, size_t queueDepth)
 {
-	debug_assert(IsAligned_Offset(alignedOfs));
-	debug_assert(IsAligned_Data(alignedData));
+#if CONFIG2_FILE_ENABLE_AIO
+	if(queueDepth > 1)
+	{
+		aiocb* const cbs = &cb;
+		timespec* const timeout = 0;	// infinite
+SUSPEND_AGAIN:
+		errno = 0;
+		const int ret = aio_suspend(&cbs, 1, timeout);
+		if(ret != 0)
+		{
+			if(errno == EINTR) // interrupted by signal
+				goto SUSPEND_AGAIN;
+			return LibError_from_errno();
+		}
 
-	IoSplitter splitter(alignedOfs, const_cast<u8*>(alignedData), size);
-	return splitter.Run(file, cb, cbData);
+		const int err = aio_error(&cb);
+		debug_assert(err != EINPROGRESS);	// else aio_return is undefined
+		ssize_t bytesTransferred = aio_return(&cb);
+		if(bytesTransferred == -1)	// transfer failed
+		{
+			errno = err;
+			return LibError_from_errno();
+		}
+		cb.aio_nbytes = (size_t)bytesTransferred;
+	}
+#endif
+
+	return INFO::OK;
 }
 
-
-LibError io_ReadAligned(const PFile& file, off_t alignedOfs, u8* alignedBuf, off_t size, IoCallback cb, uintptr_t cbData)
-{
-	debug_assert(IsAligned_Offset(alignedOfs));
-	debug_assert(IsAligned_Data(alignedBuf));
-
-	IoSplitter splitter(alignedOfs, alignedBuf, size);
-	return splitter.Run(file, cb, cbData);
-}
+}	// namespace io

source/lib/file/io/io.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -21,36 +21,314 @@
  */
 
 /*
- * IO manager: splits requests into asynchronously issued blocks,
- * thus simplifying caching and enabling periodic callbacks.
+ * provide asynchronous and synchronous I/O with hooks to allow
+ * overlapped processing or progress reporting.
  */
 
 #ifndef INCLUDED_IO
 #define INCLUDED_IO
 
+#include "lib/config2.h"
+#include "lib/alignment.h"
+#include "lib/bits.h"
 #include "lib/file/file.h"
+#include "lib/sysdep/filesystem.h"	// wtruncate
 
-// memory will be allocated from the heap, not the (limited) file cache.
-// this makes sense for write buffers that are never used again,
-// because we avoid having to displace some other cached items.
-LIB_API shared_ptr<u8> io_Allocate(size_t size, off_t ofs = 0);
+#include "lib/allocators/unique_range.h"
 
-/**
- * called after a block IO has completed.
- *
- * @return INFO::CB_CONTINUE to continue; any other value will cause the
- * IO splitter to abort immediately and return that.
- *
- * this is useful for user progress notification or processing data while
- * waiting for the next I/O to complete (without the complexity of threads).
- **/
-typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t blockSize);
+namespace ERR
+{
+	const LibError IO = -110301;
+}
 
-LIB_API LibError io_Scan(const PFile& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData);
+namespace io {
 
-LIB_API LibError io_Read(const PFile& file, off_t ofs, u8* alignedBuf, off_t size, u8*& data, IoCallback cb = 0, uintptr_t cbData = 0);
+// @return memory suitable for use as an I/O buffer (address is a
+// multiple of alignment, size is rounded up to a multiple of alignment)
+//
+// use this instead of the file cache for write buffers that are
+// never reused (avoids displacing other items).
+LIB_API UniqueRange Allocate(size_t size, size_t alignment = maxSectorSize);
 
-LIB_API LibError io_WriteAligned(const PFile& file, off_t alignedOfs, const u8* alignedData, off_t size, IoCallback cb = 0, uintptr_t cbData = 0);
-LIB_API LibError io_ReadAligned(const PFile& file, off_t alignedOfs, u8* alignedBuf, off_t size, IoCallback cb = 0, uintptr_t cbData = 0);
+
+#pragma pack(push, 1)
+
+// required information for any I/O (this is basically the same as aiocb,
+// but also applies to synchronous I/O and has shorter/nicer names.)
+struct Operation
+{
+	// @param buf can be 0, in which case temporary block buffers are allocated.
+	// otherwise, it must be padded to the I/O alignment, e.g. via io::Allocate.
+	Operation(const File& file, void* buf, off_t size, off_t offset = 0)
+		: fd(file.Descriptor()), opcode(file.Opcode())
+		, offset(offset), size(size), buf((void*)buf)
+	{
+	}
+
+	void Validate() const
+	{
+		debug_assert(fd >= 0);
+		debug_assert(opcode == LIO_READ || opcode == LIO_WRITE);
+
+		debug_assert(offset >= 0);
+		debug_assert(size >= 0);
+		// buf can legitimately be 0 (see above)
+	}
+
+	int fd;
+	int opcode;
+
+	off_t offset;
+	off_t size;
+	void* buf;
+};
+
+
+// optional information how an Operation is to be carried out
+struct Parameters
+{
+	// default to single blocking I/Os
+	Parameters()
+		: alignment(1)	// no alignment requirements
+		// use one huge "block" truncated to the requested size.
+		// (this value is a power of two as required by Validate and
+		// avoids overflowing off_t in DivideRoundUp)
+		, blockSize((SIZE_MAX/2)+1)
+		, queueDepth(1)	// disable aio
+	{
+	}
+
+	// parameters for asynchronous I/O that maximize throughput on current drives
+	struct OverlappedTag {};
+	Parameters(OverlappedTag)
+		: alignment(maxSectorSize), blockSize(128*KiB), queueDepth(32)
+	{
+	}
+
+	Parameters(size_t blockSize, size_t queueDepth, off_t alignment = maxSectorSize)
+		: alignment(alignment), blockSize(blockSize), queueDepth(queueDepth)
+	{
+	}
+
+	void Validate(const Operation& op) const
+	{
+		debug_assert(is_pow2(alignment));
+		debug_assert(alignment > 0);
+
+		debug_assert(is_pow2(blockSize));
+		debug_assert(pageSize <= blockSize);	// no upper limit needed
+
+		debug_assert(1 <= queueDepth && queueDepth <= maxQueueDepth);
+
+		debug_assert(IsAligned(op.offset, alignment));
+		// op.size doesn't need to be aligned
+		debug_assert(IsAligned(op.buf, alignment));
+	}
+
+	// (ATTO only allows 10, which improves upon 8)
+	static const size_t maxQueueDepth = 32;
+
+	off_t alignment;
+
+	size_t blockSize;
+
+	size_t queueDepth;
+};
+
+#define IO_OVERLAPPED io::Parameters(io::Parameters::OverlappedTag())
+
+
+struct DefaultCompletedHook
+{
+	/**
+	 * called after a block I/O has completed.
+	 *
+	 * @return INFO::CB_CONTINUE to proceed; any other value will
+	 * be immediately returned by Run.
+	 *
+	 * allows progress notification and processing data while waiting for
+	 * previous I/Os to complete.
+	 **/
+	LibError operator()(const u8* UNUSED(block), size_t UNUSED(blockSize)) const
+	{
+		return INFO::CB_CONTINUE;
+	}
+};
+
+
+struct DefaultIssueHook
+{
+	/**
+	 * called before a block I/O is issued.
+	 *
+	 * @return INFO::CB_CONTINUE to proceed; any other value will
+	 * be immediately returned by Run.
+	 *
+	 * allows generating the data to write while waiting for
+	 * previous I/Os to complete.
+	 **/
+	LibError operator()(aiocb& UNUSED(cb)) const
+	{
+		return INFO::CB_CONTINUE;
+	}
+};
+
+
+// ring buffer of partially initialized aiocb that can be passed
+// directly to aio_write etc. after setting offset and buffer.
+class ControlBlockRingBuffer
+{
+public:
+	ControlBlockRingBuffer(const Operation& op, const Parameters& p)
+		: controlBlocks()	// zero-initialize
+	{
+		// (default p.blockSize is "infinity", so clamp to the total size)
+		const size_t blockSize = (size_t)std::min((off_t)p.blockSize, op.size);
+
+		const bool temporaryBuffersRequested = (op.buf == 0);
+		if(temporaryBuffersRequested)
+			buffers = RVALUE(io::Allocate(blockSize * p.queueDepth, p.alignment));
+
+		for(size_t i = 0; i < ARRAY_SIZE(controlBlocks); i++)
+		{
+			aiocb& cb = operator[](i);
+			cb.aio_fildes = op.fd;
+			cb.aio_nbytes = blockSize;
+			cb.aio_lio_opcode = op.opcode;
+			if(temporaryBuffersRequested)
+				cb.aio_buf = (volatile void*)(uintptr_t(buffers.get()) + i * blockSize);
+		}
+	}
+
+	INLINE aiocb& operator[](size_t counter)
+	{
+		return controlBlocks[counter % ARRAY_SIZE(controlBlocks)];
+	}
+
+private:
+	UniqueRange buffers;
+	aiocb controlBlocks[Parameters::maxQueueDepth];
+};
+
+#pragma pack(pop)
+
+
+LIB_API LibError Issue(aiocb& cb, size_t queueDepth);
+LIB_API LibError WaitUntilComplete(aiocb& cb, size_t queueDepth);
+
+
+//-----------------------------------------------------------------------------
+// Run
+
+// (hooks must be passed by const reference to allow passing rvalues.
+// functors with non-const member data can mark them as mutable.)
+template<class CompletedHook, class IssueHook>
+static inline LibError Run(const Operation& op, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook())
+{
+	op.Validate();
+	p.Validate(op);
+
+	ControlBlockRingBuffer controlBlockRingBuffer(op, p);
+
+	const off_t numBlocks = DivideRoundUp(op.size, (off_t)p.blockSize);
+	for(off_t blocksIssued = 0, blocksCompleted = 0; blocksCompleted < numBlocks; blocksCompleted++)
+	{
+		for(; blocksIssued != numBlocks && blocksIssued < blocksCompleted + (off_t)p.queueDepth; blocksIssued++)
+		{
+			aiocb& cb = controlBlockRingBuffer[blocksIssued];
+			cb.aio_offset = op.offset + blocksIssued * p.blockSize;
+			if(op.buf)
+				cb.aio_buf = (volatile void*)(uintptr_t(op.buf) + blocksIssued * p.blockSize);
+			if(blocksIssued == numBlocks-1)
+				cb.aio_nbytes = round_up(size_t(op.size - blocksIssued * p.blockSize), size_t(p.alignment));
+
+			RETURN_IF_NOT_CONTINUE(issueHook(cb));
+
+			RETURN_ERR(Issue(cb, p.queueDepth));
+		}
+
+		aiocb& cb = controlBlockRingBuffer[blocksCompleted];
+		RETURN_ERR(WaitUntilComplete(cb, p.queueDepth));
+
+		RETURN_IF_NOT_CONTINUE(completedHook((u8*)cb.aio_buf, cb.aio_nbytes));
+	}
+
+	return INFO::OK;
+}
+
+// (overloads allow omitting parameters without requiring a template argument list)
+template<class CompletedHook>
+static inline LibError Run(const Operation& op, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook())
+{
+	return Run(op, p, completedHook, DefaultIssueHook());
+}
+
+static inline LibError Run(const Operation& op, const Parameters& p = Parameters())
+{
+	return Run(op, p, DefaultCompletedHook(), DefaultIssueHook());
+}
+
+
+//-----------------------------------------------------------------------------
+// Store
+
+// efficient writing requires preallocation, and the resulting file is
+// padded to the sector size and needs to be truncated afterwards.
+// this function takes care of both.
+template<class CompletedHook, class IssueHook>
+static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook())
+{
+	File file(pathname, LIO_WRITE);
+	io::Operation op(file, (void*)data, size);
+
+#if OS_WIN && CONFIG2_FILE_ENABLE_AIO
+	(void)waio_Preallocate(op.fd, (off_t)size, p.alignment);
+#endif
+
+	RETURN_ERR(io::Run(op, p, completedHook, issueHook));
+
+	file.Close();	// (required by wtruncate)
+
+	RETURN_ERR(wtruncate(pathname, size));
+
+	return INFO::OK;
+}
+
+template<class CompletedHook>
+static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook())
+{
+	return Store(pathname, data, size, p, completedHook, DefaultIssueHook());
+}
+
+static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters())
+{
+	return Store(pathname, data, size, p, DefaultCompletedHook(), DefaultIssueHook());
+}
+
+
+//-----------------------------------------------------------------------------
+// Load
+
+// convenience function provided for symmetry with Store
+template<class CompletedHook, class IssueHook>
+static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook())
+{
+	File file(pathname, LIO_READ);
+	io::Operation op(file, buf, size);
+	return io::Run(op, p, completedHook, issueHook);
+}
+
+template<class CompletedHook>
+static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook())
+{
+	return Load(pathname, buf, size, p, completedHook, DefaultIssueHook());
+}
+
+static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters())
+{
+	return Load(pathname, buf, size, p, DefaultCompletedHook(), DefaultIssueHook());
+}
+
+}	// namespace io
 
 #endif	// #ifndef INCLUDED_IO

source/lib/file/io/io_align.cpp

@@ -1,24 +0,0 @@
-/* Copyright (c) 2010 Wildfire Games
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include "precompiled.h"
-#include "lib/file/io/io_align.h"

source/lib/file/io/io_align.h

@@ -1,71 +0,0 @@
-/* Copyright (c) 2010 Wildfire Games
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef INCLUDED_IO_ALIGN
-#define INCLUDED_IO_ALIGN
-
-#include "lib/bits.h"	// IsAligned, round_up
-
-/**
- * block := power-of-two sized chunk of a file.
- * all transfers are expanded to naturally aligned, whole blocks.
- * (this makes caching parts of files feasible; it is also much faster
- * for some aio implementations, e.g. wposix.)
- * (blocks are also thereby page-aligned, which allows write-protecting
- * file buffers without worrying about their boundaries.)
- **/
-static const size_t BLOCK_SIZE = 1024*KiB;
-
-// note: *sizes* and *offsets* are aligned to blocks to allow zero-copy block cache.
-// that the *buffer* need only be sector-aligned (we assume 4kb for simplicity)
-// (this is a requirement of the underlying Windows OS)
-static const size_t SECTOR_SIZE = 4*KiB;
-
-
-template<class T>
-inline bool IsAligned_Data(T* address)
-{
-	return IsAligned((uintptr_t)address, SECTOR_SIZE);
-}
-
-inline bool IsAligned_Offset(off_t ofs)
-{
-	return IsAligned(ofs, BLOCK_SIZE);
-}
-
-
-inline off_t AlignedOffset(off_t ofs)
-{
-	return (off_t)round_down<size_t>(size_t(ofs), BLOCK_SIZE);
-}
-
-inline off_t AlignedSize(off_t size)
-{
-	return (off_t)round_up<size_t>(size_t(size), BLOCK_SIZE);
-}
-
-inline off_t PaddedSize(off_t size, off_t ofs)
-{
-	return (off_t)round_up<size_t>(size_t(size + ofs - AlignedOffset(ofs)), BLOCK_SIZE);
-}
-
-#endif	// #ifndef INCLUDED_IO_ALIGN

source/lib/file/io/write_buffer.cpp

@@ -25,12 +25,15 @@
 
 #include "lib/bits.h"	// IsAligned
 #include "lib/sysdep/cpu.h"
+#include "lib/allocators/shared_ptr.h"
 #include "lib/file/io/io.h"
-#include "lib/file/io/io_align.h"
+
+
+static const size_t BLOCK_SIZE = 512*KiB;
 
 
 WriteBuffer::WriteBuffer()
-	: m_capacity(4096), m_data(io_Allocate(m_capacity)), m_size(0)
+	: m_capacity(pageSize), m_data((u8*)rtl_AllocateAligned(m_capacity, maxSectorSize), AlignedDeleter()), m_size(0)
 {
 }
 
@@ -40,7 +43,8 @@ void WriteBuffer::Append(const void* data, size_t size)
 	if(m_size + size > m_capacity)
 	{
 		m_capacity = round_up_to_pow2(m_size + size);
-		shared_ptr<u8> newData = io_Allocate(m_capacity);
+		shared_ptr<u8> newData;
+		AllocateAligned(newData, m_capacity, maxSectorSize);
 		memcpy(newData.get(), m_data.get(), m_size);
 		m_data = newData;
 	}
@@ -62,12 +66,15 @@ void WriteBuffer::Overwrite(const void* data, size_t size, size_t offset)
 //-----------------------------------------------------------------------------
 
 UnalignedWriter::UnalignedWriter(const PFile& file, off_t ofs)
-	: m_file(file), m_alignedBuf(io_Allocate(BLOCK_SIZE))
+	: m_file(file), m_alignedBuf((u8*)rtl_AllocateAligned(BLOCK_SIZE, maxSectorSize), AlignedDeleter())
 {
-	m_alignedOfs = AlignedOffset(ofs);
+	m_alignedOfs = round_down(ofs, (off_t)BLOCK_SIZE);
 	const size_t misalignment = (size_t)(ofs - m_alignedOfs);
 	if(misalignment)
-		io_ReadAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE);
+	{
+		io::Operation op(*m_file.get(), m_alignedBuf.get(), BLOCK_SIZE, m_alignedOfs);
+		THROW_ERR(io::Run(op));
+	}
 	m_bytesUsed = misalignment;
 }
 
@@ -84,9 +91,10 @@ LibError UnalignedWriter::Append(const u8* data, size_t size) const
 	{
 		// optimization: write directly from the input buffer, if possible
 		const size_t alignedSize = (size / BLOCK_SIZE) * BLOCK_SIZE;
-		if(m_bytesUsed == 0 && IsAligned(data, SECTOR_SIZE) && alignedSize != 0)
+		if(m_bytesUsed == 0 && IsAligned(data, maxSectorSize) && alignedSize != 0)
 		{
-			RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, data, alignedSize));
+			io::Operation op(*m_file.get(), (void*)data, alignedSize, m_alignedOfs);
+			RETURN_ERR(io::Run(op));
 			m_alignedOfs += (off_t)alignedSize;
 			data += alignedSize;
 			size -= alignedSize;
@@ -118,7 +126,8 @@ void UnalignedWriter::Flush() const
 
 LibError UnalignedWriter::WriteBlock() const
 {
-	RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE));
+	io::Operation op(*m_file.get(), m_alignedBuf.get(), BLOCK_SIZE, m_alignedOfs);
+	RETURN_ERR(io::Run(op));
 	m_alignedOfs += BLOCK_SIZE;
 	m_bytesUsed = 0;
 	return INFO::OK;

source/lib/file/vfs/file_cache.cpp

@@ -30,7 +30,6 @@
 #include "lib/external_libraries/suppress_boost_warnings.h"
 
 #include "lib/file/common/file_stats.h"
-#include "lib/file/io/io_align.h"       // BLOCK_SIZE
 #include "lib/cache_adt.h"              // Cache
 #include "lib/bits.h"                   // round_up
 #include "lib/allocators/allocators.h"
@@ -98,7 +97,7 @@ public:
 
 	shared_ptr<u8> Allocate(size_t size, const PAllocator& pthis)
 	{
-		const size_t alignedSize = round_up(size, BLOCK_SIZE);
+		const size_t alignedSize = Align<maxSectorSize>(size);
 
 		u8* mem = (u8*)m_allocator.Allocate(alignedSize);
 		if(!mem)
@@ -114,7 +113,7 @@ public:
 
 	void Deallocate(u8* mem, size_t size)
 	{
-		const size_t alignedSize = round_up(size, BLOCK_SIZE);
+		const size_t alignedSize = Align<maxSectorSize>(size);
 
 		// (re)allow writes in case the buffer was made read-only. it would
 		// be nice to unmap the buffer, but this is not possible because

source/lib/file/vfs/vfs.cpp

@@ -150,7 +150,7 @@ public:
 				fileContents = DummySharedPtr((u8*)0);
 			else if(size > m_cacheSize)
 			{
-				fileContents = io_Allocate(size);
+				RETURN_ERR(AllocateAligned(fileContents, size, maxSectorSize));
 				RETURN_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size()));
 			}
 			else

source/lib/lib_errors.h

@@ -316,6 +316,15 @@ STMT(\
 	}\
 )
 
+
+// if expression evaluates to a negative error code, return 0.
+#define RETURN_IF_NOT_CONTINUE(expression)\
+STMT(\
+	i64 err64__ = (i64)(expression);\
+	if(err64__ != INFO::CB_CONTINUE)\
+		return err64__;\
+)
+
 // return an error and warn about it (replaces debug_warn+return)
 #define WARN_RETURN(err)\
 STMT(\
@@ -375,8 +384,8 @@ STMT(\
 STMT(\
 	if(!(ok))\
 	{\
-		debug_warn(L"FYI: WARN_RETURN_IF_FALSE reports that a function failed."\
-		           L"feel free to ignore or suppress this warning.");\
+		debug_warn(L"FYI: WARN_RETURN_IF_FALSE reports that a function failed. "\
+		           L"Feel free to ignore or suppress this warning.");\
 		return ERR::FAIL;\
 	}\
 )
@@ -392,8 +401,8 @@ STMT(\
 #define WARN_IF_FALSE(ok)\
 STMT(\
 	if(!(ok))\
-		debug_warn(L"FYI: WARN_IF_FALSE reports that a function failed."\
-		           L"feel free to ignore or suppress this warning.");\
+		debug_warn(L"FYI: WARN_IF_FALSE reports that a function failed. "\
+		           L"Feel free to ignore or suppress this warning.");\
 )
 
 

source/lib/res/sound/ogg.cpp

@@ -5,7 +5,7 @@
 #include "lib/external_libraries/vorbis.h"
 
 #include "lib/byte_order.h"
-#include "lib/file/file.h"
+#include "lib/file/io/io.h"
 #include "lib/file/file_system_util.h"
 
 
@@ -64,7 +64,8 @@ public:
 		const off_t sizeRemaining = adapter->size - adapter->offset;
 		const size_t sizeToRead = (size_t)std::min(sizeRequested, sizeRemaining);
 
-		if(adapter->file->Read(adapter->offset, (u8*)bufferToFill, sizeToRead) == INFO::OK)
+		io::Operation op(*adapter->file.get(), bufferToFill, sizeToRead, adapter->offset);
+		if(io::Run(op) == INFO::OK)
 		{
 			adapter->offset += sizeToRead;
 			return sizeToRead;

source/lib/sysdep/compiler.h

@@ -27,10 +27,6 @@
 #ifndef INCLUDED_COMPILER
 #define INCLUDED_COMPILER
 
-#include "lib/sysdep/arch.h"	// ARCH_AMD64
-#include "lib/config.h"	// CONFIG_OMIT_FP
-
-
 // detect compiler and its version (0 if not present, otherwise
 // major*100 + minor). note that more than one *_VERSION may be
 // non-zero due to interoperability (e.g. ICC with MSC).
@@ -60,18 +56,6 @@
 #endif
 
 
-// pass "omit frame pointer" setting on to the compiler
-#if MSC_VERSION && !ARCH_AMD64
-# if CONFIG_OMIT_FP
-#  pragma optimize("y", on)
-# else
-#  pragma optimize("y", off)
-# endif
-#elif GCC_VERSION
-// TODO
-#endif
-
-
 // are PreCompiled Headers supported?
 #if MSC_VERSION
 # define HAVE_PCH 1
@@ -82,19 +66,6 @@
 #endif
 
 
-// try to define _W64, if not already done
-// (this is useful for catching pointer size bugs)
-#ifndef _W64
-# if MSC_VERSION
-#  define _W64 __w64
-# elif GCC_VERSION
-#  define _W64 __attribute__((mode (__pointer__)))
-# else
-#  define _W64
-# endif
-#endif
-
-
 // check if compiling in pure C mode (not C++) with support for C99.
 // (this is more convenient than testing __STDC_VERSION__ directly)
 //
@@ -118,7 +89,7 @@
 #endif
 
 
-// (at least rudimentary) support for C++0x
+// do we have (at least rudimentary) support for C++0x?
 #ifndef HAVE_CPP0X
 # if defined(__GXX_EXPERIMENTAL_CPP0X__) || MSC_VERSION >= 1600 || ICC_VERSION >= 1200
 #  define HAVE_CPP0X 1
@@ -127,104 +98,4 @@
 # endif
 #endif
 
-
-// C99-like restrict (non-standard in C++, but widely supported in various forms).
-//
-// May be used on pointers. May also be used on member functions to indicate
-// that 'this' is unaliased (e.g. "void C::m() RESTRICT { ... }").
-// Must not be used on references - GCC supports that but VC doesn't.
-//
-// We call this "RESTRICT" to avoid conflicts with VC's __declspec(restrict),
-// and because it's not really the same as C99's restrict.
-//
-// To be safe and satisfy the compilers' stated requirements: an object accessed
-// by a restricted pointer must not be accessed by any other pointer within the
-// lifetime of the restricted pointer, if the object is modified.
-// To maximise the chance of optimisation, any pointers that could potentially
-// alias with the restricted one should be marked as restricted too.
-//
-// It would probably be a good idea to write test cases for any code that uses
-// this in an even very slightly unclear way, in case it causes obscure problems
-// in a rare compiler due to differing semantics.
-//
-// .. GCC
-#if GCC_VERSION
-# define RESTRICT __restrict__
-// .. VC8 provides __restrict
-#elif MSC_VERSION >= 1400
-# define RESTRICT __restrict
-// .. ICC supports the keyword 'restrict' when run with the /Qrestrict option,
-//    but it always also supports __restrict__ or __restrict to be compatible
-//    with GCC/MSVC, so we'll use the underscored version. One of {GCC,MSC}_VERSION
-//    should have been defined in addition to ICC_VERSION, so we should be using
-//    one of the above cases (unless it's an old VS7.1-emulating ICC).
-#elif ICC_VERSION
-# error ICC_VERSION defined without either GCC_VERSION or an adequate MSC_VERSION
-// .. unsupported; remove it from code
-#else
-# define RESTRICT
-#endif
-
-
-// C99-style __func__
-// .. newer GCC already have it
-#if GCC_VERSION >= 300
-	// nothing need be done
-// .. old GCC and MSVC have __FUNCTION__
-#elif GCC_VERSION >= 200 || MSC_VERSION
-# define __func__ __FUNCTION__
-// .. unsupported
-#else
-# define __func__ "(unknown)"
-#endif
-
-
-// tell the compiler that the code at/following this macro invocation is
-// unreachable. this can improve optimization and avoid warnings.
-//
-// this macro should not generate any fallback code; it is merely the
-// compiler-specific backend for lib.h's UNREACHABLE.
-// #define it to nothing if the compiler doesn't support such a hint.
-#define HAVE_ASSUME_UNREACHABLE 1
-#if MSC_VERSION && !ICC_VERSION // (ICC ignores this)
-# define ASSUME_UNREACHABLE __assume(0)
-#elif GCC_VERSION >= 450
-# define ASSUME_UNREACHABLE __builtin_unreachable()
-#else
-# define ASSUME_UNREACHABLE
-# undef HAVE_ASSUME_UNREACHABLE
-# define HAVE_ASSUME_UNREACHABLE 0
-#endif
-
-
-// extern "C", but does the right thing in pure-C mode
-#if defined(__cplusplus)
-# define EXTERN_C extern "C"
-#else
-# define EXTERN_C extern
-#endif
-
-#if MSC_VERSION
-# define INLINE __forceinline
-#else
-# define INLINE inline
-#endif
-
-#if MSC_VERSION
-# define CALL_CONV __cdecl
-#else
-# define CALL_CONV
-#endif
-
-#if MSC_VERSION && !ARCH_AMD64
-# define DECORATED_NAME(name) _##name
-#else
-# define DECORATED_NAME(name) name
-#endif
-
-// workaround for preprocessor limitation: macro args aren't expanded
-// before being pasted.
-#define STRINGIZE2(id) # id
-#define STRINGIZE(id) STRINGIZE2(id)
-
 #endif	// #ifndef INCLUDED_COMPILER

source/lib/sysdep/filesystem.h

@@ -86,6 +86,11 @@ extern int wclose(int fd);
 // unistd.h
 //
 
+// waio requires offsets and sizes to be multiples of the sector size.
+// to allow arbitrarily sized files, we truncate them after I/O.
+// however, ftruncate cannot be used since it is also subject to the
+// sector-alignment requirement. instead, the file must be closed and
+// this function called.
 LIB_API int wtruncate(const OsPath& pathname, off_t length);
 
 LIB_API int wunlink(const OsPath& pathname);

source/lib/sysdep/os/win/wdir_watch.cpp

@@ -32,6 +32,7 @@
 #include "lib/sysdep/os/win/win.h"
 #include "lib/sysdep/os/win/winit.h"
 #include "lib/sysdep/os/win/wutil.h"
+#include "lib/sysdep/os/win/wiocp.h"
 
 
 WINIT_REGISTER_MAIN_INIT(wdir_watch_Init);
@@ -290,84 +291,22 @@ struct DirWatch
 };
 
 
-//-----------------------------------------------------------------------------
-// CompletionPort
-
-// this appears to be the best solution for IO notification.
-// there are three alternatives:
-// - multiple threads with blocking I/O. this is rather inefficient when
-//   many directories (e.g. mods) are being watched.
-// - normal overlapped I/O: build a contiguous array of the hEvents
-//   in all OVERLAPPED structures, and WaitForMultipleObjects.
-//   it would be cumbersome to update this array when adding/removing watches.
-// - callback notification: a notification function is called when the thread
-//   that initiated the I/O (ReadDirectoryChangesW) enters an alertable
-//   wait state. it is desirable for notifications to arrive at a single
-//   known point - see dir_watch_Poll. unfortunately there doesn't appear to
-//   be a reliable and non-blocking means of entering AWS - SleepEx(1) may
-//   wait for 10..15 ms if the system timer granularity is low. even worse,
-//   it was noted in a previous project that APCs are sometimes delivered from
-//   within APIs without having used SleepEx (it seems threads sometimes enter
-//   a semi-AWS when calling the kernel).
-class CompletionPort
-{
-public:
-	CompletionPort()
-	{
-		m_hIOCP = 0;	// CreateIoCompletionPort requires 0, not INVALID_HANDLE_VALUE
-	}
-
-	~CompletionPort()
-	{
-		CloseHandle(m_hIOCP);
-		m_hIOCP = INVALID_HANDLE_VALUE;
-	}
-
-	void Attach(HANDLE hFile, uintptr_t key)
-	{
-		WinScopedPreserveLastError s;	// CreateIoCompletionPort
-
-		// (when called for the first time, ends up creating m_hIOCP)
-		m_hIOCP = CreateIoCompletionPort(hFile, m_hIOCP, (ULONG_PTR)key, 0);
-		debug_assert(wutil_IsValidHandle(m_hIOCP));
-	}
-
-	LibError Poll(size_t& bytesTransferred, uintptr_t& key, OVERLAPPED*& ovl)
-	{
-		if(m_hIOCP == 0)
-			return ERR::INVALID_HANDLE;	// NOWARN (happens if called before the first Attach)
-		for(;;)	// don't return abort notifications to caller
-		{
-			DWORD dwBytesTransferred = 0;
-			ULONG_PTR ulKey = 0;
-			ovl = 0;
-			const DWORD timeout = 0;
-			const BOOL gotPacket = GetQueuedCompletionStatus(m_hIOCP, &dwBytesTransferred, &ulKey, &ovl, timeout);
-			bytesTransferred = size_t(dwBytesTransferred);
-			key = uintptr_t(ulKey);
-			if(gotPacket)
-				return INFO::OK;
-
-			if(GetLastError() == WAIT_TIMEOUT)
-				return ERR::AGAIN;	// NOWARN (nothing pending)
-			else if(GetLastError() == ERROR_OPERATION_ABORTED)
-				continue;		// watch was canceled - ignore
-			else
-				return LibError_from_GLE();	// actual error
-		}
-	}
-
-private:
-	HANDLE m_hIOCP;
-};
-
-
 //-----------------------------------------------------------------------------
 // DirWatchManager
 
 class DirWatchManager
 {
 public:
+	DirWatchManager()
+		: hIOCP(0)
+	{
+	}
+
+	~DirWatchManager()
+	{
+		CloseHandle(hIOCP);
+	}
+
 	LibError Add(const OsPath& path, PDirWatch& dirWatch)
 	{
 		debug_assert(path.IsDirectory());
@@ -386,7 +325,7 @@ public:
 		}
 
 		PDirWatchRequest request(new DirWatchRequest(path));
-		m_completionPort.Attach(request->GetDirHandle(), (uintptr_t)request.get());
+		AttachToCompletionPort(request->GetDirHandle(), hIOCP, (uintptr_t)request.get());
 		RETURN_ERR(request->Issue());
 		dirWatch.reset(new DirWatch(&m_sentinel, request));
 		return INFO::OK;
@@ -394,8 +333,17 @@ public:
 
 	LibError Poll(DirWatchNotifications& notifications)
 	{
-		size_t bytesTransferred; uintptr_t key; OVERLAPPED* ovl;
-		RETURN_ERR(m_completionPort.Poll(bytesTransferred, key, ovl));
+		DWORD bytesTransferred; ULONG_PTR key; OVERLAPPED* ovl;
+		for(;;)	// skip notifications of canceled watches
+		{
+			const LibError ret = PollCompletionPort(hIOCP, 0, bytesTransferred, key, ovl);
+			if(ret == INFO::OK)
+				break;
+			if(GetLastError() == ERROR_OPERATION_ABORTED)
+				continue;		// watch was canceled - ignore
+			return ret;
+		}
+
 		DirWatchRequest* request = (DirWatchRequest*)key;
 		request->RetrieveNotifications(notifications);
 		RETURN_ERR(request->Issue());	// re-issue
@@ -404,7 +352,7 @@ public:
 
 private:
 	IntrusiveLink m_sentinel;
-	CompletionPort m_completionPort;
+	HANDLE hIOCP;
 };
 
 static DirWatchManager* s_dirWatchManager;

source/lib/sysdep/os/win/wfirmware.cpp

@@ -17,7 +17,7 @@ TableIds GetTableIDs(Provider provider)
 	debug_assert(tableIdsSize % sizeof(TableId) == 0);
 	TableIds tableIDs(DivideRoundUp(tableIdsSize, sizeof(TableId)), 0);
 
-	const size_t bytesWritten = pEnumSystemFirmwareTables(provider, &tableIDs[0], tableIdsSize);
+	const size_t bytesWritten = pEnumSystemFirmwareTables(provider, &tableIDs[0], (DWORD)tableIdsSize);
 	debug_assert(bytesWritten == tableIdsSize);
 
 	return tableIDs;
@@ -39,7 +39,7 @@ Table GetTable(Provider provider, TableId tableId)
 	}
 
 	Table table(tableSize, 0);
-	const size_t bytesWritten = pGetSystemFirmwareTable(provider, tableId, &table[0], tableSize);
+	const size_t bytesWritten = pGetSystemFirmwareTable(provider, tableId, &table[0], (DWORD)tableSize);
 	debug_assert(bytesWritten == tableSize);
 
 	return table;

source/lib/sysdep/os/win/whrt/counter.cpp

@@ -27,7 +27,7 @@
 #include "precompiled.h"
 #include "lib/sysdep/os/win/whrt/counter.h"
 
-#include "lib/bits.h"
+#include "lib/alignment.h"
 #include "lib/sysdep/cpu.h"	// cpu_CAS
 
 #include "lib/sysdep/os/win/whrt/tsc.h"
@@ -98,7 +98,7 @@ ICounter* CreateCounter(size_t id)
 
 	static const size_t memSize = 200;
 	static u8 mem[memSize];
-	u8* alignedMem = (u8*)round_up((uintptr_t)mem, (uintptr_t)16u);
+	u8* alignedMem = (u8*)Align<16>((uintptr_t)mem);
 	const size_t bytesLeft = mem+memSize - alignedMem;
 	ICounter* counter = ConstructCounterAt(id, alignedMem, bytesLeft);
 

source/lib/sysdep/os/win/whrt/whrt.cpp

@@ -33,7 +33,7 @@
 #include "lib/sysdep/os/win/win.h"
 #include "lib/sysdep/os/win/winit.h"
 #include "lib/sysdep/acpi.h"
-#include "lib/adts.h"
+//#include "lib/adts.h"
 #include "lib/bits.h"
 
 #include "lib/sysdep/os/win/whrt/counter.h"

source/lib/sysdep/os/win/wiocp.cpp

@@ -0,0 +1,32 @@
+#include "precompiled.h"
+#include "lib/sysdep/os/win/wiocp.h"
+
+#include "lib/file/file.h"	// ERR::IO
+#include "lib/sysdep/os/win/wutil.h"
+
+
+void AttachToCompletionPort(HANDLE hFile, HANDLE& hIOCP, ULONG_PTR key, DWORD numConcurrentThreads)
+{
+	WinScopedPreserveLastError s;	// CreateIoCompletionPort
+
+	// (when called for the first time, ends up creating hIOCP)
+	hIOCP = CreateIoCompletionPort(hFile, hIOCP, key, numConcurrentThreads);
+	debug_assert(wutil_IsValidHandle(hIOCP));
+}
+
+
+LibError PollCompletionPort(HANDLE hIOCP, DWORD timeout, DWORD& bytesTransferred, ULONG_PTR& key, OVERLAPPED*& ovl)
+{
+	if(hIOCP == 0)
+		return ERR::INVALID_HANDLE;	// NOWARN (happens if called before the first Attach)
+
+	bytesTransferred = 0;
+	key = 0;
+	ovl = 0;
+	if(GetQueuedCompletionStatus(hIOCP, &bytesTransferred, &key, &ovl, timeout))
+		return INFO::OK;
+	if(GetLastError() == WAIT_TIMEOUT)
+		return ERR::AGAIN;	// NOWARN (nothing pending)
+	else
+		return ERR::FAIL;	// NOWARN (let caller decide what to do)
+}

source/lib/sysdep/os/win/wiocp.h

@@ -0,0 +1,50 @@
+/* Copyright (c) 2010 Wildfire Games
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * I/O completion port
+ */
+
+#ifndef INCLUDED_WIOCP
+#define INCLUDED_WIOCP
+
+#include "lib/sysdep/os/win/win.h"
+
+// this appears to be the best solution for IO notification.
+// there are three alternatives:
+// - multiple threads with blocking I/O. this is rather inefficient when
+//   many directories (e.g. mods) are being watched.
+// - normal overlapped I/O: build a contiguous array of the hEvents
+//   in all OVERLAPPED structures, and WaitForMultipleObjects.
+//   it would be cumbersome to update this array when adding/removing watches.
+// - callback notification: a notification function is called when the thread
+//   that initiated the I/O (ReadDirectoryChangesW) enters an alertable
+//   wait state. it is desirable for notifications to arrive at a single
+//   known point - see dir_watch_Poll. however, other APIs might also
+//   trigger APC delivery.
+
+// @param hIOCP 0 to create a new port
+extern void AttachToCompletionPort(HANDLE hFile, HANDLE& hIOCP, ULONG_PTR key, DWORD numConcurrentThreads = 0);
+
+extern LibError PollCompletionPort(HANDLE hIOCP, DWORD timeout, DWORD& bytesTransferred, ULONG_PTR& key, OVERLAPPED*& ovl);
+
+#endif	// #ifndef INCLUDED_WIOCP

source/lib/sysdep/os/win/wnuma.cpp

@@ -23,7 +23,7 @@
 #include "precompiled.h"
 #include "lib/sysdep/numa.h"
 
-#include "lib/bits.h"	// round_up, PopulationCount
+#include "lib/bits.h"	// PopulationCount
 #include "lib/timer.h"
 #include "lib/module_init.h"
 #include "lib/allocators/allocators.h"	// page_aligned_alloc

source/lib/sysdep/os/win/wposix/waio.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -24,105 +24,312 @@
  * emulate POSIX asynchronous I/O on Windows.
  */
 
+// NB: this module is significantly faster than Intel's aio library,
+// which also returns ERROR_INVALID_PARAMETER from aio_error if the
+// file is opened with FILE_FLAG_OVERLAPPED. (it looks like they are
+// using threaded blocking IO)
+
 #include "precompiled.h"
 #include "lib/sysdep/os/win/wposix/waio.h"
 
-#include <map>
-
-#include "lib/sysdep/os/win/wposix/crt_posix.h"      // correct definitions of _open() etc.
+#include "lib/alignment.h"	// IsAligned
+#include "lib/module_init.h"
+#include "lib/sysdep/cpu.h"	// cpu_AtomicAdd
 #include "lib/sysdep/filesystem.h"	// O_NO_AIO_NP
+#include "lib/sysdep/os/win/wutil.h"	// wutil_SetPrivilege
+#include "lib/sysdep/os/win/wiocp.h"
+#include "lib/sysdep/os/win/winit.h"
 
-#include "lib/bits.h"	// IsAligned
-#include "lib/sysdep/os/win/wposix/wposix_internal.h"
-#include "lib/sysdep/os/win/wposix/wtime.h"          // timespec
-
-
-WINIT_REGISTER_MAIN_INIT(waio_Init);
 WINIT_REGISTER_MAIN_SHUTDOWN(waio_Shutdown);
 
-// note: we assume sector sizes no larger than a page.
-// (GetDiskFreeSpace allows querying the actual size, but we'd
-// have to do so for all drives, and that may change depending on whether
-// there is a DVD in the drive or not)
-// sector size is relevant because Windows aio requires all IO
-// buffers, offsets and lengths to be a multiple of it. this requirement
-// is also carried over into the vfs / file.cpp interfaces for efficiency
-// (avoids the need for copying to/from align buffers).
-const uintptr_t sectorSize = 0x1000;
+// (dynamic linking preserves compatibility with previous Windows versions)
+static WUTIL_FUNC(pSetFileCompletionNotificationModes, BOOL, (HANDLE, UCHAR));
+static WUTIL_FUNC(pSetFileIoOverlappedRange, BOOL, (HANDLE, PUCHAR, ULONG));
+static WUTIL_FUNC(pSetFileValidData, BOOL, (HANDLE, LONGLONG));
+
+// (there must be one global IOCP because aio_suspend might be called for
+// requests from different files)
+static HANDLE hIOCP;
+
 
 //-----------------------------------------------------------------------------
+// OvlAllocator
 
-// note: the Windows lowio file descriptor limit is currrently 2048.
-
-/**
- * association between POSIX file descriptor and Win32 HANDLE.
- * NB: callers must ensure thread safety.
- **/
-class HandleManager
+// allocator for OVERLAPPED (enables a special optimization, see Associate)
+struct OvlAllocator	// POD
 {
-public:
-	/**
-	 * associate an aio handle with a file descriptor.
-	 **/
-	void Associate(int fd, HANDLE hFile)
+	// freelist entries for (padded) OVERLAPPED from our storage
+	struct Entry
 	{
-		debug_assert(fd > 2);
-		debug_assert(GetFileSize(hFile, 0) != INVALID_FILE_SIZE);
-		std::pair<Map::iterator, bool> ret = m_map.insert(std::make_pair(fd, hFile));
-		debug_assert(ret.second);	// fd better not already have been associated
+		SLIST_ENTRY entry;
+		OVERLAPPED ovl;
+	};
+
+	LibError Init()
+	{
+		// the allocation must be naturally aligned to ensure it doesn't
+		// overlap another page, which might prevent SetFileIoOverlappedRange
+		// from pinning the pages if one of them is PAGE_NOACCESS.
+		storage = _mm_malloc(storageSize, storageSize);
+		if(!storage)
+			WARN_RETURN(ERR::NO_MEM);
+		memset(storage, 0, storageSize);
+
+		InitializeSListHead(&freelist);
+
+		// storageSize provides more than enough OVERLAPPED, so we
+		// pad them to the cache line size to maybe avoid a few RFOs.
+		const size_t size = Align<cacheLineSize>(sizeof(Entry));
+		for(uintptr_t offset = 0; offset+size <= storageSize; offset += size)
+		{
+			Entry* entry = (Entry*)(uintptr_t(storage) + offset);
+			debug_assert(IsAligned(entry, MEMORY_ALLOCATION_ALIGNMENT));
+			InterlockedPushEntrySList(&freelist, &entry->entry);
+		}
+
+		extant = 0;
+
+		return INFO::OK;
 	}
 
-	void Dissociate(int fd)
+	void Shutdown()
 	{
-		const size_t numRemoved = m_map.erase(fd);
-		debug_assert(numRemoved == 1);
+		debug_assert(extant == 0);
+
+		InterlockedFlushSList(&freelist);
+
+		_mm_free(storage);
+		storage = 0;
 	}
 
-	bool IsAssociated(int fd) const
+	// irrevocably enable a special optimization for all I/Os requests
+	// concerning this file, ending when the file is closed. has no effect
+	// unless Vista+ and SeLockMemoryPrivilege are available.
+	void Associate(HANDLE hFile)
 	{
-		return m_map.find(fd) != m_map.end();
+		debug_assert(extant == 0);
+
+		// pin the page in kernel address space, which means our thread
+		// won't have to be the one to service the I/O, thus avoiding an APC.
+		// ("thread agnostic I/O")
+		if(pSetFileIoOverlappedRange)
+			WARN_IF_FALSE(pSetFileIoOverlappedRange(hFile, (PUCHAR)storage, storageSize));
 	}
 
-	/**
-	 * @return aio handle associated with file descriptor or
-	 * INVALID_HANDLE_VALUE if there is none.
-	 **/
-	HANDLE Get(int fd) const
+	// @return OVERLAPPED initialized for I/O starting at offset,
+	// or 0 if all available structures have already been allocated.
+	OVERLAPPED* Allocate(off_t offset)
 	{
-		Map::const_iterator it = m_map.find(fd);
-		if(it == m_map.end())
-			return INVALID_HANDLE_VALUE;
-		return it->second;
+		Entry* entry = (Entry*)InterlockedPopEntrySList(&freelist);
+		if(!entry)
+			return 0;
+
+		OVERLAPPED& ovl = entry->ovl;
+		ovl.Internal = 0;
+		ovl.InternalHigh = 0;
+		ovl.Offset = u64_lo(offset);
+		ovl.OffsetHigh = u64_hi(offset);
+		ovl.hEvent = 0;	// (notification is via IOCP and/or polling)
+
+		cpu_AtomicAdd(&extant, +1);
+
+		return &ovl;
 	}
 
-private:
-	typedef std::map<int, HANDLE> Map;
-	Map m_map;
+	void Deallocate(OVERLAPPED* ovl)
+	{
+		cpu_AtomicAdd(&extant, -1);
+
+		const uintptr_t address = uintptr_t(ovl);
+		debug_assert(uintptr_t(storage) <= address && address < uintptr_t(storage)+storageSize);
+		InterlockedPushEntrySList(&freelist, (PSLIST_ENTRY)(address - offsetof(Entry, ovl)));
+	}
+
+	// one 4 KiB page is enough for 64 OVERLAPPED per file (i.e. plenty).
+	static const size_t storageSize = pageSize;
+
+	void* storage;
+
+#if MSC_VERSION
+# pragma warning(push)
+# pragma warning(disable:4324)	// structure was padded due to __declspec(align())
+#endif
+	__declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) SLIST_HEADER freelist;
+#if MSC_VERSION
+# pragma warning(pop)
+#endif
+
+	volatile intptr_t extant;
 };
 
-static HandleManager* handleManager;
 
+//-----------------------------------------------------------------------------
+// FileControlBlock
 
-// do we want to open a second aio-capable handle?
-static bool IsAioPossible(int fd, bool is_com_port, int oflag)
+// (must correspond to static zero-initialization of fd)
+static const intptr_t FD_AVAILABLE = 0;
+
+// information required to start asynchronous I/Os from a file
+// (aiocb stores a pointer to the originating FCB)
+struct FileControlBlock // POD
 {
-	// stdin/stdout/stderr
-	if(fd <= 2)
-		return false;
+	// search key, indicates the file descriptor with which this
+	// FCB was associated (or FD_AVAILABLE if none).
+	volatile intptr_t fd;
 
-	// COM port - we don't currently need aio access for those, and
-	// aio_reopen's CreateFileW would fail with "access denied".
-	if(is_com_port)
-		return false;
+	// second aio-enabled handle from waio_reopen
+	HANDLE hFile;
 
-	// caller is requesting we skip it (see open())
-	if(oflag & O_NO_AIO_NP)
-		return false;
+	OvlAllocator ovl;
 
-	return true;
+	LibError Init()
+	{
+		fd = FD_AVAILABLE;
+		hFile = INVALID_HANDLE_VALUE;
+		return ovl.Init();
+	}
+
+	void Shutdown()
+	{
+		debug_assert(fd == FD_AVAILABLE);
+		debug_assert(hFile == INVALID_HANDLE_VALUE);
+		ovl.Shutdown();
+	}
+};
+
+
+// NB: the Windows lowio file descriptor limit is 2048, but
+// our applications rarely open more than a few files at a time.
+static FileControlBlock fileControlBlocks[16];
+
+
+static FileControlBlock* AssociateFileControlBlock(int fd, HANDLE hFile)
+{
+	for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
+	{
+		FileControlBlock& fcb = fileControlBlocks[i];
+		if(cpu_CAS(&fcb.fd, FD_AVAILABLE, fd))	// the FCB is now ours
+		{
+			fcb.hFile = hFile;
+			fcb.ovl.Associate(hFile);
+
+			AttachToCompletionPort(hFile, hIOCP, (ULONG_PTR)&fcb);
+
+			// minor optimization: avoid posting to IOCP in rare cases
+			// where the I/O completes synchronously
+			if(pSetFileCompletionNotificationModes)
+			{
+				// (for reasons as yet unknown, this fails when the file is
+				// opened for read-only access)
+				(void)pSetFileCompletionNotificationModes(fcb.hFile, FILE_SKIP_COMPLETION_PORT_ON_SUCCESS);
+			}
+
+			return &fcb;
+		}
+	}
+
+	return 0;
 }
 
 
+static void DissociateFileControlBlock(FileControlBlock* fcb)
+{
+	fcb->hFile = INVALID_HANDLE_VALUE;
+	fcb->fd = FD_AVAILABLE;
+}
+
+
+static FileControlBlock* FindFileControlBlock(int fd)
+{
+	debug_assert(fd != FD_AVAILABLE);
+
+	for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
+	{
+		FileControlBlock& fcb = fileControlBlocks[i];
+		if(fcb.fd == fd)
+			return &fcb;
+	}
+
+	return 0;
+}
+
+
+//-----------------------------------------------------------------------------
+// init/shutdown
+
+static ModuleInitState waio_initState;
+
+static LibError waio_Init()
+{
+	for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
+		fileControlBlocks[i].Init();
+
+	WUTIL_IMPORT_KERNEL32(SetFileCompletionNotificationModes, pSetFileCompletionNotificationModes);
+
+	// NB: using these functions when the privileges are not available would
+	// trigger warnings. since callers have to check the function pointers
+	// anyway, just refrain from setting them in such cases.
+
+	if(wutil_SetPrivilege(L"SeLockMemoryPrivilege", true) == INFO::OK)
+		WUTIL_IMPORT_KERNEL32(SetFileIoOverlappedRange, pSetFileIoOverlappedRange);
+
+	if(wutil_SetPrivilege(L"SeManageVolumePrivilege", true) == INFO::OK)
+		WUTIL_IMPORT_KERNEL32(SetFileValidData, pSetFileValidData);
+
+	return INFO::OK;
+}
+
+
+static LibError waio_Shutdown()
+{
+	if(waio_initState == 0)	// we were never initialized
+		return INFO::OK;
+
+	for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++)
+		fileControlBlocks[i].Shutdown();
+
+	WARN_IF_FALSE(CloseHandle(hIOCP));
+
+	return INFO::OK;
+}
+
+
+//-----------------------------------------------------------------------------
+// OpenFile
+
+static DWORD DesiredAccess(int oflag)
+{
+	switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR))
+	{
+	case O_RDONLY:
+		// (WinXP x64 requires FILE_WRITE_ATTRIBUTES for SetFileCompletionNotificationModes)
+		return GENERIC_READ|FILE_WRITE_ATTRIBUTES;
+	case O_WRONLY:
+		return GENERIC_WRITE;
+	case O_RDWR:
+		return GENERIC_READ|GENERIC_WRITE;
+	default:
+		DEBUG_WARN_ERR(ERR::INVALID_PARAM);
+		return 0;
+	}
+}
+
+static DWORD ShareMode(int oflag)
+{
+	switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR))
+	{
+	case O_RDONLY:
+		return FILE_SHARE_READ;
+	case O_WRONLY:
+		return FILE_SHARE_WRITE;
+	case O_RDWR:
+		return FILE_SHARE_READ|FILE_SHARE_WRITE;
+	default:
+		DEBUG_WARN_ERR(ERR::INVALID_PARAM);
+		return 0;
+	}
+}
+
 static DWORD CreationDisposition(int oflag)
 {
 	if(oflag & O_CREAT)
@@ -134,68 +341,68 @@ static DWORD CreationDisposition(int oflag)
 	return OPEN_EXISTING;
 }
 
-
-// (re)open file in asynchronous mode and associate handle with fd.
-// (this works because the files default to DENY_NONE sharing)
-LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...)
+static DWORD FlagsAndAttributes()
 {
-	WinScopedPreserveLastError s;	// CreateFile
+	// - FILE_FLAG_SEQUENTIAL_SCAN is ignored when FILE_FLAG_NO_BUFFERING
+	//   is set (c.f. "Windows via C/C++", p. 324)
+	// - FILE_FLAG_WRITE_THROUGH is ~5% slower (diskspd.cpp suggests it
+	//   disables hardware caching; the overhead may also be due to the
+	//   Windows cache manager)
+	const DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING;
+	const DWORD attributes = FILE_ATTRIBUTE_NORMAL;
+	return flags|attributes;
+}
 
-	debug_assert(!(oflag & O_APPEND));	// not supported
-	if(!IsAioPossible(fd, false, oflag))
-		return INFO::SKIPPED;
+static LibError OpenFile(const OsPath& pathname, int oflag, HANDLE& hFile)
+{
+	WinScopedPreserveLastError s;
 
-	DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_SEQUENTIAL_SCAN;
-
-	// decode file access mode
-	DWORD access, share;
-	switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR))
-	{
-	case O_RDONLY:
-		access = GENERIC_READ;
-		share = FILE_SHARE_READ;
-		break;
-
-	case O_WRONLY:
-		access = GENERIC_WRITE;
-		share = FILE_SHARE_WRITE;
-		flags |= FILE_FLAG_WRITE_THROUGH;
-		break;
-	
-	case O_RDWR:
-		access = GENERIC_READ|GENERIC_WRITE;
-		share = FILE_SHARE_READ|FILE_SHARE_WRITE;
-		flags |= FILE_FLAG_WRITE_THROUGH;
-		break;
-
-	default:
-		WARN_RETURN(ERR::INVALID_PARAM);
-	}
-
-	// open file
+	const DWORD access = DesiredAccess(oflag);
+	const DWORD share  = ShareMode(oflag);
 	const DWORD create = CreationDisposition(oflag);
-	const HANDLE hFile = CreateFileW(OsString(pathname).c_str(), access, share, 0, create, FILE_ATTRIBUTE_NORMAL|flags, 0);
+	const DWORD flags  = FlagsAndAttributes();
+	hFile = CreateFileW(OsString(pathname).c_str(), access, share, 0, create, flags, 0);
 	if(hFile == INVALID_HANDLE_VALUE)
 		return LibError_from_GLE();
 
+	return INFO::OK;
+}
+
+
+//-----------------------------------------------------------------------------
+// Windows-only APIs
+
+LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...)
+{
+	debug_assert(fd > 2);
+	debug_assert(!(oflag & O_APPEND));	// not supported
+
+	if(oflag & O_NO_AIO_NP)
+		return INFO::SKIPPED;
+
+	RETURN_ERR(ModuleInit(&waio_initState, waio_Init));
+
+	HANDLE hFile;
+	RETURN_ERR(OpenFile(pathname, oflag, hFile));
+
+	if(!AssociateFileControlBlock(fd, hFile))
 	{
-		WinScopedLock lock(WAIO_CS);
-		handleManager->Associate(fd, hFile);
+		CloseHandle(hFile);
+		WARN_RETURN(ERR::LIMIT);
 	}
+
 	return INFO::OK;
 }
 
 
 LibError waio_close(int fd)
 {
-	HANDLE hFile;
-	{
-		WinScopedLock lock(WAIO_CS);
-		if(!handleManager->IsAssociated(fd))	// wasn't opened for aio
-			return INFO::SKIPPED;
-		hFile = handleManager->Get(fd);
-		handleManager->Dissociate(fd);
-	}
+	FileControlBlock* fcb = FindFileControlBlock(fd);
+	if(!fcb)
+		WARN_RETURN(ERR::INVALID_HANDLE);
+	const HANDLE hFile = fcb->hFile;
+
+	DissociateFileControlBlock(fcb);
 
 	if(!CloseHandle(hFile))
 		WARN_RETURN(ERR::INVALID_HANDLE);
@@ -204,266 +411,106 @@ LibError waio_close(int fd)
 }
 
 
-// we don't want to #define read to _read, since that's a fairly common
-// identifier. therefore, translate from MS CRT names via thunk functions.
-// efficiency is less important, and the overhead could be optimized away.
-
-int read(int fd, void* buf, size_t nbytes)
+LibError waio_Preallocate(int fd, off_t alignedSize, off_t alignment)
 {
-	return _read(fd, buf, (int)nbytes);
-}
+	debug_assert(IsAligned(alignedSize, alignment));
 
-int write(int fd, void* buf, size_t nbytes)
-{
-	return _write(fd, buf, (int)nbytes);
-}
-
-off_t lseek(int fd, off_t ofs, int whence)
-{
-	return _lseeki64(fd, ofs, whence);
-}
-
-
-//-----------------------------------------------------------------------------
-
-class aiocb::Impl
-{
-public:
-	Impl()
-	{
-		m_hFile = INVALID_HANDLE_VALUE;
-
-		// (hEvent is initialized below and the rest in Issue(), but clear out
-		// any subsequently added fields)
-		memset(&m_overlapped, 0, sizeof(m_overlapped));
-
-		const BOOL manualReset = TRUE;
-		const BOOL initialState = FALSE;
-		m_overlapped.hEvent = CreateEvent(0, manualReset, initialState, 0);
-	}
-
-	~Impl()
-	{
-		CloseHandle(m_overlapped.hEvent);
-	}
-
-	LibError Issue(HANDLE hFile, off_t ofs, void* buf, size_t size, bool isWrite)
-	{
-		WinScopedPreserveLastError s;
-
-		m_hFile = hFile;
-
-		// note: Read-/WriteFile reset m_overlapped.hEvent, so we don't have to.
-		m_overlapped.Internal = m_overlapped.InternalHigh = 0;
-		m_overlapped.Offset     = u64_lo(ofs);
-		m_overlapped.OffsetHigh = u64_hi(ofs);
-
-		DWORD bytesTransferred;
-		BOOL ok;
-		if(isWrite)
-			ok = WriteFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped);
-		else
-			ok =  ReadFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped);
-		if(!ok && GetLastError() == ERROR_IO_PENDING)	// "pending" isn't an error
-		{
-			ok = TRUE;
-			SetLastError(0);
-		}
-		return LibError_from_win32(ok);
-	}
-
-	bool HasCompleted() const
-	{
-		// NB: .Internal "was originally reserved for system use and its behavior may change".
-		// besides 0 and STATUS_PENDING, I have seen the address of a pointer to a buffer.
-		return HasOverlappedIoCompleted(&m_overlapped);
-	}
-
-	// required for WaitForMultipleObjects
-	HANDLE Event() const
-	{
-		return m_overlapped.hEvent;
-	}
-
-	LibError GetResult(size_t* pBytesTransferred)
-	{
-		DWORD bytesTransferred;
-		const BOOL wait = FALSE;	// callers should wait until HasCompleted
-		if(!GetOverlappedResult(m_hFile, &m_overlapped, &bytesTransferred, wait))
-		{
-			*pBytesTransferred = 0;
-			return LibError_from_GLE();
-		}
-		else
-		{
-			*pBytesTransferred = bytesTransferred;
-			return INFO::OK;
-		}
-	}
-
-private:
-	OVERLAPPED m_overlapped;
-	HANDLE m_hFile;
-};
-
-
-// called by aio_read, aio_write, and lio_listio.
-// cb->aio_lio_opcode specifies desired operation.
-// @return LibError, also setting errno in case of failure.
-static LibError aio_issue(struct aiocb* cb)
-{
-	// no-op (probably from lio_listio)
-	if(!cb || cb->aio_lio_opcode == LIO_NOP)
-		return INFO::SKIPPED;
-
-	// extract aiocb fields for convenience
-	const bool isWrite = (cb->aio_lio_opcode == LIO_WRITE);
-	const int fd       = cb->aio_fildes;
-	const size_t size  = cb->aio_nbytes;
-	const off_t ofs    = cb->aio_offset;
-	void* const buf    = (void*)cb->aio_buf; // from volatile void*
-
-	// Win32 requires transfers to be sector-aligned.
-	if(!IsAligned(ofs, sectorSize) || !IsAligned(buf, sectorSize) || !IsAligned(size, sectorSize))
-	{
-		errno = EINVAL;
-		WARN_RETURN(ERR::INVALID_PARAM);
-	}
-
-	HANDLE hFile;
-	{
-		WinScopedLock lock(WAIO_CS);
-		hFile = handleManager->Get(fd);
-	}
-	if(hFile == INVALID_HANDLE_VALUE)
-	{
-		errno = EINVAL;
+	FileControlBlock* fcb = FindFileControlBlock(fd);
+	if(!fcb)
 		WARN_RETURN(ERR::INVALID_HANDLE);
-	}
+	const HANDLE hFile = fcb->hFile;
 
-	debug_assert(!cb->impl);	// SUSv3 requires that the aiocb not be in use
-	cb->impl.reset(new aiocb::Impl);
+	// allocate all space up front to reduce fragmentation
+	LARGE_INTEGER size64; size64.QuadPart = alignedSize;
+	WARN_IF_FALSE(SetFilePointerEx(hFile, size64, 0, FILE_BEGIN));
+	WARN_IF_FALSE(SetEndOfFile(hFile));
 
-	LibError ret = cb->impl->Issue(hFile, ofs, buf, size, isWrite);
-	if(ret < 0)
-	{
-		LibError_set_errno(ret);
-		return ret;
-	}
+	// avoid synchronous zero-fill (see discussion in header)
+	if(pSetFileValidData)
+		WARN_IF_FALSE(pSetFileValidData(hFile, alignedSize));
 
 	return INFO::OK;
 }
 
 
-// return status of transfer
-int aio_error(const struct aiocb* cb)
-{
-	return cb->impl->HasCompleted()? 0 : EINPROGRESS;
-}
+//-----------------------------------------------------------------------------
+// helper functions
 
-
-// get bytes transferred. call exactly once for each issued request.
-ssize_t aio_return(struct aiocb* cb)
+// called by aio_read, aio_write, and lio_listio.
+// cb->aio_lio_opcode specifies desired operation.
+// @return -1 on failure (having also set errno)
+static int Issue(aiocb* cb)
 {
-	// SUSv3 says we mustn't be callable before the request has completed
-	debug_assert(cb->impl);
-	debug_assert(cb->impl->HasCompleted());
-	size_t bytesTransferred;
-	LibError ret = cb->impl->GetResult(&bytesTransferred);
-	cb->impl.reset();	// disallow calling again, as required by SUSv3
-	if(ret < 0)
+	debug_assert(IsAligned(cb->aio_offset, maxSectorSize));
+	debug_assert(IsAligned(cb->aio_buf,    maxSectorSize));
+	debug_assert(IsAligned(cb->aio_nbytes, maxSectorSize));
+
+	FileControlBlock* fcb = FindFileControlBlock(cb->aio_fildes);
+	if(!fcb || fcb->hFile == INVALID_HANDLE_VALUE)
 	{
-		LibError_set_errno(ret);
-		return (ssize_t)-1;
-	}
-	return (ssize_t)bytesTransferred;
-}
-
-
-int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* ts)
-{
-	if(n <= 0 || n > MAXIMUM_WAIT_OBJECTS)
-	{
-		WARN_ERR(ERR::INVALID_PARAM);
+		DEBUG_WARN_ERR(ERR::INVALID_HANDLE);
 		errno = EINVAL;
 		return -1;
 	}
 
-	// build array of event handles
-	HANDLE hEvents[MAXIMUM_WAIT_OBJECTS];
-	size_t numPendingIos = 0;
+	debug_assert(!cb->fcb && !cb->ovl);	// SUSv3: aiocb must not be in use
+	cb->fcb = fcb;
+	cb->ovl = fcb->ovl.Allocate(cb->aio_offset);
+	if(!cb->ovl)
+	{
+		DEBUG_WARN_ERR(ERR::LIMIT);
+		errno = EMFILE;
+		return -1;
+	}
+
+	WinScopedPreserveLastError s;
+
+	const HANDLE hFile = fcb->hFile;
+	void* const buf = (void*)cb->aio_buf; // from volatile void*
+	const DWORD size = u64_lo(cb->aio_nbytes);
+	debug_assert(u64_hi(cb->aio_nbytes) == 0);
+	OVERLAPPED* ovl = (OVERLAPPED*)cb->ovl;
+	// (there is no point in using WriteFileGather/ReadFileScatter here
+	// because the IO manager still needs to lock pages and translate them
+	// into an MDL, and we'd just be increasing the number of addresses)
+	const BOOL ok = (cb->aio_lio_opcode == LIO_WRITE)? WriteFile(hFile, buf, size, 0, ovl) : ReadFile(hFile, buf, size, 0, ovl);
+	if(ok || GetLastError() == ERROR_IO_PENDING)
+		return 0;	// success
+
+	LibError_set_errno(LibError_from_GLE());
+	return -1;
+}
+
+
+static bool AreAnyComplete(const struct aiocb* const cbs[], int n)
+{
 	for(int i = 0; i < n; i++)
 	{
-		if(!cbs[i])	// SUSv3 says NULL entries are to be ignored
+		if(!cbs[i])	// SUSv3: must ignore NULL entries
 			continue;
 
-		aiocb::Impl* impl = cbs[i]->impl.get();
-		debug_assert(impl);
-		if(!impl->HasCompleted())
-			hEvents[numPendingIos++] = impl->Event();
+		if(HasOverlappedIoCompleted((OVERLAPPED*)cbs[i]->ovl))
+			return true;
 	}
-	if(!numPendingIos)	// done, don't need to suspend.
-		return 0;
 
-	const BOOL waitAll = FALSE;
-	// convert timespec to milliseconds (ts == 0 => no timeout)
-	const DWORD timeout = ts? (DWORD)(ts->tv_sec*1000 + ts->tv_nsec/1000000) : INFINITE;
-	const DWORD result = WaitForMultipleObjects((DWORD)numPendingIos, hEvents, waitAll, timeout);
-
-	for(size_t i = 0; i < numPendingIos; i++)
-		ResetEvent(hEvents[i]);
-
-	switch(result)
-	{
-	case WAIT_FAILED:
-		WARN_ERR(ERR::FAIL);
-		errno = EIO;
-		return -1;
-
-	case WAIT_TIMEOUT:
-		errno = EAGAIN;
-		return -1;
-
-	default:
-		return 0;
-	}
+	return false;
 }
 
 
-int aio_cancel(int fd, struct aiocb* cb)
-{
-	// Win32 limitation: can't cancel single transfers -
-	// all pending reads on this file are canceled.
-	UNUSED2(cb);
-
-	HANDLE hFile;
-	{
-		WinScopedLock lock(WAIO_CS);
-		hFile = handleManager->Get(fd);
-	}
-	if(hFile == INVALID_HANDLE_VALUE)
-	{
-		WARN_ERR(ERR::INVALID_HANDLE);
-		errno = EINVAL;
-		return -1;
-	}
-
-	WARN_IF_FALSE(CancelIo(hFile));
-	return AIO_CANCELED;
-}
-
+//-----------------------------------------------------------------------------
+// API
 
 int aio_read(struct aiocb* cb)
 {
 	cb->aio_lio_opcode = LIO_READ;
-	return (aio_issue(cb) < 0)? 0 : -1;
+	return Issue(cb);
 }
 
 
 int aio_write(struct aiocb* cb)
 {
 	cb->aio_lio_opcode = LIO_WRITE;
-	return (aio_issue(cb) < 0)? 0 : -1;
+	return Issue(cb);
 }
 
 
@@ -474,7 +521,10 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se)
 
 	for(int i = 0; i < n; i++)
 	{
-		if(aio_issue(cbs[i]) < 0)
+		if(cbs[i] == 0 || cbs[i]->aio_lio_opcode == LIO_NOP)
+			continue;
+
+		if(Issue(cbs[i]) == -1)
 			return -1;
 	}
 
@@ -485,24 +535,107 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se)
 }
 
 
+int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* timeout)
+{
+	// consume all pending notifications to prevent them from piling up if
+	// requests are always complete by the time we're called
+	DWORD bytesTransferred; ULONG_PTR key; OVERLAPPED* ovl;
+	while(PollCompletionPort(hIOCP, 0, bytesTransferred, key, ovl) == INFO::OK) {}
+
+	// avoid blocking if already complete (synchronous requests don't post notifications)
+	if(AreAnyComplete(cbs, n))
+		return 0;
+
+	// caller doesn't want to block, and no requests are complete
+	if(timeout && timeout->tv_sec == 0 && timeout->tv_nsec == 0)
+	{
+		errno = EAGAIN;
+		return -1;
+	}
+
+	// reduce CPU usage by blocking until a notification arrives or a
+	// brief timeout elapses (necessary because other threads - or even
+	// the above poll - might have consumed our notification). note that
+	// re-posting notifications that don't concern the respective requests
+	// is not desirable because POSIX doesn't require aio_suspend to be
+	// called, which means notifications might pile up.
+	const DWORD milliseconds = 1;	// as short as possible (don't oversleep)
+	const LibError ret = PollCompletionPort(hIOCP, milliseconds, bytesTransferred, key, ovl);
+	if(ret != INFO::OK && ret != ERR::AGAIN)	// failed
+	{
+		debug_assert(0);
+		return -1;
+	}
+
+	// scan again (even if we got a notification, it might not concern THESE requests)
+	if(AreAnyComplete(cbs, n))
+		return 0;
+
+	// none completed, must repeat the above steps. provoke being called again by
+	// claiming to have been interrupted by a signal.
+	errno = EINTR;
+	return -1;
+}
+
+
+int aio_error(const struct aiocb* cb)
+{
+	const OVERLAPPED* ovl = (const OVERLAPPED*)cb->ovl;
+	if(!ovl)	// called after aio_return
+		return EINVAL;
+	if(!HasOverlappedIoCompleted(ovl))
+		return EINPROGRESS;
+	if(ovl->Internal != ERROR_SUCCESS)
+		return EIO;
+	return 0;
+}
+
+
+ssize_t aio_return(struct aiocb* cb)
+{
+	FileControlBlock* fcb = (FileControlBlock*)cb->fcb;
+	OVERLAPPED* ovl = (OVERLAPPED*)cb->ovl;
+	if(!fcb || !ovl)
+	{
+		errno = EINVAL;
+		return -1;
+	}
+
+	const ULONG_PTR status = ovl->Internal;
+	const ULONG_PTR bytesTransferred = ovl->InternalHigh;
+
+	cb->ovl = 0;	// prevent further calls to aio_error/aio_return
+	COMPILER_FENCE;
+	fcb->ovl.Deallocate(ovl);
+	cb->fcb = 0;	// allow reuse
+
+	return (status == ERROR_SUCCESS)? bytesTransferred : -1;
+}
+
+
+int aio_cancel(int UNUSED(fd), struct aiocb* cb)
+{
+	// (faster than calling FindFileControlBlock)
+	const HANDLE hFile = ((const FileControlBlock*)cb->fcb)->hFile;
+	if(hFile == INVALID_HANDLE_VALUE)
+	{
+		WARN_ERR(ERR::INVALID_HANDLE);
+		errno = EINVAL;
+		return -1;
+	}
+
+	// cancel all I/Os this thread issued for the given file
+	// (CancelIoEx can cancel individual operations, but is only
+	// available starting with Vista)
+	WARN_IF_FALSE(CancelIo(hFile));
+
+	return AIO_CANCELED;
+}
+
+
 int aio_fsync(int, struct aiocb*)
 {
 	WARN_ERR(ERR::NOT_IMPLEMENTED);
 	errno = ENOSYS;
 	return -1;
 }
-
-
-//-----------------------------------------------------------------------------
-
-static LibError waio_Init()
-{
-	handleManager = new HandleManager;
-	return INFO::OK;
-}
-
-static LibError waio_Shutdown()
-{
-	delete handleManager;
-	return INFO::OK;
-}

source/lib/sysdep/os/win/wposix/waio.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010 Wildfire Games
+/* Copyright (c) 2011 Wildfire Games
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files (the
@@ -29,13 +29,12 @@
 
 #include "lib/lib_errors.h"
 #include "lib/os_path.h"
+#include "lib/posix/posix_time.h"	// timespec
 #include "lib/sysdep/os/win/wposix/wposix_types.h"
 
-#include "lib/sysdep/os/win/wposix/no_crt_posix.h"
-
 // Note: transfer buffers, offsets, and lengths must be sector-aligned
-// (we don't bother copying to an align buffer because the file cache
-// already requires splitting IOs into aligned blocks)
+// (we don't bother copying to an align buffer because our block cache
+// already requires splitting IOs into naturally-aligned blocks)
 
 
 //
@@ -57,15 +56,6 @@ struct sigevent	// unused
 };
 
 
-//
-// <unistd.h>
-//
-
-extern int read (int fd, void* buf, size_t nbytes);	// thunk
-extern int write(int fd, void* buf, size_t nbytes);	// thunk
-extern off_t lseek(int fd, off_t ofs, int whence);  // thunk
-
-
 //
 // <aio.h>
 //
@@ -80,8 +70,11 @@ struct aiocb
 	struct sigevent aio_sigevent;   // Signal number and value. (unused)
 	int             aio_lio_opcode; // Operation to be performed.
 
-	class Impl;
-	shared_ptr<Impl> impl;
+	// internal use only; must be zero-initialized before
+	// calling the first aio_read/aio_write/lio_listio (aio_return also
+	// zero-initializes them)
+	void* fcb;
+	void* ovl;
 };
 
 enum
@@ -101,21 +94,55 @@ enum
 	LIO_WRITE
 };
 
-extern int aio_cancel(int, struct aiocb*);
-extern int aio_error(const struct aiocb*);
-extern int aio_fsync(int, struct aiocb*);
 extern int aio_read(struct aiocb*);
-extern ssize_t aio_return(struct aiocb*);
-struct timespec;
-extern int aio_suspend(const struct aiocb* const[], int, const struct timespec*);
 extern int aio_write(struct aiocb*);
 extern int lio_listio(int, struct aiocb* const[], int, struct sigevent*);
 
-// for use by wfilesystem's wopen/wclose:
+// (if never called, IOCP notifications will pile up.)
+extern int aio_suspend(const struct aiocb* const[], int, const struct timespec*);
 
-// (re)open file in asynchronous mode and associate handle with fd.
-// (this works because the files default to DENY_NONE sharing)
+// @return status of transfer (0 or an errno)
+extern int aio_error(const struct aiocb*);
+
+// @return bytes transferred or -1 on error.
+// frees internal storage related to the request and MUST be called
+// exactly once for each aiocb after aio_error != EINPROGRESS.
+extern ssize_t aio_return(struct aiocb*);
+
+extern int aio_cancel(int, struct aiocb*);
+
+extern int aio_fsync(int, struct aiocb*);
+
+// Windows doesn't allow aio unless the file is opened in asynchronous mode,
+// which is not possible with _wsopen_s. since we don't want to have to
+// provide a separate File class for aio-enabled files, our wopen wrapper
+// will also call this function to open a SECOND handle to the file (works
+// because CRT open() defaults to DENY_NONE sharing). the CRT's lowio
+// descriptor table remains unaffected, but our [w]aio_* functions are
+// notified of the file descriptor, which means e.g. read and aio_read can
+// both be used. this function must have been called before any
+// other [w]aio_* functions are used.
 extern LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...);
+
+// close our second aio-enabled handle to the file (called from wclose).
 extern LibError waio_close(int fd);
 
+// call this before writing a large file to preallocate clusters, thus
+// reducing fragmentation.
+//
+// @param alignedSize must be a multiple of alignment (SetEndOfFile requires
+// sector alignment; this could be avoided by using the undocumented
+// NtSetInformationFile or SetFileInformationByHandle on Vista and later).
+// use wtruncate after I/O is complete to chop off any excess padding.
+//
+// NB: writes that extend a file (i.e. ALL WRITES when creating new files)
+// are synchronous, which prevents overlapping I/O and other work.
+// (http://support.microsoft.com/default.aspx?scid=kb%3Ben-us%3B156932)
+// if Windows XP and the SE_MANAGE_VOLUME_NAME privileges are available,
+// this function sets the valid data length to avoid the synchronous zero-fill.
+// note that this exposes the previous disk contents (possibly even to
+// other users since the waio_reopen design cannot deny file sharing) until
+// the application successfully writes to the file.
+LIB_API LibError waio_Preallocate(int fd, off_t alignedSize, off_t alignment);
+
 #endif	// #ifndef INCLUDED_WAIO

source/lib/sysdep/os/win/wposix/wfilesystem.cpp

@@ -373,6 +373,26 @@ int wclose(int fd)
 // unistd.h
 //-----------------------------------------------------------------------------
 
+// we don't want to #define read to _read, since that's a fairly common
+// identifier. therefore, translate from MS CRT names via thunk functions.
+// efficiency is less important, and the overhead could be optimized away.
+
+int read(int fd, void* buf, size_t nbytes)
+{
+	return _read(fd, buf, (int)nbytes);
+}
+
+int write(int fd, void* buf, size_t nbytes)
+{
+	return _write(fd, buf, (int)nbytes);
+}
+
+off_t lseek(int fd, off_t ofs, int whence)
+{
+	return _lseeki64(fd, ofs, whence);
+}
+
+
 int wtruncate(const OsPath& pathname, off_t length)
 {
 	HANDLE hFile = CreateFileW(OsString(pathname).c_str(), GENERIC_WRITE, 0, 0, OPEN_EXISTING, 0, 0);

source/lib/sysdep/os/win/wposix/wfilesystem.h

@@ -52,4 +52,13 @@ typedef unsigned int mode_t;	// defined by MinGW but not VC
 #define S_ISDIR(m) (m & S_IFDIR)
 #define S_ISREG(m) (m & S_IFREG)
 
+
+//
+// <unistd.h>
+//
+
+extern int read (int fd, void* buf, size_t nbytes);	// thunk
+extern int write(int fd, void* buf, size_t nbytes);	// thunk
+extern off_t lseek(int fd, off_t ofs, int whence);  // thunk
+
 #endif	// #ifndef INCLUDED_WFILESYSTEM

source/lib/sysdep/os/win/wsysdep.cpp

@@ -66,7 +66,7 @@ std::wstring sys_WideFromArgv(const char* argv_i)
 	const int inputSize = -1;	// null-terminated
 	std::vector<wchar_t> buf(strlen(argv_i)+1);	// (upper bound on number of characters)
 	// NB: avoid mbstowcs because it may specify another locale
-	const int ret = MultiByteToWideChar(cp, flags, argv_i, inputSize, &buf[0], buf.size());
+	const int ret = MultiByteToWideChar(cp, flags, argv_i, (int)inputSize, &buf[0], (int)buf.size());
 	debug_assert(ret != 0);
 	return std::wstring(&buf[0]);
 }

source/lib/sysdep/os/win/wutil.cpp

@@ -438,6 +438,32 @@ WinScopedDisableWow64Redirection::~WinScopedDisableWow64Redirection()
 }
 
 
+//-----------------------------------------------------------------------------
+
+LibError wutil_SetPrivilege(const wchar_t* privilege, bool enable)
+{
+	WinScopedPreserveLastError s;
+
+	HANDLE hToken;
+	if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES|TOKEN_QUERY, &hToken))
+		return ERR::_1;
+
+	TOKEN_PRIVILEGES tp;
+	if (!LookupPrivilegeValueW(NULL, privilege, &tp.Privileges[0].Luid))
+		return ERR::_2;
+	tp.PrivilegeCount = 1;
+	tp.Privileges[0].Attributes = enable? SE_PRIVILEGE_ENABLED : 0;
+
+	SetLastError(0);
+	const BOOL ok = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, 0, 0);
+	if(!ok || GetLastError() != 0)
+		return ERR::_3;
+
+	WARN_IF_FALSE(CloseHandle(hToken));
+	return INFO::OK;
+}
+
+
 //-----------------------------------------------------------------------------
 // module handle
 

source/lib/sysdep/os/win/wutil.h

@@ -77,7 +77,6 @@ extern void wutil_Free(void* p);
 // critical sections used by win-specific code
 enum WinLockId
 {
-	WAIO_CS,
 	WDBG_SYM_CS,	// protects (non-reentrant) dbghelp.dll
 	WDIR_WATCH_CS,
 
@@ -191,6 +190,8 @@ private:
 
 //-----------------------------------------------------------------------------
 
+LIB_API LibError wutil_SetPrivilege(const wchar_t* privilege, bool enable);
+
 /**
  * @return module handle of lib code (that of the main EXE if
  * linked statically, otherwise the DLL).

source/lib/sysdep/smbios.cpp

@@ -647,7 +647,7 @@ void FieldStringizer::operator()<const char*>(size_t flags, const char*& value,
 	if(lastChar == std::string::npos)	// nothing but spaces
 		return;
 	string.resize(lastChar+1);	// strip trailing spaces
-	if(string == "To Be Filled By O.E.M.")
+	if(!stricmp(value, "To Be Filled By O.E.M."))
 		return;
 
 	WriteName(name);

source/lib/tex/tex.cpp

@@ -33,6 +33,7 @@
 
 #include "lib/timer.h"
 #include "lib/bits.h"
+#include "lib/allocators/shared_ptr.h"
 #include "lib/sysdep/cpu.h"
 
 #include "tex_codec.h"
@@ -255,7 +256,8 @@ static LibError add_mipmaps(Tex* t, size_t w, size_t h, size_t bpp, void* newDat
 		WARN_RETURN(ERR::TEX_INVALID_SIZE);
 	t->flags |= TEX_MIPMAPS;	// must come before tex_img_size!
 	const size_t mipmap_size = tex_img_size(t);
-	shared_ptr<u8> mipmapData = io_Allocate(mipmap_size, 0);
+	shared_ptr<u8> mipmapData;
+	AllocateAligned(mipmapData, mipmap_size);
 	CreateLevelData cld = { bpp/8, w, h, (const u8*)newData, data_size };
 	tex_util_foreach_mipmap(w, h, bpp, mipmapData.get(), 0, 1, create_level, &cld);
 	t->data = mipmapData;
@@ -332,7 +334,8 @@ TIMER_ACCRUE(tc_plain_transform);
 	//
 	// this is necessary even when not flipping because the initial data
 	// is read-only.
-	shared_ptr<u8> newData = io_Allocate(new_data_size);
+	shared_ptr<u8> newData;
+	AllocateAligned(newData, new_data_size);
 
 	// setup row source/destination pointers (simplifies outer loop)
 	u8* dst = (u8*)newData.get();

source/lib/tex/tex_dds.cpp

@@ -27,12 +27,15 @@
 #include "precompiled.h"
 
 #include "lib/byte_order.h"
-#include "tex_codec.h"
 #include "lib/bits.h"
 #include "lib/timer.h"
+#include "lib/allocators/shared_ptr.h"
+#include "tex_codec.h"
+
 
 // NOTE: the convention is bottom-up for DDS, but there's no way to tell.
 
+
 //-----------------------------------------------------------------------------
 // S3TC decompression
 //-----------------------------------------------------------------------------
@@ -279,7 +282,8 @@ static LibError s3tc_decompress(Tex* t)
 	const size_t dxt = t->flags & TEX_DXT;
 	const size_t out_bpp = (dxt != 1)? 32 : 24;
 	const size_t out_size = tex_img_size(t) * out_bpp / t->bpp;
-	shared_ptr<u8> decompressedData = io_Allocate(out_size);
+	shared_ptr<u8> decompressedData;
+	AllocateAligned(decompressedData, out_size, pageSize);
 
 	const size_t s3tc_block_size = (dxt == 3 || dxt == 5)? 16 : 8;
 	S3tcDecompressInfo di = { dxt, s3tc_block_size, out_bpp/8, decompressedData.get() };

source/lib/tex/tex_internal.h

@@ -28,7 +28,7 @@
 #define INCLUDED_TEX_INTERNAL
 
 #include "lib/allocators/dynarray.h"
-#include "lib/file/io/io.h"	// io_Allocate
+#include "lib/file/io/io.h"	// io::Allocate
 
 /**
  * check if the given texture format is acceptable: 8bpp grey,

source/lib/tex/tex_jpg.cpp

@@ -26,10 +26,12 @@
 
 #include "precompiled.h"
 
+#include <setjmp.h>
+
 #include "lib/external_libraries/libjpeg.h"
+#include "lib/allocators/shared_ptr.h"
 
 #include "tex_codec.h"
-#include <setjmp.h>
 
 
 // squelch "dtor / setjmp interaction" warnings.
@@ -478,7 +480,8 @@ static LibError jpg_decode_impl(DynArray* da, jpeg_decompress_struct* cinfo, Tex
 	// alloc destination buffer
 	const size_t pitch = w * bpp / 8;
 	const size_t img_size = pitch * h;	// for allow_rows
-	shared_ptr<u8> data = io_Allocate(img_size);
+	shared_ptr<u8> data;
+	AllocateAligned(data, img_size, pageSize);
 
 	// read rows
 	shared_ptr<RowPtr> rows = tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0);

source/lib/tex/tex_png.cpp

@@ -30,6 +30,7 @@
 
 #include "lib/byte_order.h"
 #include "tex_codec.h"
+#include "lib/allocators/shared_ptr.h"
 #include "lib/timer.h"
 
 #if MSC_VERSION
@@ -111,7 +112,8 @@ static LibError png_decode_impl(DynArray* da, png_structp png_ptr, png_infop inf
 		WARN_RETURN(ERR::TEX_INVALID_COLOR_TYPE);
 
 	const size_t img_size = pitch * h;
-	shared_ptr<u8> data = io_Allocate(img_size);
+	shared_ptr<u8> data;
+	AllocateAligned(data, img_size, pageSize);
 
 	shared_ptr<RowPtr> rows = tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0);
 	png_read_image(png_ptr, (png_bytepp)rows.get());

source/ps/ConfigDB.cpp

@@ -17,17 +17,18 @@
 
 #include "precompiled.h"
 
+#include <boost/algorithm/string.hpp>
+
 #include "Pyrogenesis.h"
 #include "Parser.h"
 #include "ConfigDB.h"
 #include "CLogger.h"
 #include "Filesystem.h"
 #include "scripting/ScriptingHost.h"
+#include "lib/allocators/shared_ptr.h"
 
 #include "scriptinterface/ScriptInterface.h"
 
-#include <boost/algorithm/string.hpp>
-
 typedef std::map <CStr, CConfigValueSet> TConfigMap;
 TConfigMap CConfigDB::m_Map[CFG_LAST];
 VfsPath CConfigDB::m_ConfigFile[CFG_LAST];
@@ -425,7 +426,8 @@ bool CConfigDB::WriteFile(EConfigNamespace ns, const VfsPath& path)
 		return false;
 	}
 
-	shared_ptr<u8> buf = io_Allocate(1*MiB);
+	shared_ptr<u8> buf;
+	AllocateAligned(buf, 1*MiB, maxSectorSize);
 	char* pos = (char*)buf.get();
 	TConfigMap &map=m_Map[ns];
 	for(TConfigMap::const_iterator it = map.begin(); it != map.end(); ++it)

source/ps/Util.cpp

@@ -33,7 +33,6 @@
 #include "lib/sysdep/arch/x86_x64/topology.h"
 #include "lib/sysdep/smbios.h"
 #include "lib/tex/tex.h"
-#include "lib/file/io/io_align.h"	// BLOCK_SIZE
 
 #include "ps/GameSetup/Config.h"
 #include "ps/GameSetup/GameSetup.h"
@@ -207,7 +206,6 @@ LibError tex_write(Tex* t, const VfsPath& filename)
 	// write to disk
 	LibError ret = INFO::OK;
 	{
-		(void)da_set_size(&da, round_up(da.cur_size, BLOCK_SIZE));
 		shared_ptr<u8> file = DummySharedPtr(da.base);
 		const ssize_t bytes_written = g_VFS->CreateFile(filename, file, da.pos);
 		if(bytes_written > 0)
@@ -254,7 +252,8 @@ void WriteScreenshot(const VfsPath& extension)
 
 	const size_t img_size = w * h * bpp/8;
 	const size_t hdr_size = tex_hdr_size(filename);
-	shared_ptr<u8> buf = io_Allocate(hdr_size+img_size);
+	shared_ptr<u8> buf;
+	AllocateAligned(buf, hdr_size+img_size, maxSectorSize);
 	GLvoid* img = buf.get() + hdr_size;
 	Tex t;
 	if(tex_wrap(w, h, bpp, flags, buf, hdr_size, &t) < 0)
@@ -311,7 +310,8 @@ void WriteBigScreenshot(const VfsPath& extension, int tiles)
 	void* tile_data = malloc(tile_size);
 	if(!tile_data)
 		WARN_ERR_RETURN(ERR::NO_MEM);
-	shared_ptr<u8> img_buf = io_Allocate(hdr_size+img_size);
+	shared_ptr<u8> img_buf;
+	AllocateAligned(img_buf, hdr_size+img_size, maxSectorSize);
 
 	Tex t;
 	GLvoid* img = img_buf.get() + hdr_size;

source/ps/XML/XMLWriter.cpp

@@ -23,6 +23,7 @@
 #include "ps/Filesystem.h"
 #include "ps/XML/Xeromyces.h"
 #include "lib/utf8.h"
+#include "lib/allocators/shared_ptr.h"
 #include "lib/sysdep/cpu.h"
 #include "maths/Fixed.h"
 
@@ -95,7 +96,8 @@ bool XMLWriter_File::StoreVFS(const PIVFS& vfs, const VfsPath& pathname)
 	if (m_LastElement) debug_warn(L"ERROR: Saving XML while an element is still open");
 
 	const size_t size = m_Data.length();
-	shared_ptr<u8> data = io_Allocate(size);
+	shared_ptr<u8> data;
+	AllocateAligned(data, size, maxSectorSize);
 	memcpy(data.get(), m_Data.data(), size);
 	LibError ret = vfs->CreateFile(pathname, data, size);
 	if (ret < 0)

source/renderer/Renderer.cpp

@@ -32,6 +32,7 @@
 
 #include "lib/bits.h"	// is_pow2
 #include "lib/res/graphics/ogl_tex.h"
+#include "lib/allocators/shared_ptr.h"
 #include "maths/Matrix3D.h"
 #include "maths/MathUtil.h"
 #include "ps/CLogger.h"
@@ -1773,7 +1774,8 @@ int CRenderer::LoadAlphaMaps()
 	size_t tile_w = 2+base+2;	// 2 pixel border (avoids bilinear filtering artifacts)
 	size_t total_w = round_up_to_pow2(tile_w * NumAlphaMaps);
 	size_t total_h = base; debug_assert(is_pow2(total_h));
-	shared_ptr<u8> data = io_Allocate(total_w*total_h*3);
+	shared_ptr<u8> data;
+	AllocateAligned(data, total_w*total_h*3, maxSectorSize);
 	// for each tile on row
 	for(size_t i=0;i<NumAlphaMaps;i++)
 	{

source/simulation2/components/CCmpAIManager.cpp

@@ -25,6 +25,7 @@
 #include "graphics/Terrain.h"
 #include "lib/timer.h"
 #include "lib/tex/tex.h"
+#include "lib/allocators/shared_ptr.h"
 #include "ps/CLogger.h"
 #include "ps/Filesystem.h"
 #include "ps/Util.h"
@@ -130,7 +131,8 @@ private:
 
 			const size_t img_size = w * h * bpp/8;
 			const size_t hdr_size = tex_hdr_size(filename);
-			shared_ptr<u8> buf = io_Allocate(hdr_size+img_size);
+			shared_ptr<u8> buf;
+			AllocateAligned(buf, hdr_size+img_size, maxSectorSize);
 			Tex t;
 			if (tex_wrap(w, h, bpp, flags, buf, hdr_size, &t) < 0)
 				return;