move allocators here (the old cache_allocator is now in headerless.cpp)

This was SVN commit r5438.

source/lib/allocators/allocators.cpp

@@ -0,0 +1,148 @@
+/**
+ * =========================================================================
+ * File        : allocators.cpp
+ * Project     : 0 A.D.
+ * Description : memory suballocators.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "allocators.h"
+
+#include "lib/sysdep/cpu.h"	// cpu_CAS
+#include "lib/bits.h"
+
+#include "mem_util.h"
+
+
+//-----------------------------------------------------------------------------
+// page aligned allocator
+//-----------------------------------------------------------------------------
+
+void* page_aligned_alloc(size_t unaligned_size)
+{
+	const size_t size_pa = mem_RoundUpToPage(unaligned_size);
+	u8* p = 0;
+	RETURN0_IF_ERR(mem_Reserve(size_pa, &p));
+	RETURN0_IF_ERR(mem_Commit(p, size_pa, PROT_READ|PROT_WRITE));
+	return p;
+}
+
+
+void page_aligned_free(void* p, size_t unaligned_size)
+{
+	if(!p)
+		return;
+	debug_assert(mem_IsPageMultiple((uintptr_t)p));
+	const size_t size_pa = mem_RoundUpToPage(unaligned_size);
+	(void)mem_Release((u8*)p, size_pa);
+}
+
+
+//-----------------------------------------------------------------------------
+// matrix allocator
+//-----------------------------------------------------------------------------
+
+void** matrix_alloc(uint cols, uint rows, size_t el_size)
+{
+	const size_t initial_align = 64;
+	// note: no provision for padding rows. this is a bit more work and
+	// if el_size isn't a power-of-2, performance is going to suck anyway.
+	// otherwise, the initial alignment will take care of it.
+
+	const size_t ptr_array_size = cols*sizeof(void*);
+	const size_t row_size = cols*el_size;
+	const size_t data_size = rows*row_size;
+	const size_t total_size = ptr_array_size + initial_align + data_size;
+
+	void* p = malloc(total_size);
+	if(!p)
+		return 0;
+
+	uintptr_t data_addr = (uintptr_t)p + ptr_array_size + initial_align;
+	data_addr -= data_addr % initial_align;
+
+	// alignment check didn't set address to before allocation
+	debug_assert(data_addr >= (uintptr_t)p+ptr_array_size);
+
+	void** ptr_array = (void**)p;
+	for(uint i = 0; i < cols; i++)
+	{
+		ptr_array[i] = (void*)data_addr;
+		data_addr += row_size;
+	}
+
+	// didn't overrun total allocation
+	debug_assert(data_addr <= (uintptr_t)p+total_size);
+
+	return ptr_array;
+}
+
+
+void matrix_free(void** matrix)
+{
+	free(matrix);
+}
+
+
+//-----------------------------------------------------------------------------
+// allocator optimized for single instances
+//-----------------------------------------------------------------------------
+
+void* single_calloc(void* storage, volatile uintptr_t* in_use_flag, size_t size)
+{
+	// sanity check
+	debug_assert(*in_use_flag == 0 || *in_use_flag == 1);
+
+	void* p;
+
+	// successfully reserved the single instance
+	if(cpu_CAS(in_use_flag, 0, 1))
+		p = storage;
+	// already in use (rare) - allocate from heap
+	else
+		p = new u8[size];
+
+	memset(p, 0, size);
+	return p;
+}
+
+
+void single_free(void* storage, volatile uintptr_t* in_use_flag, void* p)
+{
+	// sanity check
+	debug_assert(*in_use_flag == 0 || *in_use_flag == 1);
+
+	if(p == storage)
+	{
+		if(cpu_CAS(in_use_flag, 1, 0))
+		{
+			// ok, flag has been reset to 0
+		}
+		else
+			debug_warn("in_use_flag out of sync (double free?)");
+	}
+	// was allocated from heap
+	else
+	{
+		// single instance may have been freed by now - cannot assume
+		// anything about in_use_flag.
+
+		delete[] (u8*)p;
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// static allocator
+//-----------------------------------------------------------------------------
+
+void* static_calloc(StaticStorage* ss, size_t size)
+{
+	void* p = (void*)round_up((uintptr_t)ss->pos, 16);
+	ss->pos = (u8*)p+size;
+	debug_assert(ss->pos <= ss->end);
+	return p;
+}

source/lib/allocators/allocators.h

@@ -0,0 +1,349 @@
+/**
+ * =========================================================================
+ * File        : allocators.h
+ * Project     : 0 A.D.
+ * Description : memory suballocators.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_ALLOCATORS
+#define INCLUDED_ALLOCATORS
+
+#include <map>
+
+#include "lib/posix/posix_mman.h"	// PROT_*
+#include "lib/sysdep/cpu.h"	// cpu_CAS
+
+
+//
+// page aligned allocator
+//
+
+/**
+ * allocate memory aligned to the system page size.
+ *
+ * this is useful for file_cache_alloc, which uses this allocator to
+ * get sector-aligned (hopefully; see sys_max_sector_size) IO buffers.
+ *
+ * note that this allocator is stateless and very litte error checking
+ * can be performed.
+ *
+ * the memory is initially writable and you can use mprotect to set other
+ * access permissions if desired.
+ *
+ * @param unaligned_size minimum size [bytes] to allocate.
+ * @return page-aligned and -padded memory or 0 on error / out of memory.
+ **/
+extern void* page_aligned_alloc(size_t unaligned_size);
+
+/**
+ * free a previously allocated page-aligned region.
+ *
+ * @param p exact value returned from page_aligned_alloc
+ * @param size exact value passed to page_aligned_alloc
+ **/
+extern void page_aligned_free(void* p, size_t unaligned_size);
+
+
+// adapter that allows calling page_aligned_free as a boost::shared_ptr deleter.
+class PageAlignedDeleter
+{
+public:
+	PageAlignedDeleter(size_t size)
+		: m_size(size)
+	{
+		debug_assert(m_size != 0);
+	}
+
+	void operator()(u8* p)
+	{
+		debug_assert(m_size != 0);
+		page_aligned_free(p, m_size);
+		m_size = 0;
+	}
+
+private:
+	size_t m_size;
+};
+
+
+//
+// matrix allocator
+//
+
+/**
+ * allocate a 2D matrix accessible as matrix[col][row].
+ *
+ * takes care of the dirty work of allocating 2D matrices:
+ * - aligns data
+ * - only allocates one memory block, which is more efficient than
+ *   malloc/new for each row.
+ *
+ * @param cols, rows: dimension (cols x rows)
+ * @param el_size size [bytes] of a matrix cell
+ * @return 0 if out of memory, otherwise matrix that should be cast to
+ * type** (sizeof(type) == el_size). must be freed via matrix_free.
+ **/
+extern void** matrix_alloc(uint cols, uint rows, size_t el_size);
+
+/**
+ * free the given matrix.
+ *
+ * @param matrix allocated by matrix_alloc; no-op if 0.
+ * callers will likely want to pass variables of a different type
+ * (e.g. int**); they must be cast to void**.
+ **/
+extern void matrix_free(void** matrix);
+
+
+//-----------------------------------------------------------------------------
+// allocator optimized for single instances
+//-----------------------------------------------------------------------------
+
+/**
+ * Allocate <size> bytes of zeroed memory.
+ *
+ * intended for applications that frequently alloc/free a single
+ * fixed-size object. caller provides static storage and an in-use flag;
+ * we use that memory if available and otherwise fall back to the heap.
+ * if the application only has one object in use at a time, malloc is
+ * avoided; this is faster and avoids heap fragmentation.
+ *
+ * note: thread-safe despite use of shared static data.
+ *
+ * @param storage Caller-allocated memory of at least <size> bytes
+ * (typically a static array of bytes)
+ * @param in_use_flag Pointer to a flag we set when <storage> is in-use.
+ * @param size [bytes] to allocate
+ * @return allocated memory (typically = <storage>, but falls back to
+ * malloc if that's in-use), or 0 (with warning) if out of memory.
+ **/
+extern void* single_calloc(void* storage, volatile uintptr_t* in_use_flag, size_t size);
+
+/**
+ * Free a memory block that had been allocated by single_calloc.
+ *
+ * @param storage Exact value passed to single_calloc.
+ * @param in_use_flag Exact value passed to single_calloc.
+ * @param Exact value returned by single_calloc.
+ **/
+extern void single_free(void* storage, volatile uintptr_t* in_use_flag, void* p);
+
+#ifdef __cplusplus
+
+/**
+ * C++ wrapper on top of single_calloc that's slightly easier to use.
+ *
+ * T must be POD (Plain Old Data) because it is memset to 0!
+ **/
+template<class T> class SingleAllocator
+{
+	// evil but necessary hack: we don't want to instantiate a T directly
+	// because it may not have a default ctor. an array of uninitialized
+	// storage is used instead. single_calloc doesn't know about alignment,
+	// so we fix this by asking for an array of doubles.
+	double storage[(sizeof(T)+sizeof(double)-1)/sizeof(double)];
+	volatile uintptr_t is_in_use;
+
+public:
+	typedef T value_type;
+
+	SingleAllocator()
+	{
+		is_in_use = 0;
+	}
+
+	T* Allocate()
+	{
+		T* t = (T*)single_calloc(&storage, &is_in_use, sizeof(storage));
+		if(!t)
+			throw std::bad_alloc();
+		return t;
+	}
+
+	void Free(T* p)
+	{
+		single_free(&storage, &is_in_use, p);
+	}
+};
+
+#endif	// #ifdef __cplusplus
+
+
+//-----------------------------------------------------------------------------
+// static allocator
+//-----------------------------------------------------------------------------
+
+// dole out chunks of memory from storage reserved in the BSS.
+// freeing isn't necessary.
+
+/**
+ * opaque; initialized by STATIC_STORAGE and used by static_calloc
+ **/
+struct StaticStorage
+{
+	void* pos;
+	void* end;
+};
+
+// define <size> bytes of storage and prepare <name> for use with
+// static_calloc.
+// must be invoked from file or function scope.
+#define STATIC_STORAGE(name, size)\
+	static u8 storage[(size)];\
+	static StaticStorage name = { storage, storage+(size) }
+
+/*
+usage example:
+static Object* pObject;
+void InitObject()
+{
+	STATIC_STORAGE(ss, 100);	// includes padding
+	void* addr = static_calloc(ss, sizeof(Object));
+	pObject = new(addr) Object;
+}
+*/
+
+/**
+ * dole out memory from static storage reserved in BSS.
+ *
+ * this is useful for static objects that are used before _cinit - callers
+ * define static storage for one or several objects, use this function to
+ * retrieve an aligned pointer, then construct there via placement new.
+ *
+ * @param ss - initialized via STATIC_STORAGE
+ * @param size [bytes] to allocate
+ * @return aligned (suitable for any type) pointer
+ *
+ * raises a warning if there's not enough room (indicates incorrect usage)
+ **/
+extern void* static_calloc(StaticStorage* ss, size_t size);
+
+// (no need to free static_calloc-ed memory since it's in the BSS)
+
+
+//-----------------------------------------------------------------------------
+// OverrunProtector
+//-----------------------------------------------------------------------------
+
+/**
+OverrunProtector wraps an arbitrary object in DynArray memory and can detect
+inadvertent writes to it. this is useful for tracking down memory overruns.
+
+the basic idea is to require users to request access to the object and
+notify us when done; memory access permission is temporarily granted.
+(similar in principle to Software Transaction Memory).
+
+since this is quite slow, the protection is disabled unless
+CONFIG_OVERRUN_PROTECTION == 1; this avoids having to remove the
+wrapper code in release builds and re-write when looking for overruns.
+
+example usage:
+OverrunProtector<your_class> your_class_wrapper;
+..
+your_class* yc = your_class_wrapper.get();	// unlock, make ready for use
+if(!yc)			// your_class_wrapper's one-time alloc of a your_class-
+	abort();	// instance had failed - can't continue.
+doSomethingWith(yc);	// read/write access
+your_class_wrapper.lock();	// disallow further access until next .get()
+..
+**/
+#ifdef REDEFINED_NEW
+# include "lib/nommgr.h"
+#endif
+template<class T> class OverrunProtector
+{
+public:
+	OverrunProtector()
+	{
+		void* mem = page_aligned_alloc(sizeof(T));
+		object = new(mem) T();
+		lock();
+	}
+
+	~OverrunProtector()
+	{
+		unlock();
+		object->~T();	// call dtor (since we used placement new)
+		page_aligned_free(object, sizeof(T));
+		object = 0;
+	}
+
+	T* get()
+	{
+		unlock();
+		return object;
+	}
+
+	void lock()
+	{
+#if CONFIG_OVERRUN_PROTECTION
+		mprotect(object, sizeof(T), PROT_NONE);
+#endif
+	}
+
+private:
+	void unlock()
+	{
+#if CONFIG_OVERRUN_PROTECTION
+		mprotect(object, sizeof(T), PROT_READ|PROT_WRITE);
+#endif
+	}
+
+	T* object;
+};
+#ifdef REDEFINED_NEW
+# include "lib/mmgr.h"
+#endif
+
+
+//-----------------------------------------------------------------------------
+// AllocatorChecker
+//-----------------------------------------------------------------------------
+
+/**
+ * allocator test rig.
+ * call from each allocator operation to sanity-check them.
+ * should only be used during debug mode due to serious overhead.
+ **/
+class AllocatorChecker
+{
+public:
+	void notify_alloc(void* p, size_t size)
+	{
+		const Allocs::value_type item = std::make_pair(p, size);
+		std::pair<Allocs::iterator, bool> ret = allocs.insert(item);
+		debug_assert(ret.second == true);	// wasn't already in map
+	}
+
+	void notify_free(void* p, size_t size)
+	{
+		Allocs::iterator it = allocs.find(p);
+		if(it == allocs.end())
+			debug_warn("AllocatorChecker: freeing invalid pointer");
+		else
+		{
+			// size must match what was passed to notify_alloc
+			const size_t allocated_size = it->second;
+			debug_assert(size == allocated_size);
+
+			allocs.erase(it);
+		}
+	}
+
+	/**
+	 * allocator is resetting itself, i.e. wiping out all allocs.
+	 **/
+	void notify_clear()
+	{
+		allocs.clear();
+	}
+
+private:
+	typedef std::map<void*, size_t> Allocs;
+	Allocs allocs;
+};
+
+#endif	// #ifndef INCLUDED_ALLOCATORS

source/lib/allocators/bucket.cpp

@@ -0,0 +1,108 @@
+/**
+ * =========================================================================
+ * File        : bucket.cpp
+ * Project     : 0 A.D.
+ * Description : bucket allocator
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "bucket.h"
+
+#include "lib/bits.h"
+#include "mem_util.h"
+
+
+// power-of-2 isn't required; value is arbitrary.
+const size_t bucketSize = 4000;
+
+
+LibError bucket_create(Bucket* b, size_t el_size)
+{
+	b->freelist = 0;
+	b->el_size = mem_RoundUpToAlignment(el_size);
+
+	// note: allocating here avoids the is-this-the-first-time check
+	// in bucket_alloc, which speeds things up.
+	b->bucket = (u8*)malloc(bucketSize);
+	if(!b->bucket)
+	{
+		// cause next bucket_alloc to retry the allocation
+		b->pos = bucketSize;
+		b->num_buckets = 0;
+		WARN_RETURN(ERR::NO_MEM);
+	}
+
+	*(u8**)b->bucket = 0;	// terminate list
+	b->pos = mem_RoundUpToAlignment(sizeof(u8*));
+	b->num_buckets = 1;
+	return INFO::OK;
+}
+
+
+void bucket_destroy(Bucket* b)
+{
+	while(b->bucket)
+	{
+		u8* prev_bucket = *(u8**)b->bucket;
+		free(b->bucket);
+		b->bucket = prev_bucket;
+		b->num_buckets--;
+	}
+
+	debug_assert(b->num_buckets == 0);
+
+	// poison pill: cause subsequent alloc and free to fail
+	b->freelist = 0;
+	b->el_size = bucketSize;
+}
+
+
+void* bucket_alloc(Bucket* b, size_t size)
+{
+	size_t el_size = b->el_size? b->el_size : mem_RoundUpToAlignment(size);
+	// must fit in a bucket
+	debug_assert(el_size <= bucketSize-sizeof(u8*));
+
+	// try to satisfy alloc from freelist
+	void* el = mem_freelist_Detach(b->freelist);
+	if(el)
+		return el;
+
+	// if there's not enough space left, close current bucket and
+	// allocate another.
+	if(b->pos+el_size > bucketSize)
+	{
+		u8* bucket = (u8*)malloc(bucketSize);
+		if(!bucket)
+			return 0;
+		*(u8**)bucket = b->bucket;
+		b->bucket = bucket;
+		// skip bucket list field and align (note: malloc already
+		// aligns to at least 8 bytes, so don't take b->bucket into account)
+		b->pos = mem_RoundUpToAlignment(sizeof(u8*));;
+		b->num_buckets++;
+	}
+
+	void* ret = b->bucket+b->pos;
+	b->pos += el_size;
+	return ret;
+}
+
+
+void bucket_free(Bucket* b, void* el)
+{
+	if(b->el_size == 0)
+	{
+		DEBUG_WARN_ERR(ERR::LOGIC);	// cannot free variable-size items
+		return;
+	}
+
+	mem_freelist_AddToFront(b->freelist, el);
+
+	// note: checking if <el> was actually allocated from <b> is difficult:
+	// it may not be in the currently open bucket, so we'd have to
+	// iterate over the list - too much work.
+}

source/lib/allocators/bucket.h

@@ -0,0 +1,96 @@
+/**
+ * =========================================================================
+ * File        : bucket.h
+ * Project     : 0 A.D.
+ * Description : bucket allocator
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_BUCKET
+#define INCLUDED_BUCKET
+
+/**
+ * allocator design goals:
+ * - either fixed- or variable-sized blocks;
+ * - allow freeing individual blocks if they are all fixed-size;
+ * - never relocates;
+ * - no fixed limit.
+ *
+ * note: this type of allocator is called "region-based" in the literature
+ * and is also known as "obstack"; see "Reconsidering Custom Memory
+ * Allocation" (Berger, Zorn, McKinley).
+ * if individual variable-size elements must be freeable, consider "reaps":
+ * basically a combination of region and heap, where frees go to the heap and
+ * allocs exhaust that memory first and otherwise use the region.
+ *
+ * opaque! do not read/write any fields!
+ **/
+struct Bucket
+{
+	/**
+	 * currently open bucket.
+	 **/
+	u8* bucket;
+
+	/**
+	 * offset of free space at end of current bucket (i.e. # bytes in use).
+	 **/
+	size_t pos;
+
+	void* freelist;
+
+	size_t el_size : 16;
+
+	/**
+	 * records # buckets allocated; verifies the list of buckets is correct.
+	 **/
+	uint num_buckets : 16;
+};
+
+
+/**
+ * ready the Bucket object for use.
+ *
+ * @param Bucket*
+ * @param el_size 0 to allow variable-sized allocations (which cannot be
+ * freed individually); otherwise, it specifies the number of bytes that
+ * will be returned by bucket_alloc (whose size parameter is then ignored).
+ * @return LibError.
+ **/
+extern LibError bucket_create(Bucket* b, size_t el_size);
+
+/**
+ * free all memory that ensued from <b>.
+ *
+ * future alloc and free calls on this Bucket will fail.
+ *
+ * @param Bucket*
+ **/
+extern void bucket_destroy(Bucket* b);
+
+/**
+ * Dole out memory from the Bucket.
+ * exhausts the freelist before returning new entries to improve locality.
+ *
+ * @param Bucket*
+ * @param size bytes to allocate; ignored if bucket_create's el_size was not 0.
+ * @return allocated memory, or 0 if the Bucket would have to be expanded and
+ * there isn't enough memory to do so.
+ **/
+extern void* bucket_alloc(Bucket* b, size_t size);
+
+/**
+ * make an entry available for reuse in the given Bucket.
+ *
+ * this is not allowed if created for variable-size elements.
+ * rationale: avoids having to pass el_size here and compare with size when
+ * allocating; also prevents fragmentation and leaking memory.
+ *
+ * @param Bucket*
+ * @param el entry allocated via bucket_alloc.
+ **/
+extern void bucket_free(Bucket* b, void* el);
+
+#endif	// #ifndef INCLUDED_BUCKET

source/lib/allocators/dynarray.cpp

@@ -0,0 +1,186 @@
+/**
+ * =========================================================================
+ * File        : dynarray.cpp
+ * Project     : 0 A.D.
+ * Description : dynamic (expandable) array
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "dynarray.h"
+
+#include "lib/posix/posix_mman.h"	// PROT_* constants for da_set_prot
+#include "lib/sysdep/cpu.h"
+#include "mem_util.h"
+
+
+// indicates that this DynArray must not be resized or freed
+// (e.g. because it merely wraps an existing memory range).
+// stored in da->prot to reduce size; doesn't conflict with any PROT_* flags.
+const int DA_NOT_OUR_MEM = 0x40000000;
+
+static LibError validate_da(DynArray* da)
+{
+	if(!da)
+		WARN_RETURN(ERR::INVALID_PARAM);
+	u8* const base           = da->base;
+	const size_t max_size_pa = da->max_size_pa;
+	const size_t cur_size    = da->cur_size;
+	const size_t pos         = da->pos;
+	const int prot           = da->prot;
+
+	if(debug_is_pointer_bogus(base))
+		WARN_RETURN(ERR::_1);
+	// note: don't check if base is page-aligned -
+	// might not be true for 'wrapped' mem regions.
+//	if(!mem_IsPageMultiple((uintptr_t)base))
+//		WARN_RETURN(ERR::_2);
+	if(!mem_IsPageMultiple(max_size_pa))
+		WARN_RETURN(ERR::_3);
+	if(cur_size > max_size_pa)
+		WARN_RETURN(ERR::_4);
+	if(pos > cur_size || pos > max_size_pa)
+		WARN_RETURN(ERR::_5);
+	if(prot & ~(PROT_READ|PROT_WRITE|PROT_EXEC|DA_NOT_OUR_MEM))
+		WARN_RETURN(ERR::_6);
+
+	return INFO::OK;
+}
+
+#define CHECK_DA(da) RETURN_ERR(validate_da(da))
+
+
+LibError da_alloc(DynArray* da, size_t max_size)
+{
+	const size_t max_size_pa = mem_RoundUpToPage(max_size);
+
+	u8* p;
+	RETURN_ERR(mem_Reserve(max_size_pa, &p));
+
+	da->base        = p;
+	da->max_size_pa = max_size_pa;
+	da->cur_size    = 0;
+	da->cur_size_pa = 0;
+	da->prot        = PROT_READ|PROT_WRITE;
+	da->pos         = 0;
+	CHECK_DA(da);
+	return INFO::OK;
+}
+
+
+LibError da_free(DynArray* da)
+{
+	CHECK_DA(da);
+
+	u8* p            = da->base;
+	size_t size_pa   = da->max_size_pa;
+	bool was_wrapped = (da->prot & DA_NOT_OUR_MEM) != 0;
+
+	// wipe out the DynArray for safety
+	// (must be done here because mem_Release may fail)
+	memset(da, 0, sizeof(*da));
+
+	// skip mem_Release if <da> was allocated via da_wrap_fixed
+	// (i.e. it doesn't actually own any memory). don't complain;
+	// da_free is supposed to be called even in the above case.
+	if(!was_wrapped)
+		RETURN_ERR(mem_Release(p, size_pa));
+	return INFO::OK;
+}
+
+
+LibError da_set_size(DynArray* da, size_t new_size)
+{
+	CHECK_DA(da);
+
+	if(da->prot & DA_NOT_OUR_MEM)
+		WARN_RETURN(ERR::LOGIC);
+
+	// determine how much to add/remove
+	const size_t cur_size_pa = mem_RoundUpToPage(da->cur_size);
+	const size_t new_size_pa = mem_RoundUpToPage(new_size);
+	const ssize_t size_delta_pa = (ssize_t)new_size_pa - (ssize_t)cur_size_pa;
+
+	// not enough memory to satisfy this expand request: abort.
+	// note: do not complain - some allocators (e.g. file_cache)
+	// legitimately use up all available space.
+	if(new_size_pa > da->max_size_pa)
+		return ERR::LIMIT;	// NOWARN
+
+	u8* end = da->base + cur_size_pa;
+	// expanding
+	if(size_delta_pa > 0)
+		RETURN_ERR(mem_Commit(end, size_delta_pa, da->prot));
+	// shrinking
+	else if(size_delta_pa < 0)
+		RETURN_ERR(mem_Decommit(end+size_delta_pa, -size_delta_pa));
+	// else: no change in page count, e.g. if going from size=1 to 2
+	// (we don't want mem_* to have to handle size=0)
+
+	da->cur_size = new_size;
+	da->cur_size_pa = new_size_pa;
+	CHECK_DA(da);
+	return INFO::OK;
+}
+
+
+LibError da_reserve(DynArray* da, size_t size)
+{
+	if(da->pos+size > da->cur_size_pa)
+		RETURN_ERR(da_set_size(da, da->cur_size_pa+size));
+	da->cur_size = std::max(da->cur_size, da->pos+size);
+	return INFO::OK;
+}
+
+
+LibError da_set_prot(DynArray* da, int prot)
+{
+	CHECK_DA(da);
+
+	// somewhat more subtle: POSIX mprotect requires the memory have been
+	// mmap-ed, which it probably wasn't here.
+	if(da->prot & DA_NOT_OUR_MEM)
+		WARN_RETURN(ERR::LOGIC);
+
+	da->prot = prot;
+	RETURN_ERR(mem_Protect(da->base, da->cur_size_pa, prot));
+
+	CHECK_DA(da);
+	return INFO::OK;
+}
+
+
+LibError da_wrap_fixed(DynArray* da, u8* p, size_t size)
+{
+	da->base        = p;
+	da->max_size_pa = mem_RoundUpToPage(size);
+	da->cur_size    = size;
+	da->cur_size_pa = da->max_size_pa;
+	da->prot        = PROT_READ|PROT_WRITE|DA_NOT_OUR_MEM;
+	da->pos         = 0;
+	CHECK_DA(da);
+	return INFO::OK;
+}
+
+
+LibError da_read(DynArray* da, void* data, size_t size)
+{
+	// make sure we have enough data to read
+	if(da->pos+size > da->cur_size)
+		WARN_RETURN(ERR::FAIL);
+
+	cpu_memcpy(data, da->base+da->pos, size);
+	da->pos += size;
+	return INFO::OK;
+}
+
+
+LibError da_append(DynArray* da, const void* data, size_t size)
+{
+	RETURN_ERR(da_reserve(da, size));
+	cpu_memcpy(da->base+da->pos, data, size);
+	da->pos += size;
+	return INFO::OK;
+}

source/lib/allocators/dynarray.h

@@ -0,0 +1,133 @@
+/**
+ * =========================================================================
+ * File        : dynarray.h
+ * Project     : 0 A.D.
+ * Description : dynamic (expandable) array
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_DYNARRAY
+#define INCLUDED_DYNARRAY
+
+/**
+ * provides a memory range that can be expanded but doesn't waste
+ * physical memory or relocate itself.
+ *
+ * works by preallocating address space and committing as needed.
+ * used as a building block for other allocators.
+ **/
+struct DynArray
+{
+	u8* base;
+	size_t max_size_pa;	 /// reserved
+	size_t cur_size;	 /// committed
+	size_t cur_size_pa;
+
+	/**
+	 * mprotect flags applied to newly committed pages
+	 **/
+	int prot;
+
+	size_t pos;
+};
+
+
+/**
+ * ready the DynArray object for use.
+ *
+ * no virtual memory is actually committed until calls to da_set_size.
+ *
+ * @param da DynArray.
+ * @param max_size size [bytes] of address space to reserve (*);
+ * the DynArray can never expand beyond this.
+ * (* rounded up to next page size multiple)
+ * @return LibError.
+ **/
+extern LibError da_alloc(DynArray* da, size_t max_size);
+
+/**
+ * free all memory (address space + physical) that constitutes the
+ * given array.
+ *
+ * use-after-free is impossible because the memory is unmapped.
+ *
+ * @param DynArray* da; zeroed afterwards.
+ * @return LibError
+ **/
+extern LibError da_free(DynArray* da);
+
+/**
+ * expand or shrink the array: changes the amount of currently committed
+ * (i.e. usable) memory pages.
+ *
+ * @param da DynArray.
+ * @param new_size target size (rounded up to next page multiple).
+ * pages are added/removed until this is met.
+ * @return LibError.
+ **/
+extern LibError da_set_size(DynArray* da, size_t new_size);
+
+/**
+ * Make sure at least <size> bytes starting at da->pos are committed and
+ * ready for use.
+ *
+ * @param DynArray*
+ * @param size Minimum amount to guarantee [bytes]
+ * @return LibError
+ **/
+extern LibError da_reserve(DynArray* da, size_t size);
+
+/**
+ * change access rights of the array memory.
+ *
+ * used to implement write-protection. affects the currently committed
+ * pages as well as all subsequently added pages.
+ *
+ * @param da DynArray.
+ * @param prot a combination of the PROT_* values used with mprotect.
+ * @return LibError.
+ **/
+extern LibError da_set_prot(DynArray* da, int prot);
+
+/**
+ * "wrap" (i.e. store information about) the given buffer in a DynArray.
+ *
+ * this is used to allow calling da_read or da_append on normal buffers.
+ * da_free should be called when the DynArray is no longer needed,
+ * even though it doesn't free this memory (but does zero the DynArray).
+ *
+ * @param da DynArray. Note: any future operations on it that would
+ * change the underlying memory (e.g. da_set_size) will fail.
+ * @param p target memory (no alignment/padding requirements)
+ * @param size maximum size (no alignment requirements)
+ * @return LibError.
+ **/
+extern LibError da_wrap_fixed(DynArray* da, u8* p, size_t size);
+
+/**
+ * "read" from array, i.e. copy into the given buffer.
+ *
+ * starts at offset DynArray.pos and advances this.
+ *
+ * @param da DynArray.
+ * @param data_dst destination memory
+ * @param size [bytes] to copy
+ * @return LibError.
+ **/
+extern LibError da_read(DynArray* da, void* data_dst, size_t size);
+
+/**
+ * "write" to array, i.e. copy from the given buffer.
+ *
+ * starts at offset DynArray.pos and advances this.
+ *
+ * @param da DynArray.
+ * @param data_src source memory
+ * @param size [bytes] to copy
+ * @return LibError.
+ **/
+extern LibError da_append(DynArray* da, const void* data_src, size_t size);
+
+#endif	// #ifndef INCLUDED_DYNARRAY

source/lib/allocators/headerless.cpp

@@ -0,0 +1,753 @@
+/**
+ * =========================================================================
+ * File        : headerless.cpp
+ * Project     : 0 A.D.
+ * Description : (header-.less) pool-based heap allocator
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "headerless.h"
+
+#include "mem_util.h"
+#include "pool.h"
+#include "lib/bits.h"
+
+
+static const size_t minAlignment = 16;
+static const bool performSanityChecks = true;
+
+// shared by the Impl::Allocate and FreedBlock::Validate
+static bool IsValidSize(size_t size);
+
+
+//-----------------------------------------------------------------------------
+
+// this combines the boundary tags and link fields into one structure,
+// which is safer than direct pointer arithmetic.
+//
+// it is written to freed memory, which is fine because IsValidSize ensures
+// the allocations are large enough.
+class FreedBlock
+{
+	friend class RangeList;	// manipulates link fields directly
+
+public:
+	// (required for RangeList::m_sentinel)
+	FreedBlock()
+	{
+	}
+
+	FreedBlock(u32 id, size_t size)
+		:  m_magic(s_magic), m_size(size), m_id(id)
+	{
+	}
+
+	~FreedBlock()
+	{
+		// clear all fields to prevent accidental reuse
+		prev = next = 0;
+		m_id = 0;
+		m_size = ~0u;
+		m_magic = 0;
+	}
+
+	size_t Size() const
+	{
+		return m_size;
+	}
+
+	/**
+	 * @return whether this appears to be a FreedBlock instance with the
+	 * desired ID. for additional safety, also call Validate().
+	 **/
+	bool IsFreedBlock(u32 id) const
+	{
+		if(m_id != id)
+			return false;
+		if(m_magic != s_magic)
+			return false;
+		return true;
+	}
+
+	/**
+	 * warn if any invariant doesn't hold.
+	 **/
+	void Validate(u32 id) const
+	{
+		if(!performSanityChecks) return;
+
+		// note: RangeList::Validate implicitly checks the prev and next
+		// fields by iterating over the list.
+
+		// the sentinel of empty lists has prev == next, but we're only
+		// called for actual blocks, so that should never happen.
+		debug_assert(prev != next);
+
+		debug_assert(IsValidSize(m_size));
+		debug_assert(IsFreedBlock(id));
+	}
+
+private:
+	// note: the magic and ID fields are stored at both ends of this
+	// class to increase the chance of detecting memory corruption.
+	static const uintptr_t s_magic = 0xFF55AA00;
+	uintptr_t m_magic;
+
+	FreedBlock* prev;
+	FreedBlock* next;
+
+	// size [bytes] of the entire memory block, including header and footer
+	size_t m_size;
+
+	// this differentiates between headers and footers.
+	u32 m_id;
+};
+
+
+static bool IsValidSize(size_t size)
+{
+	// note: we disallow the questionable practice of zero-byte allocations
+	// because they may be indicative of bugs.
+
+	if(size < sizeof(FreedBlock))
+		return false;
+
+	if(size % minAlignment)
+		return false;
+
+	return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// freelists
+//-----------------------------------------------------------------------------
+
+// policy: address-ordered good fit
+// mechanism: segregated range lists of power-of-two size classes
+
+struct AddressOrder
+{
+	static bool ShouldInsertAfter(FreedBlock* predecessor, FreedBlock* current)
+	{
+		return current > predecessor;
+	}
+};
+
+// "range list" is a freelist of similarly-sized blocks.
+class RangeList
+{
+public:
+	RangeList()
+	{
+		Reset();
+	}
+
+	void Reset()
+	{
+		m_sentinel.prev = &m_sentinel;
+		m_sentinel.next = &m_sentinel;
+		m_head = &m_sentinel;
+		m_freeBlocks = 0;
+		m_freeBytes = 0;
+	}
+
+	template<class InsertPolicy>
+	void Insert(FreedBlock* freedBlock)
+	{
+		// find freedBlock after which to insert
+		FreedBlock* predecessor;
+		for(predecessor = m_head; predecessor != &m_sentinel; predecessor = predecessor->next)
+		{
+			if(InsertPolicy::ShouldInsertAfter(predecessor, freedBlock))
+				break;
+		}
+
+		freedBlock->prev = predecessor;
+		freedBlock->next = predecessor->next;
+		predecessor->next->prev = freedBlock;
+		predecessor->next = freedBlock;
+
+		m_freeBlocks++;
+		m_freeBytes += freedBlock->Size();
+	}
+
+	/**
+	 * @return the first freed block of size >= minSize or 0 if none exists.
+	 **/
+	FreedBlock* Find(size_t minSize)
+	{
+		for(FreedBlock* freedBlock = m_head; freedBlock != &m_sentinel; freedBlock = freedBlock->next)
+		{
+			if(freedBlock->Size() >= minSize)
+				return freedBlock;
+		}
+
+		// none found, so average block size is less than the desired size
+		debug_assert(m_freeBytes/m_freeBlocks < minSize);
+		return 0;
+	}
+
+	void Remove(FreedBlock* freedBlock)
+	{
+		freedBlock->next->prev = freedBlock->prev;
+		freedBlock->prev->next = freedBlock->next;
+
+		debug_assert(m_freeBlocks != 0);
+		debug_assert(m_freeBytes >= freedBlock->Size());
+		m_freeBlocks--;
+		m_freeBytes -= freedBlock->Size();
+	}
+
+	void Validate(u32 id) const
+	{
+		if(!performSanityChecks) return;
+
+		size_t freeBlocks = 0, freeBytes = 0;
+
+		for(FreedBlock* freedBlock = m_head; freedBlock != &m_sentinel; freedBlock = freedBlock->next)
+		{
+			freedBlock->Validate(id);
+			freeBlocks++;
+			freeBytes += freedBlock->Size();
+		}
+
+		for(FreedBlock* freedBlock = m_head; freedBlock != &m_sentinel; freedBlock = freedBlock->prev)
+		{
+			freedBlock->Validate(id);
+			freeBlocks++;
+			freeBytes += freedBlock->Size();
+		}
+
+		// our idea of the number and size of free blocks is correct
+		debug_assert(freeBlocks == m_freeBlocks*2 && freeBytes == m_freeBytes*2);
+		// if empty, state must be as established by Reset
+		debug_assert(!IsEmpty() || (m_head->next == m_head->prev && m_head == &m_sentinel));
+	}
+
+	bool IsEmpty() const
+	{
+		return (m_freeBlocks == 0);
+	}
+
+	size_t FreeBlocks() const
+	{
+		return m_freeBlocks;
+	}
+
+	size_t FreeBytes() const
+	{
+		return m_freeBytes;
+	}
+
+private:
+	// a sentinel simplifies Insert and Remove. we store it here instead of
+	// in a separate array to improve locality (it is actually accessed).
+	mutable FreedBlock m_sentinel;
+
+	FreedBlock* m_head;
+
+	size_t m_freeBlocks;
+	size_t m_freeBytes;
+};
+
+
+//-----------------------------------------------------------------------------
+
+class SegregatedRangeLists
+{
+public:
+	SegregatedRangeLists()
+	{
+		Reset();
+	}
+
+	void Reset()
+	{
+		m_bitmap = 0;
+		for(size_t i = 0; i < numRangeLists; i++)
+			m_rangeLists[i].Reset();
+	}
+
+	void Insert(FreedBlock* freedBlock)
+	{
+		const uint sizeClass = SizeClass(freedBlock->Size());
+		m_rangeLists[sizeClass].Insert<AddressOrder>(freedBlock);
+
+		m_bitmap |= BIT(sizeClass);
+	}
+
+	/**
+	 * @return the first freed block of size >= minSize or 0 if none exists.
+	 **/
+	FreedBlock* Find(size_t minSize)
+	{
+		// iterate over all large enough, non-empty size classes
+		// (zero overhead for empty size classes)
+		const uint minSizeClass = SizeClass(minSize);
+		uint sizeClassBits = m_bitmap & (~0u << minSizeClass);
+		while(sizeClassBits)
+		{
+			const uint size = ValueOfLeastSignificantOneBit(sizeClassBits);
+			sizeClassBits &= ~size;	// remove from sizeClassBits
+			const uint sizeClass = SizeClass(size);
+
+			FreedBlock* freedBlock = m_rangeLists[sizeClass].Find(minSize);
+			if(freedBlock)
+				return freedBlock;
+		}
+
+		// apparently all classes above minSizeClass are empty,
+		// or the above would have succeeded.
+		debug_assert(m_bitmap < BIT(minSizeClass+1));
+		return 0;
+	}
+
+	void Remove(FreedBlock* freedBlock)
+	{
+		const uint sizeClass = SizeClass(freedBlock->Size());
+		m_rangeLists[sizeClass].Remove(freedBlock);
+
+		// (masking with !IsEmpty() << sizeClass would probably be faster)
+		if(m_rangeLists[sizeClass].IsEmpty())
+			m_bitmap &= ~BIT(sizeClass);
+	}
+
+	void Validate(u32 id) const
+	{
+		for(size_t i = 0; i < numRangeLists; i++)
+		{
+			m_rangeLists[i].Validate(id);
+
+			// both bitmap and list must agree on whether they are empty
+			debug_assert(((m_bitmap & BIT(i)) == 0) == m_rangeLists[i].IsEmpty());
+		}
+	}
+
+	size_t FreeBlocks() const
+	{
+		size_t freeBlocks = 0;
+		for(size_t i = 0; i < numRangeLists; i++)
+			freeBlocks += m_rangeLists[i].FreeBlocks();
+		return freeBlocks;
+	}
+
+	size_t FreeBytes() const
+	{
+		size_t freeBytes = 0;
+		for(size_t i = 0; i < numRangeLists; i++)
+			freeBytes += m_rangeLists[i].FreeBytes();
+		return freeBytes;
+	}
+
+private:
+	/**
+	 * @return "size class" of a given size.
+	 * class i > 0 contains blocks of size (2**(i-1), 2**i].
+	 **/
+	static uint SizeClass(size_t size)
+	{
+		return ceil_log2((uint)size);
+	}
+
+	static uintptr_t ValueOfLeastSignificantOneBit(uintptr_t x)
+	{
+		return (x & -(intptr_t)x);
+	}
+
+	// segregated, i.e. one list per size class.
+	static const size_t numRangeLists = sizeof(uintptr_t)*CHAR_BIT;
+	RangeList m_rangeLists[numRangeLists];
+
+	// bit i set <==> size class i's freelist is not empty.
+	// this allows finding a non-empty list in O(1).
+	uintptr_t m_bitmap;
+};
+
+
+//-----------------------------------------------------------------------------
+// coalescing
+//-----------------------------------------------------------------------------
+
+// policy: immediately coalesce
+// mechanism: boundary tags
+
+// note: the id and magic values are all that differentiates tags from
+// user data. this isn't 100% reliable, but as with headers, we don't want
+// to insert extra boundary tags into the allocated memory.
+
+// note: footers are also represented as FreedBlock. this is easier to
+// implement but a bit inefficient since we don't need all its fields.
+
+class BoundaryTagManager
+{
+public:
+	BoundaryTagManager()
+		: m_freeBlocks(0), m_freeBytes(0)
+	{
+	}
+
+	FreedBlock* WriteTags(u8* p, size_t size)
+	{
+#include "lib/nommgr.h"
+		FreedBlock* freedBlock = new(p) FreedBlock(s_headerId, size);
+		(void)new(Footer(freedBlock)) FreedBlock(s_footerId, size);
+#include "lib/mmgr.h"
+		Validate(freedBlock);
+
+		m_freeBlocks++;
+		m_freeBytes += size;
+		return freedBlock;
+	}
+
+	void RemoveTags(FreedBlock* freedBlock)
+	{
+		Validate(freedBlock);
+
+		debug_assert(m_freeBlocks != 0);
+		debug_assert(m_freeBytes > freedBlock->Size());
+		m_freeBlocks--;
+		m_freeBytes -= freedBlock->Size();
+
+		freedBlock->~FreedBlock();
+		Footer(freedBlock)->~FreedBlock();
+	}
+
+	FreedBlock* PrecedingBlock(u8* p, u8* beginningOfPool) const
+	{
+		if(p == beginningOfPool)	// avoid accessing invalid memory
+			return 0;
+
+		FreedBlock* precedingBlock;
+		{
+			FreedBlock* const footer = (FreedBlock*)(p - sizeof(FreedBlock));
+			if(!footer->IsFreedBlock(s_footerId))
+				return 0;
+			footer->Validate(s_footerId);
+			precedingBlock = (FreedBlock*)(p - footer->Size());
+		}
+
+		Validate(precedingBlock);
+		return precedingBlock;
+	}
+
+	FreedBlock* FollowingBlock(u8* p, size_t size, u8* endOfPool) const
+	{
+		if(p+size == endOfPool)	// avoid accessing invalid memory
+			return 0;
+
+		FreedBlock* const followingBlock = (FreedBlock*)(p + size);
+		if(!followingBlock->IsFreedBlock(s_headerId))
+			return 0;
+
+		Validate(followingBlock);
+		return followingBlock;
+	}
+
+	size_t FreeBlocks() const
+	{
+		return m_freeBlocks;
+	}
+
+	size_t FreeBytes() const
+	{
+		return m_freeBytes;
+	}
+
+	// (generated via GUID)
+	static const u32 s_headerId = 0x111E8E6Fu;
+	static const u32 s_footerId = 0x4D745342u;
+
+private:
+	void Validate(FreedBlock* freedBlock) const
+	{
+		if(!performSanityChecks) return;
+
+		// the existence of freedBlock means our bookkeeping better have
+		// records of at least that much memory.
+		debug_assert(m_freeBlocks != 0);
+		debug_assert(m_freeBytes >= freedBlock->Size());
+
+		freedBlock->Validate(s_headerId);
+		Footer(freedBlock)->Validate(s_footerId);
+	}
+
+	static FreedBlock* Footer(FreedBlock* freedBlock)
+	{
+		u8* const p = (u8*)freedBlock;
+		return (FreedBlock*)(p + freedBlock->Size() - sizeof(FreedBlock));
+	}
+
+	size_t m_freeBlocks;
+	size_t m_freeBytes;
+};
+
+
+//-----------------------------------------------------------------------------
+// stats
+//-----------------------------------------------------------------------------
+
+class Stats
+{
+public:
+	void OnReset()
+	{
+		if(!performSanityChecks) return;
+
+		m_totalAllocatedBlocks = m_totalAllocatedBytes = 0;
+		m_totalDeallocatedBlocks = m_totalDeallocatedBytes = 0;
+		m_currentExtantBlocks = m_currentExtantBytes = 0;
+		m_currentFreeBlocks = m_currentFreeBytes = 0;
+	}
+
+	void OnAllocate(size_t size)
+	{
+		if(!performSanityChecks) return;
+
+		m_totalAllocatedBlocks++;
+		m_totalAllocatedBytes += size;
+
+		m_currentExtantBlocks++;
+		m_currentExtantBytes += size;
+	}
+
+	void OnDeallocate(size_t size)
+	{
+		if(!performSanityChecks) return;
+
+		m_totalDeallocatedBlocks++;
+		m_totalDeallocatedBytes += size;
+		debug_assert(m_totalDeallocatedBlocks <= m_totalAllocatedBlocks);
+		debug_assert(m_totalDeallocatedBytes <= m_totalDeallocatedBytes);
+
+		debug_assert(m_currentExtantBlocks != 0);
+		debug_assert(m_currentExtantBytes >= size);
+		m_currentExtantBlocks--;
+		m_currentExtantBytes -= size;
+	}
+
+	void OnAddToFreelist(size_t size)
+	{
+		m_currentFreeBlocks++;
+		m_currentFreeBytes += size;
+	}
+
+	void OnRemoveFromFreelist(size_t size)
+	{
+		if(!performSanityChecks) return;
+
+		debug_assert(m_currentFreeBlocks != 0);
+		debug_assert(m_currentFreeBytes >= size);
+		m_currentFreeBlocks--;
+		m_currentFreeBytes -= size;
+	}
+
+	void Validate() const
+	{
+		if(!performSanityChecks) return;
+
+		debug_assert(m_totalDeallocatedBlocks <= m_totalAllocatedBlocks);
+		debug_assert(m_totalDeallocatedBytes <= m_totalAllocatedBytes);
+
+		debug_assert(m_currentExtantBlocks == m_totalAllocatedBlocks-m_totalDeallocatedBlocks);
+		debug_assert(m_currentExtantBytes == m_totalAllocatedBytes-m_totalDeallocatedBytes);
+	}
+
+	size_t FreeBlocks() const
+	{
+		return m_currentFreeBlocks;
+	}
+
+	size_t FreeBytes() const
+	{
+		return m_currentFreeBytes;
+	}
+
+private:
+	size_t m_totalAllocatedBlocks, m_totalAllocatedBytes;
+	size_t m_totalDeallocatedBlocks, m_totalDeallocatedBytes;
+	size_t m_currentExtantBlocks, m_currentExtantBytes;
+	size_t m_currentFreeBlocks, m_currentFreeBytes;
+};
+
+
+//-----------------------------------------------------------------------------
+// HeaderlessAllocator::Impl
+//-----------------------------------------------------------------------------
+
+static void AssertEqual(size_t x1, size_t x2, size_t x3)
+{
+	debug_assert(x1 == x2 && x2 == x3);
+}
+
+class HeaderlessAllocator::Impl
+{
+public:
+	Impl(size_t poolSize)
+	{
+		(void)pool_create(&m_pool, poolSize, 0);
+
+		Reset();
+	}
+
+	~Impl()
+	{
+		Validate();
+
+		(void)pool_destroy(&m_pool);
+	}
+
+	void Reset()
+	{
+		Validate();
+
+		pool_free_all(&m_pool);
+		m_segregatedRangeLists.Reset();
+		m_stats.OnReset();
+
+		Validate();
+	}
+
+	void* Allocate(size_t size) throw()
+	{
+		debug_assert(IsValidSize(size));
+		Validate();
+
+		void* p = TakeAndSplitFreeBlock(size);
+		if(!p)
+		{
+			p = pool_alloc(&m_pool, size);
+			if(!p)			// both failed; don't throw bad_alloc because
+				return 0;	// this often happens with the file cache.
+		}
+
+		// (NB: we must not update the statistics if allocation failed)
+		m_stats.OnAllocate(size);
+
+		Validate();
+		return p;
+	}
+
+	void Deallocate(u8* p, size_t size)
+	{
+		debug_assert((uintptr_t)p % minAlignment == 0);
+		debug_assert(IsValidSize(size));
+		debug_assert(pool_contains(&m_pool, p));
+		debug_assert(pool_contains(&m_pool, p+size-1));
+
+		Validate();
+
+		m_stats.OnDeallocate(size);
+		Coalesce(p, size);
+		AddToFreelist(p, size);
+
+		Validate();
+	}
+
+	void Validate() const
+	{
+		if(!performSanityChecks) return;
+
+		m_segregatedRangeLists.Validate(BoundaryTagManager::s_headerId);
+		m_stats.Validate();
+
+		AssertEqual(m_stats.FreeBlocks(), m_segregatedRangeLists.FreeBlocks(), m_boundaryTagManager.FreeBlocks());
+		AssertEqual(m_stats.FreeBytes(), m_segregatedRangeLists.FreeBytes(), m_boundaryTagManager.FreeBytes());
+	}
+
+private:
+	void AddToFreelist(u8* p, size_t size)
+	{
+		FreedBlock* freedBlock = m_boundaryTagManager.WriteTags(p, size);
+		m_segregatedRangeLists.Insert(freedBlock);
+		m_stats.OnAddToFreelist(size);
+	}
+
+	void RemoveFromFreelist(FreedBlock* freedBlock)
+	{
+		m_stats.OnRemoveFromFreelist(freedBlock->Size());
+		m_segregatedRangeLists.Remove(freedBlock);
+		m_boundaryTagManager.RemoveTags(freedBlock);
+	}
+
+	/**
+	 * expand a block by coalescing it with its free neighbor(s).
+	 **/
+	void Coalesce(u8*& p, size_t& size)
+	{
+		{
+			FreedBlock* precedingBlock = m_boundaryTagManager.PrecedingBlock(p, m_pool.da.base);
+			if(precedingBlock)
+			{
+				p -= precedingBlock->Size();
+				size += precedingBlock->Size();
+				RemoveFromFreelist(precedingBlock);
+			}
+		}
+
+		{
+			FreedBlock* followingBlock = m_boundaryTagManager.FollowingBlock(p, size, m_pool.da.base+m_pool.da.pos);
+			if(followingBlock)
+			{
+				size += followingBlock->Size();
+				RemoveFromFreelist(followingBlock);
+			}
+		}
+	}
+
+	void* TakeAndSplitFreeBlock(size_t size)
+	{
+		u8* p;
+		size_t leftoverSize = 0;
+		{
+			FreedBlock* freedBlock = m_segregatedRangeLists.Find(size);
+			if(!freedBlock)
+				return 0;
+
+			p = (u8*)freedBlock;
+			leftoverSize = freedBlock->Size() - size;
+			RemoveFromFreelist(freedBlock);
+		}
+
+		if(IsValidSize(leftoverSize))
+			AddToFreelist(p+size, leftoverSize);
+
+		return p;
+	}
+
+	Pool m_pool;
+	SegregatedRangeLists m_segregatedRangeLists;
+	BoundaryTagManager m_boundaryTagManager;
+	Stats m_stats;
+};
+
+
+//-----------------------------------------------------------------------------
+
+HeaderlessAllocator::HeaderlessAllocator(size_t poolSize)
+	: impl(new Impl(poolSize))
+{
+}
+
+void HeaderlessAllocator::Reset()
+{
+	return impl.get()->Reset();
+}
+
+void* HeaderlessAllocator::Allocate(size_t size) throw()
+{
+	return impl.get()->Allocate(size);
+}
+
+void HeaderlessAllocator::Deallocate(void* p, size_t size)
+{
+	return impl.get()->Deallocate((u8*)p, size);
+}
+
+void HeaderlessAllocator::Validate() const
+{
+	return impl.get()->Validate();
+}

source/lib/allocators/headerless.h

@@ -0,0 +1,74 @@
+/**
+ * =========================================================================
+ * File        : headerless.h
+ * Project     : 0 A.D.
+ * Description : (header-less) pool-based heap allocator
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_HEADERLESS
+#define INCLUDED_HEADERLESS
+
+#include <boost/shared_ptr.hpp>
+
+/**
+ * (header-less) pool-based heap allocator
+ * provides Allocate and Deallocate without requiring in-band headers;
+ * this is useful when allocating page-aligned I/O buffers
+ * (headers would waste an entire page per buffer)
+ *
+ * policy:
+ * - allocation: first exhaust the freelist, then allocate more
+ * - freelist: address-ordered good fit, always split blocks
+ * - coalescing: immediate
+ * mechanism:
+ * - coalescing: boundary tags in freed memory with distinct bit patterns
+ * - freelist: segregated range lists of power-of-two size classes
+ *
+ * note: this module basically implements a (rather complex) freelist and
+ * could be made independent of the Pool allocation scheme. however, reading
+ * neighboring boundary tags may cause segmentation violations; knowing the
+ * bounds of valid committed memory (i.e. Pool extents) avoids this.
+ **/
+class HeaderlessAllocator
+{
+public:
+	/**
+	 * @param poolSize maximum amount of memory that can be allocated.
+	 * this much virtual address space is reserved up-front (see Pool).
+	 **/
+	HeaderlessAllocator(size_t poolSize);
+
+	/**
+	 * restore the original state (as if newly constructed).
+	 * this includes reclaiming all extant allocations.
+	 **/
+	void Reset();
+
+	/**
+	 * @param size [bytes] must be a multiple of the minimum alignment and
+	 * enough to store a block header. (this allocator is designed for
+	 * page-aligned requests but can handle smaller amounts.)
+	 * @return allocated memory or 0 if the pool is too fragmented or full.
+	 **/
+	void* Allocate(size_t size) throw();
+
+	/**
+	 * deallocate memory.
+	 * @param size must be exactly as specified to Allocate.
+	 **/
+	void Deallocate(void* p, size_t size);
+
+	/**
+	 * perform sanity checks; ensure allocator state is consistent.
+	 **/
+	void Validate() const;
+
+private:
+	class Impl;
+	boost::shared_ptr<Impl> impl;
+};
+
+#endif	// #ifndef INCLUDED_HEADERLESS

source/lib/allocators/mem_util.cpp

@@ -0,0 +1,126 @@
+/**
+ * =========================================================================
+ * File        : mem_util.cpp
+ * Project     : 0 A.D.
+ * Description : memory allocator helper routines.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "mem_util.h"
+
+#include "lib/bits.h"				// round_up
+#include "lib/posix/posix_mman.h"
+#include "lib/sysdep/cpu.h"			// cpu_PageSize
+
+
+size_t mem_PageSize()
+{
+	static const size_t page_size = cpu_PageSize();
+	return page_size;
+}
+
+bool mem_IsPageMultiple(uintptr_t x)
+{
+	return (x & (mem_PageSize()-1)) == 0;
+}
+
+size_t mem_RoundUpToPage(size_t size)
+{
+	return round_up(size, mem_PageSize());
+}
+
+size_t mem_RoundUpToAlignment(size_t size)
+{
+	// all allocators should align to at least this many bytes:
+	const size_t alignment = 8;
+	return round_up(size, alignment);
+}
+
+
+//-----------------------------------------------------------------------------
+
+static inline LibError LibError_from_mmap(void* ret, bool warn_if_failed = true)
+{
+	if(ret != MAP_FAILED)
+		return INFO::OK;
+	return LibError_from_errno(warn_if_failed);
+}
+
+// "anonymous" effectively means mapping /dev/zero, but is more efficient.
+// MAP_ANONYMOUS is not in SUSv3, but is a very common extension.
+// unfortunately, MacOS X only defines MAP_ANON, which Solaris says is
+// deprecated. workaround there: define MAP_ANONYMOUS in terms of MAP_ANON.
+#ifndef MAP_ANONYMOUS
+# define MAP_ANONYMOUS MAP_ANON
+#endif
+
+static const int mmap_flags = MAP_PRIVATE|MAP_ANONYMOUS;
+
+LibError mem_Reserve(size_t size, u8** pp)
+{
+	errno = 0;
+	void* ret = mmap(0, size, PROT_NONE, mmap_flags|MAP_NORESERVE, -1, 0);
+	*pp = (u8*)ret;
+	return LibError_from_mmap(ret);
+}
+
+LibError mem_Release(u8* p, size_t size)
+{
+	errno = 0;
+	int ret = munmap(p, size);
+	return LibError_from_posix(ret);
+}
+
+LibError mem_Commit(u8* p, size_t size, int prot)
+{
+	// avoid misinterpretation by mmap.
+	if(prot == PROT_NONE)
+		WARN_RETURN(ERR::INVALID_PARAM);
+
+	errno = 0;
+	void* ret = mmap(p, size, prot, mmap_flags|MAP_FIXED, -1, 0);
+	return LibError_from_mmap(ret);
+}
+
+LibError mem_Decommit(u8* p, size_t size)
+{
+	errno = 0;
+	void* ret = mmap(p, size, PROT_NONE, mmap_flags|MAP_NORESERVE|MAP_FIXED, -1, 0);
+	return LibError_from_mmap(ret);
+}
+
+LibError mem_Protect(u8* p, size_t size, int prot)
+{
+	errno = 0;
+	int ret = mprotect(p, size, prot);
+	return LibError_from_posix(ret);
+}
+
+
+//-----------------------------------------------------------------------------
+
+// "freelist" is a pointer to the first unused element (0 if there are none);
+// its memory holds a pointer to the next free one in list.
+
+void mem_freelist_AddToFront(void*& freelist, void* el)
+{
+	debug_assert(el != 0);
+
+	void* prev_el = freelist;
+	freelist = el;
+	*(void**)el = prev_el;
+}
+
+
+void* mem_freelist_Detach(void*& freelist)
+{
+	void* el = freelist;
+	// nothing in list
+	if(!el)
+		return 0;
+	freelist = *(void**)el;
+	return el;
+}

source/lib/allocators/mem_util.h

@@ -0,0 +1,44 @@
+/**
+ * =========================================================================
+ * File        : mem_util.h
+ * Project     : 0 A.D.
+ * Description : memory allocator helper routines.
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_MEM_UTIL
+#define INCLUDED_MEM_UTIL
+
+
+/**
+ * @return page size
+ *
+ * (this routine caches the result of cpu_PageSize and ensures the value
+ * is available before static initializers have run.)
+ **/
+extern size_t mem_PageSize();
+extern bool mem_IsPageMultiple(uintptr_t x);
+
+extern size_t mem_RoundUpToPage(size_t size);
+extern size_t mem_RoundUpToAlignment(size_t size);
+
+
+// very thin wrapper on top of sys/mman.h that makes the intent more obvious
+// (its commit/decommit semantics are difficult to tell apart)
+extern LibError mem_Reserve(size_t size, u8** pp);
+extern LibError mem_Release(u8* p, size_t size);
+extern LibError mem_Commit(u8* p, size_t size, int prot);
+extern LibError mem_Decommit(u8* p, size_t size);
+extern LibError mem_Protect(u8* p, size_t size, int prot);
+
+
+// note: element memory is used to store a pointer to the next free element.
+// rationale for the function-based interface: a class encapsulating the
+// freelist pointer would force each header to include mem_util.h;
+// instead, implementations need only declare a void* pointer.
+void mem_freelist_AddToFront(void*& freelist, void* el);
+void* mem_freelist_Detach(void*& freelist);
+
+#endif	// #ifndef INCLUDED_MEM_UTIL

source/lib/allocators/pool.cpp

@@ -0,0 +1,105 @@
+/**
+ * =========================================================================
+ * File        : pool.cpp
+ * Project     : 0 A.D.
+ * Description : pool allocator
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#include "precompiled.h"
+#include "pool.h"
+
+#include "mem_util.h"
+
+
+LibError pool_create(Pool* p, size_t max_size, size_t el_size)
+{
+	if(el_size == POOL_VARIABLE_ALLOCS)
+		p->el_size = 0;
+	else
+		p->el_size = mem_RoundUpToAlignment(el_size);
+	p->freelist = 0;
+	RETURN_ERR(da_alloc(&p->da, max_size));
+	return INFO::OK;
+}
+
+
+LibError pool_destroy(Pool* p)
+{
+	// don't be picky and complain if the freelist isn't empty;
+	// we don't care since it's all part of the da anyway.
+	// however, zero it to prevent further allocs from succeeding.
+	p->freelist = 0;
+	return da_free(&p->da);
+}
+
+
+bool pool_contains(const Pool* p, void* el)
+{
+	// outside of our range
+	if(!(p->da.base <= el && el < p->da.base+p->da.pos))
+		return false;
+	// sanity check: it should be aligned (if pool has fixed-size elements)
+	if(p->el_size)
+		debug_assert((uintptr_t)((u8*)el - p->da.base) % p->el_size == 0);
+	return true;
+}
+
+
+void* pool_alloc(Pool* p, size_t size)
+{
+	// if pool allows variable sizes, go with the size parameter,
+	// otherwise the pool el_size setting.
+	const size_t el_size = p->el_size? p->el_size : mem_RoundUpToAlignment(size);
+
+	// note: this can never happen in pools with variable-sized elements
+	// because they disallow pool_free.
+	void* el = mem_freelist_Detach(p->freelist);
+	if(el)
+		goto have_el;
+
+	// alloc a new entry
+	{
+		// expand, if necessary
+		if(da_reserve(&p->da, el_size) < 0)
+			return 0;
+
+		el = p->da.base + p->da.pos;
+		p->da.pos += el_size;
+	}
+
+have_el:
+	debug_assert(pool_contains(p, el));	// paranoia
+	return el;
+}
+
+
+void pool_free(Pool* p, void* el)
+{
+	// only allowed to free items if we were initialized with
+	// fixed el_size. (this avoids having to pass el_size here and
+	// check if requested_size matches that when allocating)
+	if(p->el_size == 0)
+	{
+		debug_warn("cannot free variable-size items");
+		return;
+	}
+
+	if(pool_contains(p, el))
+		mem_freelist_AddToFront(p->freelist, el);
+	else
+		debug_warn("invalid pointer (not in pool)");
+}
+
+
+void pool_free_all(Pool* p)
+{
+	p->freelist = 0;
+
+	// must be reset before da_set_size or CHECK_DA will complain.
+	p->da.pos = 0;
+
+	da_set_size(&p->da, 0);
+}

source/lib/allocators/pool.h

@@ -0,0 +1,161 @@
+/**
+ * =========================================================================
+ * File        : pool.h
+ * Project     : 0 A.D.
+ * Description : pool allocator
+ * =========================================================================
+ */
+
+// license: GPL; see lib/license.txt
+
+#ifndef INCLUDED_POOL
+#define INCLUDED_POOL
+
+#include "dynarray.h"
+
+/**
+ * allocator design parameters:
+ * - O(1) alloc and free;
+ * - either fixed- or variable-sized blocks;
+ * - doesn't preallocate the entire pool;
+ * - returns sequential addresses.
+ *
+ * opaque! do not read/write any fields!
+ **/
+struct Pool
+{
+	DynArray da;
+
+	/**
+	 * size of elements. = 0 if pool set up for variable-sized
+	 * elements, otherwise rounded up to pool alignment.
+	 **/
+	size_t el_size;
+
+	/**
+	 * pointer to freelist (opaque); see freelist_*.
+	 * never used (remains 0) if elements are of variable size.
+	 **/
+	void* freelist;
+};
+
+/**
+ * pass as pool_create's <el_size> param to indicate variable-sized allocs
+ * are required (see below).
+ **/
+const size_t POOL_VARIABLE_ALLOCS = ~0u;
+
+/**
+ * Ready Pool for use.
+ *
+ * @param Pool*
+ * @param max_size Max size [bytes] of the Pool; this much
+ * (rounded up to next page multiple) virtual address space is reserved.
+ * no virtual memory is actually committed until calls to pool_alloc.
+ * @param el_size Number of bytes that will be returned by each
+ * pool_alloc (whose size parameter is then ignored). Can be 0 to
+ * allow variable-sized allocations, but pool_free is then unusable.
+ * @return LibError
+ **/
+extern LibError pool_create(Pool* p, size_t max_size, size_t el_size);
+
+/**
+ * free all memory (address space + physical) that constitutes the
+ * given Pool.
+ *
+ * future alloc and free calls on this pool will fail.
+ * continued use of the allocated memory (*) is
+ * impossible because it is marked not-present via MMU.
+ * (* no matter if in freelist or unused or "allocated" to user)
+ *
+ * @param Pool*
+ * @return LibError.
+ **/
+extern LibError pool_destroy(Pool* p);
+
+/**
+ * indicate whether a pointer was allocated from the given pool.
+ *
+ * this is useful for callers that use several types of allocators.
+ *
+ * @param Pool*
+ * @return bool.
+ **/
+extern bool pool_contains(const Pool* p, void* el);
+
+/**
+ * Dole out memory from the pool.
+ * exhausts the freelist before returning new entries to improve locality.
+ *
+ * @param Pool*
+ * @param size bytes to allocate; ignored if pool_create's el_size was not 0.
+ * @return allocated memory, or 0 if the Pool would have to be expanded and
+ * there isn't enough memory to do so.
+ **/
+extern void* pool_alloc(Pool* p, size_t size);
+
+/**
+ * Make a fixed-size element available for reuse in the given Pool.
+ *
+ * this is not allowed if the Pool was created for variable-size elements.
+ * rationale: avoids having to pass el_size here and compare with size when
+ * allocating; also prevents fragmentation and leaking memory.
+ *
+ * @param Pool*
+ * @param el Element returned by pool_alloc.
+ **/
+extern void pool_free(Pool* p, void* el);
+
+/**
+ * "free" all user allocations that ensued from the given Pool.
+ *
+ * this resets it as if freshly pool_create-d, but doesn't release the
+ * underlying reserved virtual memory.
+ *
+ * @param Pool*
+ **/
+extern void pool_free_all(Pool* p);
+
+
+#ifdef __cplusplus
+
+/**
+ * C++ wrapper on top of pool_alloc that's slightly easier to use.
+ *
+ * T must be POD (Plain Old Data) because it is memset to 0!
+ **/
+template<class T>
+class PoolAllocator
+{
+public:
+	explicit PoolAllocator(size_t maxElements)
+	{
+		(void)pool_create(&m_pool, maxElements*sizeof(T), sizeof(T));
+	}
+
+	~PoolAllocator()
+	{
+		(void)pool_destroy(&m_pool);
+	}
+
+	T* AllocateZeroedMemory()
+	{
+		T* t = (T*)pool_alloc(&m_pool, 0);
+		if(!t)
+			throw std::bad_alloc();
+		memset(t, 0, sizeof(T));
+		return t;
+	}
+
+	void Free(T* t)
+	{
+		pool_free(&m_pool, t);
+	}
+
+private:
+	Pool m_pool;
+};
+
+#endif
+
+#endif	// #ifndef INCLUDED_POOL

source/lib/allocators/tests/test_allocators.h

@@ -0,0 +1,45 @@
+#include "lib/self_test.h"
+
+#include "lib/allocators/allocators.h"
+#include "lib/allocators/dynarray.h"
+#include "lib/byte_order.h"
+
+class TestAllocators : public CxxTest::TestSuite 
+{
+public:
+	void test_da()
+	{
+		DynArray da;
+
+		// basic test of functionality (not really meaningful)
+		TS_ASSERT_OK(da_alloc(&da, 1000));
+		TS_ASSERT_OK(da_set_size(&da, 1000));
+		TS_ASSERT_OK(da_set_prot(&da, PROT_NONE));
+		TS_ASSERT_OK(da_free(&da));
+
+		// test wrapping existing mem blocks for use with da_read
+		u8 data[4] = { 0x12, 0x34, 0x56, 0x78 };
+		TS_ASSERT_OK(da_wrap_fixed(&da, data, sizeof(data)));
+		u8 buf[4];
+		TS_ASSERT_OK(da_read(&da, buf, 4));
+		TS_ASSERT_EQUALS(read_le32(buf), 0x78563412);	// read correct value
+		debug_skip_next_err(ERR::FAIL);
+		TS_ASSERT(da_read(&da, buf, 1) < 0);		// no more data left
+		TS_ASSERT_OK(da_free(&da));
+	}
+
+	void test_matrix()
+	{
+		// not much we can do here; allocate a matrix, write to it and
+		// make sure it can be freed.
+		// (note: can't check memory layout because "matrix" is int** -
+		// array of pointers. the matrix interface doesn't guarantee
+		// that data comes in row-major order after the row pointers)
+		int** m = (int**)matrix_alloc(3, 3, sizeof(int));
+		m[0][0] = 1;
+		m[0][1] = 2;
+		m[1][0] = 3;
+		m[2][2] = 4;
+		matrix_free((void**)m);
+	}
+};

source/lib/allocators/tests/test_headerless.h

@@ -0,0 +1,113 @@
+#include "lib/self_test.h"
+
+#include "lib/allocators/headerless.h"
+
+class TestHeaderless: public CxxTest::TestSuite 
+{
+public:
+	void test_Basic()
+	{
+		HeaderlessAllocator a(8192);
+
+		// can't Allocate unaligned sizes
+		TS_ASSERT_EQUALS(a.Allocate(1), 0);
+
+		// can't Allocate too small amounts
+		TS_ASSERT_EQUALS(a.Allocate(16), 0);
+
+		// can Allocate the entire pool
+		char* p1 = (char*)a.Allocate(4096);
+		char* p2 = (char*)a.Allocate(4096);
+		TS_ASSERT_DIFFERS(p1, 0);
+		TS_ASSERT_DIFFERS(p2, 0);
+
+		// back-to-back (non-freelist) allocations should be contiguous
+		TS_ASSERT_EQUALS(p1+4096, p2);
+
+		// allocations are writable
+		p1[0] = 11;
+		p1[4095] = 12;
+	}
+
+	void test_Free()
+	{
+		// Deallocate allows immediate reuse of the freed pointer
+		HeaderlessAllocator a(4096);
+		void* p1 = a.Allocate(1024);
+		a.Deallocate(p1, 1024);
+		void* p2 = a.Allocate(1024);
+		TS_ASSERT_EQUALS(p1, p2);
+	}
+
+	void test_Coalesce()
+	{
+		HeaderlessAllocator a(0x10000);
+
+		// can Allocate non-power-of-two sizes
+		void* p1 = a.Allocate(0x5670);
+		void* p2 = a.Allocate(0x7890);
+		void* p3 = a.Allocate(0x1230);
+		TS_ASSERT_DIFFERS(p1, 0);
+		TS_ASSERT_DIFFERS(p2, 0);
+		TS_ASSERT_DIFFERS(p3, 0);
+
+		// must be able to allocate the entire range after freeing the items
+		a.Deallocate(p1, 0x5670);
+		a.Deallocate(p2, 0x7890);
+		a.Deallocate(p3, 0x1230);
+		void* p4 = a.Allocate(0x10000);
+		TS_ASSERT_DIFFERS(p4, 0);
+	}
+
+	void test_Reset()
+	{
+		// after Reset, must return the same pointer as a freshly constructed instance
+		HeaderlessAllocator a(4096);
+		void* p1 = a.Allocate(128);
+		a.Reset();
+		void* p2 = a.Allocate(128);
+		TS_ASSERT_EQUALS(p1, p2);
+	}
+
+	// will the allocator survive a series of random but valid Allocate/Deallocate?
+	void test_Randomized()
+	{
+		const size_t poolSize = 1024*1024;
+		HeaderlessAllocator a(poolSize);
+
+		typedef std::map<void*, size_t> AllocMap;
+		AllocMap allocs;
+
+		srand(1);
+
+		for(int i = 0; i < 1000; i++)
+		{
+			// allocate
+			if(rand() >= RAND_MAX/2)
+			{
+				const size_t maxSize = (size_t)((rand() / (float)RAND_MAX) * poolSize);
+				const size_t size = maxSize & ~0xFu;
+				void* p = a.Allocate(size);
+				if(!p)
+					continue;
+				TS_ASSERT(allocs.find(p) == allocs.end());
+				allocs[p] = size;
+			}
+			// free
+			else
+			{
+				if(allocs.empty())
+					continue;
+				// find random allocation to deallocate
+				AllocMap::iterator it = allocs.begin();
+				const int numToSkip = rand() % allocs.size();
+				for(int skip = 0; skip < numToSkip; skip++)
+					++it;
+				void* p = (*it).first;
+				size_t size = (*it).second;
+				allocs.erase(it);
+				a.Deallocate(p, size);
+			}
+		}
+	}
+};