allocator: add single_calloc (generalization of code previously in wposix for WDIR); add da_reserve; expand pool allocator to allow variable size entries

byte_order: add FOURCC_BE and _LE versions (needed for zip)
ogl: quick hack: prevent crash on laptop by providing a
pglDrawRangeElementsEXT stub
file, h_mgr: update pool_alloc call site

tex: fix incorrect indexing/stride for mipmap generation

vfs_mount: disable archives (zip code is WIP)
zip: heavy WIP. add support for writing archives (needed for thesis);
revised Zip read code. sped up archive open. further major changes
pending.

wposix: moved allocator code (see above)

This was SVN commit r3298.

source/lib/allocators.cpp

@@ -16,11 +16,69 @@
 //   http://www.stud.uni-karlsruhe.de/~urkt/
 
 #include "precompiled.h"
+
 #include "posix.h"
+#include "sysdep/cpu.h"	// CAS
 
 #include "allocators.h"
 
 
+//-----------------------------------------------------------------------------
+// allocator optimized for single instances
+//-----------------------------------------------------------------------------
+
+// intended for applications that frequently alloc/free a single
+// fixed-size object. caller provides static storage and an in-use flag;
+// we use that memory if available and otherwise fall back to the heap.
+// if the application only has one object in use at a time, malloc is
+// avoided; this is faster and avoids heap fragmentation.
+//
+// thread-safe.
+
+void* single_calloc(void* storage, volatile uintptr_t* in_use_flag, size_t size)
+{
+	// sanity check
+	debug_assert(*in_use_flag == 0 || *in_use_flag == 1);
+
+	void* p;
+
+	// successfully reserved the single instance
+	if(CAS(in_use_flag, 0, 1))
+		p = storage;
+	// already in use (rare) - allocate from heap
+	else
+		p = malloc(size);
+
+	memset(p, 0, size);
+	return p;
+}
+
+
+void single_free(void* storage, volatile uintptr_t* in_use_flag, void* p)
+{
+	// sanity check
+	debug_assert(*in_use_flag == 0 || *in_use_flag == 1);
+
+	if(p == storage)
+	{
+		if(CAS(in_use_flag, 1, 0))
+		{
+			// ok, flag has been reset to 0
+		}
+		else
+			debug_warn("in_use_flag out of sync (double free?)");
+	}
+	// was allocated from heap
+	else
+	{
+		// single instance may have been freed by now - cannot assume
+		// anything about in_use_flag.
+
+		free(p);
+	}
+}
+
+
 //-----------------------------------------------------------------------------
 // dynamic (expandable) array
 //-----------------------------------------------------------------------------
@@ -224,6 +282,20 @@ LibError da_set_size(DynArray* da, size_t new_size)
 }
 
 
+// make sure at least <size> bytes starting at <pos> are committed and
+// ready for use.
+LibError da_reserve(DynArray* da, size_t size)
+{
+	// default to page size (the OS won't commit less anyway);
+	// grab more if request requires it.
+	const size_t expand_amount = MIN(4*KiB, size);
+
+	if(da->pos + size > da->cur_size)
+		return da_set_size(da, da->cur_size + expand_amount);
+	return ERR_OK;
+}
+
+
 // change access rights of the array memory; used to implement
 // write-protection. affects the currently committed pages as well as
 // all subsequently added pages.
@@ -266,7 +338,7 @@ LibError da_read(DynArray* da, void* data, size_t size)
 // starts at offset DynArray.pos and advances this.
 LibError da_append(DynArray* da, const void* data, size_t size)
 {
-	RETURN_ERR(da_set_size(da, da->pos+size));
+	RETURN_ERR(da_reserve(da, size));
 	memcpy2(da->base+da->pos, data, size);
 	da->pos += size;
 	return ERR_OK;
@@ -279,7 +351,7 @@ LibError da_append(DynArray* da, const void* data, size_t size)
 
 // design parameters:
 // - O(1) alloc and free;
-// - fixed-size blocks;
+// - fixed XOR variable size blocks;
 // - doesn't preallocate the entire pool;
 // - returns sequential addresses.
 
@@ -308,15 +380,18 @@ static void* freelist_pop(void** pfreelist)
 static const size_t POOL_CHUNK = 4*KiB;
 
 
-// ready <p> for use. pool_alloc will return chunks of memory that
-// are exactly <el_size> bytes. <max_size> is the upper limit [bytes] on
+// ready <p> for use. <max_size> is the upper limit [bytes] on
 // pool size (this is how much address space is reserved).
 //
-// note: el_size must at least be enough for a pointer (due to freelist
-// implementation) but not exceed the expand-by amount.
+// <el_size> can be 0 to allow variable-sized allocations
+//  (which cannot be freed individually);
+// otherwise, it specifies the number of bytes that will be
+// returned by pool_alloc (whose size parameter is then ignored).
+// in the latter case, size must at least be enough for a pointer
+//  (due to freelist implementation).
 LibError pool_create(Pool* p, size_t max_size, size_t el_size)
 {
-	if(el_size < sizeof(void*) || el_size > POOL_CHUNK)
+	if(el_size != 0 && el_size < sizeof(void*))
 		CHECK_ERR(ERR_INVALID_PARAM);
 
 	RETURN_ERR(da_alloc(&p->da, max_size));
@@ -355,8 +430,17 @@ bool pool_contains(Pool* p, void* el)
 // return an entry from the pool, or 0 if it would have to be expanded and
 // there isn't enough memory to do so.
 // exhausts the freelist before returning new entries to improve locality.
-void* pool_alloc(Pool* p)
+//
+// if the pool was set up with fixed-size elements, <size> is ignored;
+// otherwise, <size> bytes are allocated.
+void* pool_alloc(Pool* p, size_t size)
 {
+	// if pool allows variable sizes, go with the size parameter,
+	// otherwise the pool el_size setting.
+	const size_t el_size = p->el_size? p->el_size : size;
+
+	// note: this can never happen in pools with variable-sized elements
+	// because they disallow pool_free.
 	void* el = freelist_pop(&p->freelist);
 	if(el)
 		goto have_el;
@@ -364,12 +448,11 @@ void* pool_alloc(Pool* p)
 	// alloc a new entry
 	{
 		// expand, if necessary
-		if(p->pos + p->el_size > p->da.cur_size)
-			if(da_set_size(&p->da, p->da.cur_size + POOL_CHUNK) < 0)
-				return 0;
+		if(da_reserve(&p->da, el_size) < 0)
+			return 0;
 
 		el = p->da.base + p->pos;
-		p->pos += p->el_size;
+		p->pos += el_size;
 	}
 
 have_el:
@@ -379,8 +462,19 @@ have_el:
 
 
 // make <el> available for reuse in the given pool.
+//
+// this is not allowed if the pool was set up for variable-size elements.
+// (copying with fragmentation would defeat the point of a pool - simplicity)
+// we could allow this, but instead warn and bail to make sure it
+// never happens inadvertently (leaking memory in the pool).
 void pool_free(Pool* p, void* el)
 {
+	if(p->el_size == 0)
+	{
+		debug_warn("pool is set up for variable-size items");
+		return;
+	}
+
 	if(pool_contains(p, el))
 		freelist_push(&p->freelist, el);
 	else
@@ -388,6 +482,16 @@ void pool_free(Pool* p, void* el)
 }
 
 
+// "free" all allocations that ensued from the given Pool.
+// this resets it as if freshly pool_create-d, but doesn't release the
+// underlying memory.
+void pool_free_all(Pool* p)
+{
+	p->pos = 0;
+	p->freelist = 0;
+}
+
+
 //-----------------------------------------------------------------------------
 // bucket allocator
 //-----------------------------------------------------------------------------
@@ -396,6 +500,7 @@ void pool_free(Pool* p, void* el)
 // - variable-size allocations;
 // - no reuse of allocations, can only free all at once;
 // - no init necessary;
+// - never relocates;
 // - no fixed limit.
 
 // must be constant and power-of-2 to allow fast modulo.

source/lib/allocators.h

@@ -22,6 +22,23 @@
 #include "lib/posix.h"	// PROT_* constants for da_set_prot
 
 
+//
+// allocator optimized for single instances
+//
+
+// intended for applications that frequently alloc/free a single
+// fixed-size object. caller provides static storage and an in-use flag;
+// we use that memory if available and otherwise fall back to the heap.
+// if the application only has one object in use at a time, malloc is
+// avoided; this is faster and avoids heap fragmentation.
+//
+// thread-safe.
+
+extern void* single_calloc(void* storage, volatile uintptr_t* in_use_flag, size_t size);
+
+extern void single_free(void* storage, volatile uintptr_t* in_use_flag, void* p);
+
+
 //
 // dynamic (expandable) array
 //
@@ -57,6 +74,10 @@ extern LibError da_free(DynArray* da);
 // new_size (rounded up to the next page size multiple) is met.
 extern LibError da_set_size(DynArray* da, size_t new_size);
 
+// make sure at least <size> bytes starting at <pos> are committed and
+// ready for use.
+extern LibError da_reserve(DynArray* da, size_t size);
+
 // change access rights of the array memory; used to implement
 // write-protection. affects the currently committed pages as well as
 // all subsequently added pages.
@@ -85,7 +106,7 @@ extern LibError da_append(DynArray* da, const void* data_src, size_t size);
 
 // design parameters:
 // - O(1) alloc and free;
-// - fixed-size blocks;
+// - fixed XOR variable size blocks;
 // - doesn't preallocate the entire pool;
 // - returns sequential addresses.
 
@@ -93,21 +114,27 @@ extern LibError da_append(DynArray* da, const void* data_src, size_t size);
 struct Pool
 {
 	DynArray da;
+
+	// size of elements; see pool_create.
 	size_t el_size;
 
 	// all bytes in da up to this mark are in circulation or freelist.
 	size_t pos;
 
 	// pointer to freelist (opaque); see freelist_*.
+	// never used (remains 0) if elements are of variable size.
 	void* freelist;
 };
 
-// ready <p> for use. pool_alloc will return chunks of memory that
-// are exactly <el_size> bytes. <max_size> is the upper limit [bytes] on
+// ready <p> for use. <max_size> is the upper limit [bytes] on
 // pool size (this is how much address space is reserved).
 //
-// note: el_size must at least be enough for a pointer (due to freelist
-// implementation) but not exceed the expand-by amount.
+// <el_size> can be 0 to allow variable-sized allocations
+//  (which cannot be freed individually);
+// otherwise, it specifies the number of bytes that will be
+// returned by pool_alloc (whose size parameter is then ignored).
+// in the latter case, size must at least be enough for a pointer
+//  (due to freelist implementation).
 extern LibError pool_create(Pool* p, size_t max_size, size_t el_size);
 
 // free all memory that ensued from <p>. all elements are made unusable
@@ -122,11 +149,24 @@ extern bool pool_contains(Pool* p, void* el);
 // return an entry from the pool, or 0 if it would have to be expanded and
 // there isn't enough memory to do so.
 // exhausts the freelist before returning new entries to improve locality.
-extern void* pool_alloc(Pool* p);
+//
+// if the pool was set up with fixed-size elements, <size> is ignored;
+// otherwise, <size> bytes are allocated.
+extern void* pool_alloc(Pool* p, size_t size);
 
 // make <el> available for reuse in the given pool.
+//
+// this is not allowed if the pool was set up for variable-size elements.
+// (copying with fragmentation would defeat the point of a pool - simplicity)
+// we could allow this, but instead warn and bail to make sure it
+// never happens inadvertently (leaking memory in the pool).
 extern void pool_free(Pool* p, void* el);
 
+// "free" all allocations that ensued from the given Pool.
+// this resets it as if freshly pool_create-d, but doesn't release the
+// underlying memory.
+extern void pool_free_all(Pool* p);
+
 
 //
 // bucket allocator
@@ -136,6 +176,7 @@ extern void pool_free(Pool* p, void* el);
 // - variable-size allocations;
 // - no reuse of allocations, can only free all at once;
 // - no init necessary;
+// - never relocates;
 // - no fixed limit.
 
 // opaque! do not read/write any fields!

source/lib/byte_order.h

@@ -9,12 +9,15 @@
 // the additional u8 cast ensures each character is treated as unsigned
 // (otherwise, they'd be promoted to signed int before the u32 cast,
 // which would break things).
+#define FOURCC_BE(a,b,c,d) ( ((u32)(u8)a) << 24 | ((u32)(u8)b) << 16 | \
+                             ((u32)(u8)c) << 8  | ((u32)(u8)d) << 0  )
+#define FOURCC_LE(a,b,c,d) ( ((u32)(u8)a) << 0  | ((u32)(u8)b) << 8  | \
+                             ((u32)(u8)c) << 16 | ((u32)(u8)d) << 24 )
+
 #if BYTE_ORDER == BIG_ENDIAN
-#define FOURCC(a,b,c,d) ( ((u32)(u8)a) << 24 | ((u32)(u8)b) << 16 | \
-	((u32)(u8)c) << 8  | ((u32)(u8)d) << 0  )
+# define FOURCC FOURCC_BE
 #else
-#define FOURCC(a,b,c,d) ( ((u32)(u8)a) << 0  | ((u32)(u8)b) << 8  | \
-	((u32)(u8)c) << 16 | ((u32)(u8)d) << 24 )
+# define FOURCC FOURCC_LE
 #endif
 
 

source/lib/ogl.cpp

@@ -351,6 +351,10 @@ LibError ogl_get_gfx_info()
 }
 
 
+static void __stdcall emu_glDrawRangeElementsEXT(GLenum, GLuint, GLuint, GLsizei, GLenum, GLvoid*)
+{
+}
+
 // call after each video mode change, since thereafter extension functions
 // may have changed [address].
 void oglInit()
@@ -368,6 +372,9 @@ void oglInit()
 
 	importExtensionFunctions();
 
+if(!pglDrawRangeElementsEXT)
+pglDrawRangeElementsEXT = emu_glDrawRangeElementsEXT;
+
 	glGetIntegerv(GL_MAX_TEXTURE_SIZE, &ogl_max_tex_size);
 	glGetIntegerv(GL_MAX_TEXTURE_UNITS, &ogl_max_tex_units);
 }

source/lib/res/file/file.cpp

@@ -726,7 +726,7 @@ static inline void aiocb_pool_shutdown()
 static inline aiocb* aiocb_pool_alloc()
 {
 	ONCE(aiocb_pool_init());
-	return (aiocb*)pool_alloc(&aiocb_pool);
+	return (aiocb*)pool_alloc(&aiocb_pool, 0);
 }
 
 static inline void aiocb_pool_free(void* cb)

source/lib/res/file/vfs_mount.cpp

@@ -247,6 +247,8 @@ typedef Archives::const_iterator ArchiveCIt;
 // was successfully added to the list. see comments below.
 static LibError enqueue_archive(const char* name, const char* P_archive_dir, Archives* archives)
 {
+archives=0;// HACK HACK HACK: disables zip files (WIP)
+
 	// caller doesn't want us to check if this is a Zip file. this is the
 	// case in all subdirectories of the mount point, since checking for all
 	// mounted files would be slow. see mount_dir_tree.

source/lib/res/graphics/tex.cpp

@@ -116,6 +116,8 @@ static void create_level(uint level, uint level_w, uint level_h,
 	const u8* restrict level_data, size_t level_data_size, void* restrict ctx)
 {
 	CreateLevelData* cld = (CreateLevelData*)ctx;
+	const size_t src_w = cld->prev_level_w;
+	const size_t src_h = cld->prev_level_h;
 	const u8* src = cld->prev_level_data;
 	u8* dst = (u8*)level_data;
 
@@ -128,36 +130,42 @@ static void create_level(uint level, uint level_w, uint level_h,
 	else
 	{
 		const uint num_components = cld->num_components;
-		const size_t dx = num_components, dy = dx*level_w*2;
+		const size_t dx = num_components, dy = dx*src_w;
 
 		// special case: image is too small for 2x2 filter
 		if(cld->prev_level_w == 1 || cld->prev_level_h == 1)
 		{
-			for(uint y = 0; y < level_h; y++)
+			// image is either a horizontal or vertical line.
+			// their memory layout is the same (packed pixels), so no special
+			// handling is needed; just pick max dimension.
+			for(uint y = 0; y < MAX(src_w, src_h); y += 2)
 			{
 				for(uint i = 0; i < num_components; i++)
 				{
-					*dst++ = (src[0]+src[dy]+1)/2;
+					*dst++ = (src[0]+src[dx]+1)/2;
 					src += 1;
 				}
-				src += dy;
+
+				src += dx;	// skip to next pixel (since box is 2x2)
 			}
 		}
 		// normal
 		else
 		{
-			for(uint y = 0; y < level_h; y++)
+			for(uint y = 0; y < src_h; y += 2)
 			{
-				for(uint x = 0; x < level_w; x++)
+				for(uint x = 0; x < src_w; x += 2)
 				{
 					for(uint i = 0; i < num_components; i++)
 					{
 						*dst++ = (src[0]+src[dx]+src[dy]+src[dx+dy]+2)/4;
 						src += 1;
 					}
-					src += dx;
+
+					src += dx;	// skip to next pixel (since box is 2x2)
 				}
-				src += dy;
+
+				src += dy;	// skip to next row (since box is 2x2)
 			}
 		}
 

source/lib/res/h_mgr.cpp

@@ -429,7 +429,7 @@ static void fn_store(HDATA* hd, const char* fn)
 	if(hd->type->user_size+size <= HDATA_USER_SIZE)
 		hd->fn = (const char*)hd->user + hd->type->user_size;
 	else if(size <= FN_POOL_EL_SIZE)
-		hd->fn = (const char*)pool_alloc(&fn_pool);
+		hd->fn = (const char*)pool_alloc(&fn_pool, 0);
 
 	// in case none of the above applied and/or were successful:
 	// fall back to heap alloc.

source/lib/sysdep/win/wposix.cpp

@@ -24,7 +24,7 @@
 #include "lib.h"
 #include "posix.h"
 #include "win_internal.h"
-#include "sysdep/cpu.h"
+#include "allocators.h"
 
 
 // cast intptr_t to HANDLE; centralized for easier changing, e.g. avoiding
@@ -332,11 +332,9 @@ struct WDIR
 };
 
 
-// suballocator - satisfies most requests with a reusable static instance.
-// this avoids hundreds of alloc/free which would fragment the heap.
-// to guarantee thread-safety, we fall back to malloc if the instance is
-// already in use. (it's important to avoid suprises since this is such a
-// low-level routine).
+// suballocator - satisfies most requests with a reusable static instance,
+// thus speeding up allocation and avoiding heap fragmentation.
+// thread-safe.
 
 static WDIR global_wdir;
 static uintptr_t global_wdir_is_in_use;
@@ -344,26 +342,12 @@ static uintptr_t global_wdir_is_in_use;
 // zero-initializes the WDIR (code below relies on this)
 static inline WDIR* wdir_alloc()
 {
-	WDIR* d;
-
-	// successfully reserved the global instance
-	if(CAS(&global_wdir_is_in_use, 0, 1))
-	{
-		d = &global_wdir;
-		memset(d, 0, sizeof(*d));
-	}
-	else
-		d = (WDIR*)calloc(1, sizeof(WDIR));
-
-	return d;
+	return (WDIR*)single_calloc(&global_wdir, &global_wdir_is_in_use, sizeof(WDIR));
 }
 
 static inline void wdir_free(WDIR* d)
 {
-	if(d == &global_wdir)
-		global_wdir_is_in_use = 0;
-	else
-		free(d);
+	single_free(&global_wdir, &global_wdir_is_in_use, d);
 }