# Add -archivebuild mode to generate .zip files for releases, with automatic compression of textures.

Fix terrain manager to understand .cached.dds files.
Fix IArchiveWriter so you can pass it absolute paths of files to load,
and different relative paths for storing inside the archive.
Fix fs_util::ForEachFile when called on the VFS root.

This was SVN commit r8130.

source/graphics/TerrainTextureManager.cpp

@@ -31,6 +31,8 @@
 #include "ps/CLogger.h"
 #include "ps/Filesystem.h"
 
+#include <boost/algorithm/string.hpp>
+
 #define LOG_CATEGORY L"graphics"
 
 CTerrainTextureManager::CTerrainTextureManager():
@@ -111,6 +113,19 @@ void CTerrainTextureManager::LoadTextures(const CTerrainPropertiesPtr& props, co
 	VfsPaths pathnames;
 	if(fs_util::GetPathnames(g_VFS, path, 0, pathnames) < 0)
 		return;
+
+	// If we have any .cached.dds files then strip that extension to get the
+	// 'real' texture name
+	for(size_t i = 0; i < pathnames.size(); i++)
+	{
+		if(boost::algorithm::ends_with(pathnames[i].leaf(), L".cached.dds"))
+			pathnames[i] = pathnames[i].branch_path() / boost::algorithm::erase_last_copy(pathnames[i].leaf(), L".cached.dds");
+	}
+
+	// Remove any duplicates created by the stripping
+	std::sort(pathnames.begin(), pathnames.end());
+	pathnames.erase(std::unique(pathnames.begin(), pathnames.end()), pathnames.end());
+
 	for(size_t i = 0; i < pathnames.size(); i++)
 	{
 		// skip files that obviously aren't textures.

source/graphics/TextureManager.cpp

@@ -271,7 +271,7 @@ public:
 		VfsPath sourcePath = texture->m_Properties.m_Path;
 		VfsPath sourceDir = sourcePath.branch_path();
 		std::wstring sourceName = sourcePath.leaf();
-		VfsPath archiveCachePath = sourceDir / (sourceName + L".dds");
+		VfsPath archiveCachePath = sourceDir / (sourceName + L".cached.dds");
 
 		// Try the archive cache file first
 		if (CanUseArchiveCache(sourcePath, archiveCachePath))
@@ -371,6 +371,35 @@ public:
 		m_TextureConverter.ConvertTexture(texture, sourcePath, looseCachePath, settings);
 	}
 
+	bool GenerateCachedTexture(const VfsPath& sourcePath, VfsPath& archiveCachePath)
+	{
+		VfsPath sourceDir = sourcePath.branch_path();
+		std::wstring sourceName = sourcePath.leaf();
+		archiveCachePath = sourceDir / (sourceName + L".cached.dds");
+
+		CTextureProperties textureProps(sourcePath);
+		CTexturePtr texture = CreateTexture(textureProps);
+		CTextureConverter::Settings settings = GetConverterSettings(texture);
+
+		// TODO: we ought to use a higher-quality compression mode here
+		// (since we don't care about performance)
+
+		if (!m_TextureConverter.ConvertTexture(texture, sourcePath, L"cache"/archiveCachePath, settings))
+			return false;
+
+		while (true)
+		{
+			CTexturePtr textureOut;
+			VfsPath dest;
+			bool ok;
+			if (m_TextureConverter.Poll(textureOut, dest, ok))
+				return ok;
+
+			// Spin-loop is dumb but it works okay for now
+			SDL_Delay(0);
+		}
+	}
+
 	bool MakeProgress()
 	{
 		// Process any completed conversion tasks
@@ -672,3 +701,8 @@ bool CTextureManager::MakeProgress()
 {
 	return m->MakeProgress();
 }
+
+bool CTextureManager::GenerateCachedTexture(const VfsPath& path, VfsPath& outputPath)
+{
+	return m->GenerateCachedTexture(path, outputPath);
+}

source/graphics/TextureManager.h

@@ -100,6 +100,14 @@ public:
 	 */
 	bool MakeProgress();
 
+	/**
+	 * Synchronously converts and compresses and saves the texture,
+	 * and returns the output path (minus a "cache/" prefix). This
+	 * is intended for pre-caching textures in release archives.
+	 * @return true on success
+	 */
+	bool GenerateCachedTexture(const VfsPath& path, VfsPath& outputPath);
+
 private:
 	CTextureManagerImpl* m;
 };

source/lib/file/archive/archive.h

@@ -87,8 +87,11 @@ struct IArchiveWriter
 	 * archived versions. however, the archive builder usually adds files
 	 * precisely because they aren't in archives, and the cache would
 	 * thrash anyway, so this is deemed acceptable.
+	 *
+	 * @param sourcepathname the path to the source file on the filesystem
+	 * @param pathname the path to use for the file inside the archive
 	 **/
-	virtual LibError AddFile(const fs::wpath& pathname) = 0;
+	virtual LibError AddFile(const fs::wpath& sourcepathname, const fs::wpath& pathame) = 0;
 };
 
 typedef shared_ptr<IArchiveWriter> PIArchiveWriter;

source/lib/file/archive/archive_zip.cpp

@@ -560,10 +560,10 @@ public:
 		wtruncate(pathname.string().c_str(), m_fileSize);
 	}
 
-	LibError AddFile(const fs::wpath& pathname)
+	LibError AddFile(const fs::wpath& sourcepathname, const fs::wpath& pathname)
 	{
 		FileInfo fileInfo;
-		RETURN_ERR(GetFileInfo(pathname, &fileInfo));
+		RETURN_ERR(GetFileInfo(sourcepathname, &fileInfo));
 		const off_t usize = fileInfo.Size();
 		// skip 0-length files.
 		// rationale: zip.cpp needs to determine whether a CDFH entry is
@@ -577,7 +577,7 @@ public:
 			return INFO::SKIPPED;
 
 		PFile file(new File);
-		RETURN_ERR(file->Open(pathname, 'r'));
+		RETURN_ERR(file->Open(sourcepathname, 'r'));
 
 		const size_t pathnameLength = pathname.string().length();
 

source/lib/file/file_system_util.cpp

@@ -112,7 +112,15 @@ LibError ForEachFile(const PIVFS& fs, const VfsPath& startPath, FileCallback cb,
 			break;
 
 		for(size_t i = 0; i < subdirectoryNames.size(); i++)
-			pendingDirectories.push(AddSlash(path/subdirectoryNames[i]));
+		{
+			VfsPath pathname;
+			if (path.string() == L"/") // special case for startPath == L""
+				pathname = AddSlash(VfsPath(subdirectoryNames[i]));
+			else
+				pathname = AddSlash(path/subdirectoryNames[i]);
+
+			pendingDirectories.push(pathname);
+		}
 		pendingDirectories.pop();
 	}
 

source/main.cpp

@@ -38,9 +38,11 @@ that of Atlas depending on commandline parameters.
 #include "lib/input.h"
 #include "lib/ogl.h"
 #include "lib/timer.h"
+#include "lib/utf8.h"
 #include "lib/external_libraries/sdl.h"
 #include "lib/res/sound/snd_mgr.h"
 
+#include "ps/ArchiveBuilder.h"
 #include "ps/CConsole.h"
 #include "ps/Filesystem.h"
 #include "ps/Game.h"
@@ -454,6 +456,25 @@ static void RunGameOrAtlas(int argc, const char* argv[])
 		return;
 	}
 
+	// run in archive-building mode if requested
+	if (args.Has("archivebuild"))
+	{
+		Paths paths(args);
+
+		fs::wpath mod = wstring_from_utf8(args.Get("archivebuild"));
+		fs::wpath zip;
+		if (args.Has("archivebuild-output"))
+			zip = wstring_from_utf8(args.Get("archivebuild-output"));
+		else
+			zip = mod.leaf()+L".zip";
+
+		CArchiveBuilder builder(mod, paths.Cache());
+		builder.Build(zip);
+
+		CXeromyces::Terminate();
+		return;
+	}
+
 	const double res = timer_Resolution();
 	g_frequencyFilter = CreateFrequencyFilter(res, 30.0);
 

source/ps/ArchiveBuilder.cpp

@@ -0,0 +1,110 @@
+/* Copyright (C) 2010 Wildfire Games.
+ * This file is part of 0 A.D.
+ *
+ * 0 A.D. is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * 0 A.D. is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "precompiled.h"
+
+#include "ArchiveBuilder.h"
+
+#include "graphics/TextureManager.h"
+#include "lib/path_util.h"
+#include "lib/tex/tex_codec.h"
+#include "lib/file/archive/archive_zip.h"
+
+#include <boost/algorithm/string.hpp>
+
+CArchiveBuilder::CArchiveBuilder(const fs::wpath& mod, const fs::wpath& tempdir) :
+	m_TempDir(tempdir)
+{
+	tex_codec_register_all();
+
+	m_VFS = CreateVfs(20*MiB);
+
+	DeleteDirectory(m_TempDir/L"_archivecache"); // clean up in case the last run failed
+
+	m_VFS->Mount(L"cache/", m_TempDir/L"_archivecache/");
+
+	m_VFS->Mount(L"", AddSlash(mod), VFS_MOUNT_MUST_EXIST);
+
+	// Collect the list of files before loading any base mods
+	fs_util::ForEachFile(m_VFS, L"", &CollectFileCB, (uintptr_t)static_cast<void*>(this), 0, fs_util::DIR_RECURSIVE);
+}
+
+CArchiveBuilder::~CArchiveBuilder()
+{
+	m_VFS.reset();
+
+	DeleteDirectory(m_TempDir/L"_archivecache");
+
+	tex_codec_unregister_all();
+}
+
+void CArchiveBuilder::AddBaseMod(const fs::wpath& mod)
+{
+	m_VFS->Mount(L"", AddSlash(mod), VFS_MOUNT_MUST_EXIST);
+}
+
+void CArchiveBuilder::Build(const fs::wpath& archive)
+{
+	PIArchiveWriter writer = CreateArchiveWriter_Zip(archive);
+
+	// Use CTextureManager instead of CTextureConverter directly,
+	// so it can deal with all the loading of settings.xml files
+	CTextureManager texman(m_VFS, true);
+
+	for (size_t i = 0; i < m_Files.size(); ++i)
+	{
+		LibError ret;
+
+		fs::wpath realPath;
+		ret = m_VFS->GetRealPath(m_Files[i], realPath);
+		debug_assert(ret == INFO::OK);
+
+		// Compress textures and store the new cached version instead of the original
+		if (boost::algorithm::starts_with(m_Files[i].string(), L"art/textures/") &&
+			tex_is_known_extension(m_Files[i]) &&
+			// Skip some subdirectories where the engine doesn't use CTextureManager yet:
+			!boost::algorithm::starts_with(m_Files[i].string(), L"art/textures/cursors/") &&
+			!boost::algorithm::starts_with(m_Files[i].string(), L"art/textures/terrain/alphamaps/special/")
+		)
+		{
+			VfsPath cachedPath;
+			debug_printf(L"Converting texture %ls\n", realPath.string().c_str());
+			bool ok = texman.GenerateCachedTexture(m_Files[i], cachedPath);
+			debug_assert(ok);
+
+			fs::wpath cachedRealPath;
+			ret = m_VFS->GetRealPath(L"cache"/cachedPath, cachedRealPath);
+			debug_assert(ret == INFO::OK);
+
+			writer->AddFile(cachedRealPath, cachedPath.string());
+		}
+		// TODO: should cache XML->XMB and DAE->PMD and DAE->PSA conversions too
+		else
+		{
+			debug_printf(L"Adding %ls\n", realPath.string().c_str());
+			writer->AddFile(realPath, m_Files[i].string());
+		}
+	}
+}
+
+LibError CArchiveBuilder::CollectFileCB(const VfsPath& pathname, const FileInfo& UNUSED(fileInfo), const uintptr_t cbData)
+{
+	CArchiveBuilder* self = static_cast<CArchiveBuilder*>((void*)cbData);
+	self->m_Files.push_back(pathname);
+
+	return INFO::OK;
+}

source/ps/ArchiveBuilder.h

@@ -0,0 +1,64 @@
+/* Copyright (C) 2010 Wildfire Games.
+ * This file is part of 0 A.D.
+ *
+ * 0 A.D. is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * 0 A.D. is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef INCLUDED_ARCHIVEBUILDER
+#define INCLUDED_ARCHIVEBUILDER
+
+#include "lib/file/file_system_util.h"
+#include "ps/CStr.h"
+
+/**
+ * Packages a mod's files into a distributable archive.
+ * This includes various game-specific knowledge on how to convert
+ * and cache certain files into more efficient formats (PNG -> DDS, etc).
+ */
+class CArchiveBuilder
+{
+public:
+	/**
+	 * Initialise the archive builder for processing the given mod.
+	 * Assumes no graphics code (especially tex_codec) has been initialised yet.
+	 *
+	 * @param mod path to data/mods/foo directory, containing files for conversion
+	 * @param tempdir path to a writable directory for temporary files
+	 */
+	CArchiveBuilder(const fs::wpath& mod, const fs::wpath& tempdir);
+
+	~CArchiveBuilder();
+
+	/**
+	 * Add a mod which will be loaded but not archived, to provide
+	 * files like textures.xml needed for the conversion.
+	 * @param mod path to data/mods/foo directory, containing files for loading
+	 */
+	void AddBaseMod(const fs::wpath& mod);
+
+	/**
+	 * Do all the processing and packing of files into the archive.
+	 * @param archive path of .zip file to generate (will be overwritten if it exists)
+	 */
+	void Build(const fs::wpath& archive);
+
+private:
+	static LibError CollectFileCB(const VfsPath& pathname, const FileInfo& fileInfo, const uintptr_t cbData);
+
+	PIVFS m_VFS;
+	std::vector<VfsPath> m_Files;
+	fs::wpath m_TempDir;
+};
+
+#endif // INCLUDED_ARCHIVEBUILDER

source/ps/Filesystem.cpp

@@ -29,9 +29,14 @@ PIVFS g_VFS;
 
 static std::vector<std::pair<FileReloadFunc, void*> > g_ReloadFuncs;
 
+bool FileExists(const PIVFS& vfs, const VfsPath& pathname)
+{
+	return vfs->GetFileInfo(pathname, 0) == INFO::OK;
+}
+
 bool FileExists(const VfsPath& pathname)
 {
-	return g_VFS->GetFileInfo(pathname, 0) == INFO::OK;
+	return FileExists(g_VFS, pathname);
 }
 
 void RegisterFileReloadFunc(FileReloadFunc func, void* obj)

source/ps/Filesystem.h

@@ -30,6 +30,8 @@
 
 extern PIVFS g_VFS;
 
+extern bool FileExists(const PIVFS& vfs, const VfsPath& pathname);
+
 extern bool FileExists(const VfsPath& pathname);
 
 /**

source/ps/XML/Xeromyces.cpp

@@ -101,7 +101,7 @@ PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename)
 	debug_assert(g_XeromycesStarted);
 
 	// Make sure the .xml actually exists
-	if (! FileExists(filename))
+	if (! FileExists(vfs, filename))
 	{
 		LOG(CLogger::Error, LOG_CATEGORY, L"CXeromyces: Failed to find XML file %ls", filename.string().c_str());
 		return PSRETURN_Xeromyces_XMLOpenFailed;
@@ -145,7 +145,7 @@ PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename)
 	GetXMBPath(vfs, filename, xmbFilename, xmbPath);
 
 	// If the file exists, use it
-	if (FileExists(xmbPath))
+	if (FileExists(vfs, xmbPath))
 	{
 		if (ReadXMBFile(vfs, xmbPath))
 			return PSRETURN_OK;