Carltonus/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

Speed up GetAvailableMods for archives.

Browse Source 
Refs 64bfa089af and 44ec2e324e

When a .zip file is encountered by the VFS population, it reads the info
for all files in the archives. This is quite slow for the public archive
(400-500ms on my computer), which means calling GetEngineInfo()
repeatedly is impossible.

By only opening the external mod.json, we skip most of the work. The
archive can still be opened if needed as fallback.


Differential Revision: https://code.wildfiregames.com/D3216
This was SVN commit r25446.
This commit is contained in:
wraitii 2021-05-16 13:50:05 +00:00
parent 5ff4fa19c0
commit e908733220
10 changed files with 134 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
source/lib/sysdep/os/osx/osx_atlas.mm
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
@ -25,11 +25,15 @@
#import "osx_atlas.h"
#include <vector>
#include "lib/types.h"
#include "ps/CStr.h"
extern std::vector<CStr> g_modsLoaded;
#include <vector>
namespace Mod
{
extern std::vector<CStr> g_ModsLoaded;
}
void startNewAtlasProcess()
{
@ -39,7 +43,7 @@ void startNewAtlasProcess()
[args addObject:@"--editor"];
// Pass mods on the command line.
for (const CStr& mod : g_modsLoaded)
for (const CStr& mod : Mod::g_ModsLoaded)
{
std::string arg = std::string("-mod=") + mod;
[args addObject:[[NSString alloc] initWithUTF8String:arg.c_str()]];

6
source/main.cpp
View File

@ -99,7 +99,7 @@ that of Atlas depending on commandline parameters.
#include <chrono>
extern CmdLineArgs g_args;
extern CmdLineArgs g_CmdLineArgs;
extern CStrW g_UniqueLogPostfix;
// Marks terrain as modified so the minimap can repaint (is there a cleaner way of handling this?)
@ -183,7 +183,7 @@ static InReaction MainInputHandler(const SDL_Event_* ev)
case SDL_DROPFILE:
{
char* dropped_filedir = ev->ev.drop.file;
const Paths paths(g_args);
const Paths paths(g_CmdLineArgs);
CModInstaller installer(paths.UserData() / "mods", paths.Cache());
installer.Install(std::string(dropped_filedir), g_ScriptContext, true);
SDL_free(dropped_filedir);
@ -512,7 +512,7 @@ static void RunGameOrAtlas(int argc, const char* argv[])
{
CmdLineArgs args(argc, argv);
g_args = args;
g_CmdLineArgs = args;
if (args.Has("version"))
{

4
source/ps/GameSetup/CmdLineArgs.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2018 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -20,6 +20,8 @@
#include "lib/sysdep/sysdep.h"
CmdLineArgs g_CmdLineArgs;
namespace
{

2
source/ps/GameSetup/GameSetup.cpp
View File

@ -945,7 +945,7 @@ bool Init(const CmdLineArgs& args, int flags)
return false;
}
}
else if (!EnableModsOrSetDefault(args, g_modsLoaded, false))
else if (!EnableModsOrSetDefault(args, Mod::g_ModsLoaded, false))
return false;
}

179
source/ps/Mod.cpp
View File

@ -19,12 +19,14 @@
#include "ps/Mod.h"
#include "i18n/L10n.h"
#include "lib/file/file_system.h"
#include "lib/file/vfs/vfs.h"
#include "lib/utf8.h"
#include "ps/Filesystem.h"
#include "ps/GameSetup/GameSetup.h"
#include "ps/GameSetup/Paths.h"
#include "ps/Profiler2.h"
#include "ps/Pyrogenesis.h"
#include "scriptinterface/Object.h"
#include "scriptinterface/ScriptInterface.h"
@ -33,22 +35,71 @@
#include <algorithm>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <fstream>
#include <sstream>
#include <unordered_map>
std::vector<CStr> g_modsLoaded;
std::vector<CStr> g_incompatibleMods;
std::vector<CStr> g_failedMods;
namespace Mod
{
std::vector<CStr> g_ModsLoaded;
std::vector<CStr> g_IncompatibleMods;
std::vector<CStr> g_FailedMods;
std::vector<std::vector<CStr>> g_LoadedModVersions;
CmdLineArgs g_args;
JS::Value Mod::GetAvailableMods(const ScriptInterface& scriptInterface)
bool ParseModJSON(const ScriptRequest& rq, const PIVFS& vfs, OsPath modsPath, OsPath mod, JS::MutableHandleValue json)
{
ScriptRequest rq(scriptInterface);
JS::RootedObject obj(rq.cx, JS_NewPlainObject(rq.cx));
// Attempt to open mod.json first.
std::ifstream modjson;
modjson.open((modsPath / mod / L"mod.json").string8());
const Paths paths(g_args);
if (!modjson.is_open())
{
modjson.close();
// Fallback: open the archive and read mod.json there.
// This can take in the hundreds of milliseconds with large mods.
vfs->Clear();
if (vfs->Mount(L"", modsPath / mod / "", VFS_MOUNT_MUST_EXIST, VFS_MIN_PRIORITY) < 0)
return false;
CVFSFile modinfo;
if (modinfo.Load(vfs, L"mod.json", false) != PSRETURN_OK)
return false;
if (!Script::ParseJSON(rq, modinfo.GetAsString(), json))
return false;
// Attempt to write the mod.json file so we'll take the fast path next time.
std::ofstream out_mod_json((modsPath / mod / L"mod.json").string8());
if (out_mod_json.good())
{
out_mod_json << modinfo.GetAsString();
out_mod_json.close();
}
else
{
// Print a warning - we'll keep trying, which could have adverse effects.
if (L10n::IsInitialised())
LOGWARNING(g_L10n.Translate("Could not write external mod.json for zipped mod '%s'. The mod should be reinstalled."), mod.string8());
else
LOGWARNING("Could not write external mod.json for zipped mod '%s'. The mod should be reinstalled.", mod.string8());
}
return true;
}
else
{
std::stringstream buffer;
buffer << modjson.rdbuf();
return Script::ParseJSON(rq, buffer.str(), json);
}
}
JS::Value GetAvailableMods(const ScriptInterface& scriptInterface)
{
PROFILE2("GetAvailableMods");
const Paths paths(g_CmdLineArgs);
// loop over all possible paths
OsPath modPath = paths.RData()/"mods";
@ -63,115 +114,96 @@ JS::Value Mod::GetAvailableMods(const ScriptInterface& scriptInterface)
PIVFS vfs = CreateVfs();
ScriptRequest rq(scriptInterface);
JS::RootedValue value(rq.cx, Script::CreateObject(rq));
for (DirectoryNames::iterator iter = modDirs.begin(); iter != modDirs.end(); ++iter)
{
vfs->Clear();
// Mount with lowest priority, we don't want to overwrite anything
if (vfs->Mount(L"", modPath / *iter / "", VFS_MOUNT_MUST_EXIST, VFS_MIN_PRIORITY) < 0)
continue;
CVFSFile modinfo;
if (modinfo.Load(vfs, L"mod.json", false) != PSRETURN_OK)
continue;
JS::RootedValue json(rq.cx);
if (!Script::ParseJSON(rq, modinfo.GetAsString(), &json))
if (!ParseModJSON(rq, vfs, modPath, *iter, &json))
continue;
// Valid mod, add it to our structure
JS_SetProperty(rq.cx, obj, utf8_from_wstring(iter->string()).c_str(), json);
// Valid mod data, add it to our structure
Script::SetProperty(rq, value, utf8_from_wstring(iter->string()).c_str(), json);
}
GetDirectoryEntries(modUserPath, NULL, &modDirsUser);
bool dev = InDevelopmentCopy();
for (DirectoryNames::iterator iter = modDirsUser.begin(); iter != modDirsUser.end(); ++iter)
{
// If we are in a dev copy we do not mount mods in the user mod folder that
// are already present in the mod folder, thus we skip those here.
if (dev && std::binary_search(modDirs.begin(), modDirs.end(), *iter))
continue;
vfs->Clear();
// Mount with lowest priority, we don't want to overwrite anything
if (vfs->Mount(L"", modUserPath / *iter / "", VFS_MOUNT_MUST_EXIST, VFS_MIN_PRIORITY) < 0)
continue;
CVFSFile modinfo;
if (modinfo.Load(vfs, L"mod.json", false) != PSRETURN_OK)
// Ignore mods in the user folder if we have already found them in modDirs.
if (std::binary_search(modDirs.begin(), modDirs.end(), *iter))
continue;
JS::RootedValue json(rq.cx);
if (!Script::ParseJSON(rq, modinfo.GetAsString(), &json))
if (!ParseModJSON(rq, vfs, modUserPath, *iter, &json))
continue;
// Valid mod, add it to our structure
JS_SetProperty(rq.cx, obj, utf8_from_wstring(iter->string()).c_str(), json);
// Valid mod data, add it to our structure
Script::SetProperty(rq, value, utf8_from_wstring(iter->string()).c_str(), json);
}
return JS::ObjectValue(*obj);
return value.get();
}
const std::vector<CStr>& Mod::GetEnabledMods()
const std::vector<CStr>& GetEnabledMods()
{
return g_modsLoaded;
return g_ModsLoaded;
}
const std::vector<CStr>& Mod::GetIncompatibleMods()
const std::vector<CStr>& GetIncompatibleMods()
{
return g_incompatibleMods;
return g_IncompatibleMods;
}
const std::vector<CStr>& Mod::GetFailedMods()
const std::vector<CStr>& GetFailedMods()
{
return g_failedMods;
return g_FailedMods;
}
const std::vector<CStr>& Mod::GetModsFromArguments(const CmdLineArgs& args, int flags)
const std::vector<CStr>& GetModsFromArguments(const CmdLineArgs& args, int flags)
{
const bool initMods = (flags & INIT_MODS) == INIT_MODS;
const bool addPublic = (flags & INIT_MODS_PUBLIC) == INIT_MODS_PUBLIC;
if (!initMods)
return g_modsLoaded;
return g_ModsLoaded;
g_modsLoaded = args.GetMultiple("mod");
g_ModsLoaded = args.GetMultiple("mod");
if (addPublic)
g_modsLoaded.insert(g_modsLoaded.begin(), "public");
g_ModsLoaded.insert(g_ModsLoaded.begin(), "public");
g_modsLoaded.insert(g_modsLoaded.begin(), "mod");
g_ModsLoaded.insert(g_ModsLoaded.begin(), "mod");
return g_modsLoaded;
return g_ModsLoaded;
}
void Mod::SetDefaultMods()
void SetDefaultMods()
{
g_modsLoaded.clear();
g_modsLoaded.insert(g_modsLoaded.begin(), "mod");
g_ModsLoaded.clear();
g_ModsLoaded.insert(g_ModsLoaded.begin(), "mod");
}
void Mod::ClearIncompatibleMods()
void ClearIncompatibleMods()
{
g_incompatibleMods.clear();
g_failedMods.clear();
g_IncompatibleMods.clear();
g_FailedMods.clear();
}
bool Mod::CheckAndEnableMods(const ScriptInterface& scriptInterface, const std::vector<CStr>& mods)
bool CheckAndEnableMods(const ScriptInterface& scriptInterface, const std::vector<CStr>& mods)
{
ScriptRequest rq(scriptInterface);
JS::RootedValue availableMods(rq.cx, GetAvailableMods(scriptInterface));
if (!AreModsCompatible(scriptInterface, mods, availableMods))
{
g_failedMods = mods;
g_FailedMods = mods;
return false;
}
g_modsLoaded = mods;
g_ModsLoaded = mods;
return true;
}
bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::vector<CStr>& mods, const JS::RootedValue& availableMods)
bool AreModsCompatible(const ScriptInterface& scriptInterface, const std::vector<CStr>& mods, const JS::RootedValue& availableMods)
{
ScriptRequest rq(scriptInterface);
std::unordered_map<CStr, std::vector<CStr>> modDependencies;
@ -186,12 +218,12 @@ bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::v
// Requested mod is not available, fail
if (!Script::HasProperty(rq, availableMods, mod.c_str()))
{
g_incompatibleMods.push_back(mod);
g_IncompatibleMods.push_back(mod);
continue;
}
if (!Script::GetProperty(rq, availableMods, mod.c_str(), &modData))
{
g_incompatibleMods.push_back(mod);
g_IncompatibleMods.push_back(mod);
continue;
}
@ -236,13 +268,13 @@ bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::v
const std::unordered_map<CStr, CStr>::iterator it = modNameVersions.find(modToCheck);
if (it == modNameVersions.end())
{
g_incompatibleMods.push_back(mod);
g_IncompatibleMods.push_back(mod);
continue;
}
// 0.0.25(0ad) , <=, 0.0.24(required version)
if (!CompareVersionStrings(it->second, op, versionToCheck))
{
g_incompatibleMods.push_back(mod);
g_IncompatibleMods.push_back(mod);
continue;
}
break;
@ -251,10 +283,10 @@ bool Mod::AreModsCompatible(const ScriptInterface& scriptInterface, const std::v
}
return g_incompatibleMods.empty();
return g_IncompatibleMods.empty();
}
bool Mod::CompareVersionStrings(const CStr& version, const CStr& op, const CStr& required)
bool CompareVersionStrings(const CStr& version, const CStr& op, const CStr& required)
{
std::vector<CStr> versionSplit;
std::vector<CStr> requiredSplit;
@ -288,7 +320,7 @@ bool Mod::CompareVersionStrings(const CStr& version, const CStr& op, const CStr&
}
void Mod::CacheEnabledModVersions(const shared_ptr<ScriptContext>& scriptContext)
void CacheEnabledModVersions(const shared_ptr<ScriptContext>& scriptContext)
{
ScriptInterface scriptInterface("Engine", "CacheEnabledModVersions", scriptContext);
ScriptRequest rq(scriptInterface);
@ -297,7 +329,7 @@ void Mod::CacheEnabledModVersions(const shared_ptr<ScriptContext>& scriptContext
g_LoadedModVersions.clear();
for (const CStr& mod : g_modsLoaded)
for (const CStr& mod : g_ModsLoaded)
{
// Ignore mod mod as it is irrelevant for compatibility checks
if (mod == "mod")
@ -312,7 +344,7 @@ void Mod::CacheEnabledModVersions(const shared_ptr<ScriptContext>& scriptContext
}
}
JS::Value Mod::GetLoadedModsWithVersions(const ScriptInterface& scriptInterface)
JS::Value GetLoadedModsWithVersions(const ScriptInterface& scriptInterface)
{
ScriptRequest rq(scriptInterface);
JS::RootedValue returnValue(rq.cx);
@ -320,11 +352,11 @@ JS::Value Mod::GetLoadedModsWithVersions(const ScriptInterface& scriptInterface)
return returnValue;
}
JS::Value Mod::GetEngineInfo(const ScriptInterface& scriptInterface)
JS::Value GetEngineInfo(const ScriptInterface& scriptInterface)
{
ScriptRequest rq(scriptInterface);
JS::RootedValue mods(rq.cx, Mod::GetLoadedModsWithVersions(scriptInterface));
JS::RootedValue mods(rq.cx, GetLoadedModsWithVersions(scriptInterface));
JS::RootedValue metainfo(rq.cx);
Script::CreateObject(
@ -337,3 +369,4 @@ JS::Value Mod::GetEngineInfo(const ScriptInterface& scriptInterface)
return metainfo;
}
}

7
source/ps/Mod.h
View File

@ -22,11 +22,14 @@
#include "ps/GameSetup/CmdLineArgs.h"
#include "scriptinterface/ScriptForward.h"
extern std::vector<CStr> g_modsLoaded;
extern CmdLineArgs g_args;
#include <vector>
extern CmdLineArgs g_CmdLineArgs;
namespace Mod
{
extern std::vector<CStr> g_ModsLoaded;
JS::Value GetAvailableMods(const ScriptInterface& scriptInterface);
const std::vector<CStr>& GetEnabledMods();
const std::vector<CStr>& GetIncompatibleMods();

11
source/ps/ModInstaller.cpp
View File

@ -25,9 +25,7 @@
#include "scriptinterface/ScriptInterface.h"
#include "scriptinterface/JSON.h"
#ifdef OS_WIN
#include <fstream>
#endif
CModInstaller::CModInstaller(const OsPath& modsdir, const OsPath& tempdir) :
m_ModsDir(modsdir), m_TempDir(tempdir / "_modscache"), m_CacheDir("cache/")
@ -88,23 +86,20 @@ CModInstaller::ModInstallationResult CModInstaller::Install(
// Create a directory with the following structure:
// mod-name/
// mod-name.zip
// mod.json
CreateDirectories(modDir, 0700);
if (wrename(modTemp, modPath) != 0)
return FAIL_ON_MOD_MOVE;
DeleteDirectory(modTemp.Parent());
#ifdef OS_WIN
// On Windows, write the contents of mod.json to a separate file next to the archive:
// mod-name/
// mod-name.zip
// mod.json
std::ofstream mod_json((modDir / "mod.json").string8());
if (mod_json.good())
{
mod_json << modinfo.GetAsString();
mod_json.close();
}
#endif // OS_WIN
else
return FAIL_ON_JSON_WRITE;
m_InstalledMods.emplace_back(modName);

3
source/ps/ModInstaller.h
View File

@ -38,7 +38,8 @@ public:
FAIL_ON_MOD_LOAD,
FAIL_ON_PARSE_JSON,
FAIL_ON_EXTRACT_NAME,
FAIL_ON_MOD_MOVE
FAIL_ON_MOD_MOVE,
FAIL_ON_JSON_WRITE
};
/**

4
source/ps/ModIo.cpp
View File

@ -309,7 +309,7 @@ void ModIo::StartDownloadMod(u32 idx)
if (idx >= m_ModData.size())
return;
const Paths paths(g_args);
const Paths paths(g_CmdLineArgs);
const OsPath modUserPath = paths.UserData()/"mods";
const OsPath modPath = modUserPath/m_ModData[idx].properties["name_id"];
if (!DirectoryExists(modPath) && INFO::OK != CreateDirectories(modPath, 0700, false))
@ -482,7 +482,7 @@ bool ModIo::AdvanceRequest(const ScriptInterface& scriptInterface)
m_DownloadProgressData.status = DownloadProgressStatus::SUCCESS;
{
Paths paths(g_args);
Paths paths(g_CmdLineArgs);
CModInstaller installer(paths.UserData() / "mods", paths.Cache());
installer.Install(m_DownloadFilePath, g_ScriptContext, false);
}

2
source/ps/VisualReplay.cpp
View File

@ -44,7 +44,7 @@ const u8 minimumReplayDuration = 3;
OsPath VisualReplay::GetDirectoryPath()
{
return Paths(g_args).UserData() / "replays" / engine_version;
return Paths(g_CmdLineArgs).UserData() / "replays" / engine_version;
}
OsPath VisualReplay::GetCacheFilePath()