abian/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

XMB Improvements, parse JS into XMB, make strings more efficient.

Browse Source 
XMB format is bumped to 4, invalidating all cached files. The
differences are:
 - element/attribute names are stored after the elements themselves, and
not before. This allows writing XMB data in one pass instead of two.
 - names themselves becomes offsets (instead of arbitrary integers),
making getting the string from the int name much more efficient.

XMBFile is renamed to XMBData to clarify that it does not, in fact,
refer to a file on disk.

XMBData::GetElementString is also changed to return a const char*, thus
not creating an std::string. A string_view version is added where
convenient.

The XML->XMB and JS->XMB conversion functions and the corresponding
storage are moved to `ps/XMB`, since that format doesn't particularly
relate to XML. CXeromyces becomes lighter and more focused as a result.
The XML->XMB conversion also benefits from the above streamlining.

Note that in a few cases, string_view gets printed to CLogger via
data(), which is generally not legal, but we know that the strings are
null-terminated here. Our libfmt (version 4) doesn't support
string_view, that would be v5.

Differential Revision: https://code.wildfiregames.com/D3909
This was SVN commit r25375.
This commit is contained in:
wraitii 2021-05-04 13:02:34 +00:00
parent a1010b83d3
commit cdd75deafb
20 changed files with 764 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build/premake/premake5.lua
View File

@ -731,6 +731,7 @@ function setup_all_libs ()
"ps/scripting",
"network/scripting",
"ps/GameSetup",
"ps/XMB",
"ps/XML",
"soundmanager",
"soundmanager/data",

4
source/graphics/MapReader.cpp
View File

@ -491,7 +491,7 @@ void CXMLReader::Init(const VfsPath& xml_filename)
#undef EL
XMBElement root = xmb_file.GetRoot();
ENSURE(xmb_file.GetElementString(root.GetNodeName()) == "Scenario");
ENSURE(xmb_file.GetElementStringView(root.GetNodeName()) == "Scenario");
nodes = root.GetChildNodes();
// find out total number of entities+nonentities
@ -517,7 +517,7 @@ void CXMLReader::Init(const VfsPath& xml_filename)
CStr CXMLReader::ReadScriptSettings()
{
XMBElement root = xmb_file.GetRoot();
ENSURE(xmb_file.GetElementString(root.GetNodeName()) == "Scenario");
ENSURE(xmb_file.GetElementStringView(root.GetNodeName()) == "Scenario");
nodes = root.GetChildNodes();
XMBElement settings = nodes.GetFirstNamedItem(xmb_file.GetElementID("ScriptSettings"));

4
source/graphics/ObjectBase.cpp
View File

@ -195,7 +195,7 @@ bool CObjectBase::LoadVariant(const CXeromyces& XeroFile, const XMBElement& vari
if (variant.GetNodeName() != el_variant)
{
LOGERROR("Invalid variant format (unrecognised root element '%s')", XeroFile.GetElementString(variant.GetNodeName()).c_str());
LOGERROR("Invalid variant format (unrecognised root element '%s')", XeroFile.GetElementString(variant.GetNodeName()));
return false;
}
@ -821,7 +821,7 @@ bool CActorDef::Load(const VfsPath& pathname)
if (root.GetNodeName() != el_actor && root.GetNodeName() != el_qualitylevels)
{
LOGERROR("Invalid actor format (actor '%s', unrecognised root element '%s')",
pathname.string8().c_str(), XeroFile.GetElementString(root.GetNodeName()).c_str());
pathname.string8().c_str(), XeroFile.GetElementString(root.GetNodeName()));
return false;
}

3
source/graphics/TerrainProperties.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2019 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -28,7 +28,6 @@
#include "maths/MathUtil.h"
#include "ps/CLogger.h"
#include "ps/Filesystem.h"
#include "ps/XML/XeroXMB.h"
#include "ps/XML/Xeromyces.h"
CTerrainProperties::CTerrainProperties(CTerrainPropertiesPtr parent):

4
source/graphics/TerrainTextureEntry.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -71,7 +71,7 @@ CTerrainTextureEntry::CTerrainTextureEntry(CTerrainPropertiesPtr properties, con
if (root.GetNodeName() != el_terrain)
{
LOGERROR("Invalid terrain format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName()).c_str());
LOGERROR("Invalid terrain format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName()));
return;
}

2
source/graphics/TextureConverter.cpp
View File

@ -227,7 +227,7 @@ CTextureConverter::SettingsFile* CTextureConverter::LoadSettings(const VfsPath&
}
else
{
LOGERROR("Invalid attribute name <file %s='...'>", XeroFile.GetAttributeString(attr.Name).c_str());
LOGERROR("Invalid attribute name <file %s='...'>", XeroFile.GetAttributeString(attr.Name));
}
}

26
source/gui/CGUI.cpp
View File

@ -532,7 +532,7 @@ void CGUI::LoadXmlFile(const VfsPath& Filename, std::unordered_set<VfsPath>& Pat
return;
XMBElement node = XeroFile.GetRoot();
CStr root_name(XeroFile.GetElementString(node.GetNodeName()));
std::string_view root_name(XeroFile.GetElementStringView(node.GetNodeName()));
if (root_name == "objects")
Xeromyces_ReadRootObjects(node, &XeroFile, Paths);
@ -543,7 +543,7 @@ void CGUI::LoadXmlFile(const VfsPath& Filename, std::unordered_set<VfsPath>& Pat
else if (root_name == "setup")
Xeromyces_ReadRootSetup(node, &XeroFile);
else
LOGERROR("CGUI::LoadXmlFile encountered an unknown XML root node type: %s", root_name.c_str());
LOGERROR("CGUI::LoadXmlFile encountered an unknown XML root node type: %s", root_name.data());
}
void CGUI::LoadedXmlFiles()
@ -595,7 +595,7 @@ void CGUI::Xeromyces_ReadRootSetup(XMBElement Element, CXeromyces* pFile)
{
for (XMBElement child : Element.GetChildNodes())
{
CStr name(pFile->GetElementString(child.GetNodeName()));
std::string_view name(pFile->GetElementStringView(child.GetNodeName()));
if (name == "scrollbar")
Xeromyces_ReadScrollBarStyle(child, pFile);
@ -989,7 +989,7 @@ void CGUI::Xeromyces_ReadSprite(XMBElement Element, CXeromyces* pFile)
for (XMBElement child : Element.GetChildNodes())
{
CStr ElementName(pFile->GetElementString(child.GetNodeName()));
std::string_view ElementName(pFile->GetElementStringView(child.GetNodeName()));
if (ElementName == "image")
Xeromyces_ReadImage(child, pFile, *Sprite);
@ -1026,7 +1026,7 @@ void CGUI::Xeromyces_ReadImage(XMBElement Element, CXeromyces* pFile, CGUISprite
for (XMBAttribute attr : Element.GetAttributes())
{
CStr attr_name(pFile->GetAttributeString(attr.Name));
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
CStrW attr_value(attr.Value.FromUTF8());
if (attr_name == "texture")
@ -1109,7 +1109,7 @@ void CGUI::Xeromyces_ReadImage(XMBElement Element, CXeromyces* pFile, CGUISprite
// Look for effects
for (XMBElement child : Element.GetChildNodes())
{
CStr ElementName(pFile->GetElementString(child.GetNodeName()));
std::string_view ElementName(pFile->GetElementStringView(child.GetNodeName()));
if (ElementName == "effect")
{
if (Image->m_Effects)
@ -1131,7 +1131,7 @@ void CGUI::Xeromyces_ReadEffects(XMBElement Element, CXeromyces* pFile, SGUIImag
{
for (XMBAttribute attr : Element.GetAttributes())
{
CStr attr_name(pFile->GetAttributeString(attr.Name));
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
if (attr_name == "add_color")
{
@ -1152,14 +1152,14 @@ void CGUI::Xeromyces_ReadStyle(XMBElement Element, CXeromyces* pFile)
for (XMBAttribute attr : Element.GetAttributes())
{
CStr attr_name(pFile->GetAttributeString(attr.Name));
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
// The "name" setting is actually the name of the style
// and not a new default
if (attr_name == "name")
name = attr.Value;
else
style.m_SettingsDefaults.emplace(attr_name, attr.Value.FromUTF8());
style.m_SettingsDefaults.emplace(std::string(attr_name), attr.Value.FromUTF8());
}
m_Styles.erase(name);
@ -1179,7 +1179,7 @@ void CGUI::Xeromyces_ReadScrollBarStyle(XMBElement Element, CXeromyces* pFile)
for (XMBAttribute attr : Element.GetAttributes())
{
CStr attr_name = pFile->GetAttributeString(attr.Name);
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
CStr attr_value(attr.Value);
if (attr_value == "null")
@ -1256,7 +1256,7 @@ void CGUI::Xeromyces_ReadIcon(XMBElement Element, CXeromyces* pFile)
for (XMBAttribute attr : Element.GetAttributes())
{
CStr attr_name(pFile->GetAttributeString(attr.Name));
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
CStr attr_value(attr.Value);
if (attr_value == "null")
@ -1288,13 +1288,13 @@ void CGUI::Xeromyces_ReadTooltip(XMBElement Element, CXeromyces* pFile)
for (XMBAttribute attr : Element.GetAttributes())
{
CStr attr_name(pFile->GetAttributeString(attr.Name));
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
CStr attr_value(attr.Value);
if (attr_name == "name")
object->SetName("__tooltip_" + attr_value);
else
object->SetSettingFromString(attr_name, attr_value.FromUTF8(), true);
object->SetSettingFromString(std::string(attr_name), attr_value.FromUTF8(), true);
}
if (!AddObject(*m_BaseObject, *object))

8
source/gui/ObjectTypes/COList.cpp
View File

@ -203,13 +203,13 @@ bool COList::HandleAdditionalChildren(const XMBElement& child, CXeromyces* pFile
for (XMBAttribute attr : child.GetAttributes())
{
CStr attr_name(pFile->GetAttributeString(attr.Name));
std::string_view attr_name(pFile->GetAttributeStringView(attr.Name));
CStr attr_value(attr.Value);
if (attr_name == "color")
{
if (!CGUI::ParseString<CGUIColor>(&m_pGUI, attr_value.FromUTF8(), column.m_TextColor))
LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.c_str(), attr_value.c_str());
LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.data(), attr_value.c_str());
}
else if (attr_name == "id")
{
@ -219,7 +219,7 @@ bool COList::HandleAdditionalChildren(const XMBElement& child, CXeromyces* pFile
{
bool hidden = false;
if (!CGUI::ParseString<bool>(&m_pGUI, attr_value.FromUTF8(), hidden))
LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.c_str(), attr_value.c_str());
LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.data(), attr_value.c_str());
else
column.m_Hidden = hidden;
}
@ -227,7 +227,7 @@ bool COList::HandleAdditionalChildren(const XMBElement& child, CXeromyces* pFile
{
float width;
if (!CGUI::ParseString<float>(&m_pGUI, attr_value.FromUTF8(), width))
LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.c_str(), attr_value.c_str());
LOGERROR("GUI: Error parsing '%s' (\"%s\")", attr_name.data(), attr_value.c_str());
else
{
// Check if it's a relative value, and save as decimal if so.

112
source/ps/XML/XeroXMB.cpp → source/ps/XMB/XMBData.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2015 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -17,16 +17,9 @@
#include "precompiled.h"
#include "Xeromyces.h"
#include "lib/byte_order.h" // FOURCC_LE
// external linkage (also used by Xeromyces.cpp)
const char* HeaderMagicStr = "XMB0";
const char* UnfinishedHeaderMagicStr = "XMBu";
// Arbitrary version number - change this if we update the code and
// need to invalidate old users' caches
const u32 XMBVersion = 3;
#include "ps/XMB/XMBStorage.h"
#include "ps/XML/Xeromyces.h"
template<typename T>
static inline T read(const void* ptr)
@ -36,85 +29,42 @@ static inline T read(const void* ptr)
return ret;
}
bool XMBFile::Initialise(const char* FileData)
bool XMBData::Initialise(const XMBStorage& doc)
{
m_Pointer = FileData;
const char* start = reinterpret_cast<const char*>(doc.m_Buffer.get());
m_Pointer = start;
char Header[5] = { 0 };
strncpy_s(Header, 5, m_Pointer, 4);
m_Pointer += 4;
// (c.f. @return documentation of this function)
if(!strcmp(Header, UnfinishedHeaderMagicStr))
if (strcmp(Header, XMBStorage::UnfinishedHeaderMagicStr) == 0)
return false;
ENSURE(!strcmp(Header, HeaderMagicStr) && "Invalid XMB header!");
ENSURE(strcmp(Header, XMBStorage::HeaderMagicStr) == 0 && "Invalid XMB header!");
u32 Version = read<u32>(m_Pointer);
m_Pointer += 4;
if (Version != XMBVersion)
if (Version != XMBStorage::XMBVersion)
return false;
int i;
// FIXME Check that m_Pointer doesn't end up past the end of the buffer
// (it shouldn't be all that dangerous since we're only doing read-only
// access, but it might crash on an invalid file, reading a couple of
// billion random element names from RAM)
#ifdef XERO_USEMAP
// Build a std::map of all the names->ids
u32 ElementNameCount = read<u32>(m_Pointer); m_Pointer += 4;
for (i = 0; i < ElementNameCount; ++i)
m_ElementNames[ReadZStr8()] = i;
u32 AttributeNameCount = read<u32>(m_Pointer); m_Pointer += 4;
for (i = 0; i < AttributeNameCount; ++i)
m_AttributeNames[ReadZStr8()] = i;
#else
// Ignore all the names for now, and skip over them
// (remembering the position of the first)
m_ElementPointer = start + read<u32>(m_Pointer); m_Pointer += 4;
m_ElementNameCount = read<int>(m_Pointer); m_Pointer += 4;
m_ElementPointer = m_Pointer;
for (i = 0; i < m_ElementNameCount; ++i)
m_Pointer += 4 + read<int>(m_Pointer); // skip over the string
m_AttributePointer = start + read<u32>(m_Pointer); m_Pointer += 4;
m_AttributeNameCount = read<int>(m_Pointer); m_Pointer += 4;
m_AttributePointer = m_Pointer;
for (i = 0; i < m_AttributeNameCount; ++i)
m_Pointer += 4 + read<int>(m_Pointer); // skip over the string
#endif
// At this point m_Pointer points to the element start, as expected.
return true; // success
}
std::string XMBFile::ReadZStr8()
{
int Length = read<int>(m_Pointer);
m_Pointer += 4;
std::string String (m_Pointer); // reads up until the first NULL
m_Pointer += Length;
return String;
}
XMBElement XMBFile::GetRoot() const
XMBElement XMBData::GetRoot() const
{
return XMBElement(m_Pointer);
}
#ifdef XERO_USEMAP
int XMBFile::GetElementID(const char* Name) const
{
return m_ElementNames[Name];
}
int XMBFile::GetAttributeID(const char* Name) const
{
return m_AttributeNames[Name];
}
#else // #ifdef XERO_USEMAP
int XMBFile::GetElementID(const char* Name) const
int XMBData::GetElementID(const char* Name) const
{
const char* Pos = m_ElementPointer;
@ -126,7 +76,7 @@ int XMBFile::GetElementID(const char* Name) const
// See if this could be the right string, checking its
// length and then its contents
if (read<int>(Pos) == len && strncasecmp(Pos+4, Name, len) == 0)
return i;
return static_cast<int>(Pos - m_ElementPointer);
// If not, jump to the next string
Pos += 4 + read<int>(Pos);
}
@ -134,7 +84,7 @@ int XMBFile::GetElementID(const char* Name) const
return -1;
}
int XMBFile::GetAttributeID(const char* Name) const
int XMBData::GetAttributeID(const char* Name) const
{
const char* Pos = m_AttributePointer;
@ -146,35 +96,33 @@ int XMBFile::GetAttributeID(const char* Name) const
// See if this could be the right string, checking its
// length and then its contents
if (read<int>(Pos) == len && strncasecmp(Pos+4, Name, len) == 0)
return i;
return static_cast<int>(Pos - m_AttributePointer);
// If not, jump to the next string
Pos += 4 + read<int>(Pos);
}
// Failed
return -1;
}
#endif // #ifdef XERO_USEMAP / #else
// Relatively inefficient, so only use when
// laziness overcomes the need for speed
std::string XMBFile::GetElementString(const int ID) const
const char* XMBData::GetElementString(const int ID) const
{
const char* Pos = m_ElementPointer;
for (int i = 0; i < ID; ++i)
Pos += 4 + read<int>(Pos);
return std::string(Pos+4);
return reinterpret_cast<const char*>(m_ElementPointer + ID + 4);
}
std::string XMBFile::GetAttributeString(const int ID) const
const char* XMBData::GetAttributeString(const int ID) const
{
const char* Pos = m_AttributePointer;
for (int i = 0; i < ID; ++i)
Pos += 4 + read<int>(Pos);
return std::string(Pos+4);
return reinterpret_cast<const char*>(m_AttributePointer + ID + 4);
}
std::string_view XMBData::GetElementStringView(const int ID) const
{
return std::string_view(reinterpret_cast<const char*>(m_ElementPointer + ID + 4), read<int>(m_ElementPointer + ID) - 1);
}
std::string_view XMBData::GetAttributeStringView(const int ID) const
{
return std::string_view(reinterpret_cast<const char*>(m_AttributePointer + ID + 4), read<int>(m_AttributePointer + ID) - 1);
}
int XMBElement::GetNodeName() const
{

79
source/ps/XML/XeroXMB.h → source/ps/XMB/XMBData.h
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -20,30 +20,27 @@
Brief outline:
XMB is a binary representation of XML, with some limitations
XMB originated as a binary representation of XML, with some limitations
but much more efficiency (particularly for loading simple data
classes that don't need much initialisation).
Main limitations:
* Can't correctly handle mixed text/elements inside elements -
"<div> <b> Text </b> </div>" and "<div> Te<b/>xt </div>" are
considered identical.
* Tries to avoid using strings - you usually have to load the
numeric IDs and use them instead.
Theoretical file structure:
XMB_File {
char Header[4]; // because everyone has one; currently "XMB0"
u32 Version;
int ElementNameCount;
ZStr8 ElementNames[];
int OffsetFromStartToElementNames;
int ElementNameCount;
int OffsetFromStartToAttributeNames;
int AttributeNameCount;
XMB_Node Root;
ZStr8 ElementNames[];
ZStr8 AttributeNames[];
XMB_Node Root;
}
XMB_Node {
@ -83,71 +80,57 @@ XMB_Text {
#ifndef INCLUDED_XEROXMB
#define INCLUDED_XEROXMB
// Define to use a std::map for name lookups rather than a linear search.
// (The map is usually slower.)
//#define XERO_USEMAP
#include <string>
#ifdef XERO_USEMAP
# include <map>
#endif
#include "ps/CStr.h"
// File headers, to make sure it doesn't try loading anything other than an XMB
extern const char* HeaderMagicStr;
extern const char* UnfinishedHeaderMagicStr;
extern const u32 XMBVersion;
#include <string>
#include <string_view>
class CXeromyces;
class XMBStorage;
class XMBElement;
class XMBElementList;
class XMBAttributeList;
class XMBFile
class XMBData
{
public:
XMBFile() : m_Pointer(NULL) {}
XMBData() : m_Pointer(nullptr) {}
// Initialise from the contents of an XMB file.
// FileData must remain allocated and unchanged while
// the XMBFile is being used.
// @return indication of success; main cause for failure is attempting to
// load a partially valid XMB file (e.g. if the game was interrupted
// while writing it), which we detect by checking the magic string.
// It also fails when trying to load an XMB file with a different version.
bool Initialise(const char* FileData);
/*
* Initialise from the contents of an XMBStorage.
* @param doc must remain allocated and unchanged while
* the XMBData is being used.
* @return indication of success; main cause for failure is attempting to
* load a partially valid XMB file (e.g. if the game was interrupted
* while writing it), which we detect by checking the magic string.
* It also fails when trying to load an XMB file with a different version.
*/
bool Initialise(const XMBStorage& doc);
// Returns the root element
XMBElement GetRoot() const;
// Returns internal ID for a given element/attribute string.
int GetElementID(const char* Name) const;
int GetAttributeID(const char* Name) const;
// For lazy people (e.g. me) when speed isn't vital:
// Returns element/attribute string for a given internal ID.
const char* GetElementString(const int ID) const;
const char* GetAttributeString(const int ID) const;
// Returns element/attribute string for a given internal ID
std::string GetElementString(const int ID) const;
std::string GetAttributeString(const int ID) const;
std::string_view GetElementStringView(const int ID) const;
std::string_view GetAttributeStringView(const int ID) const;
private:
const char* m_Pointer;
#ifdef XERO_USEMAP
std::map<std::string, int> m_ElementNames;
std::map<std::string, int> m_AttributeNames;
#else
int m_ElementNameCount;
int m_AttributeNameCount;
const char* m_ElementPointer;
const char* m_AttributePointer;
#endif
std::string ReadZStr8();
};
class XMBElement

469
source/ps/XMB/XMBStorage.cpp Normal file
View File

@ -0,0 +1,469 @@
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
*/
#include "precompiled.h"
#include "XMBStorage.h"
#include "lib/file/io/write_buffer.h"
#include "scriptinterface/ScriptExtraHeaders.h"
#include "scriptinterface/ScriptInterface.h"
#include <libxml/parser.h>
#include <unordered_map>
const char* XMBStorage::HeaderMagicStr = "XMB0";
const char* XMBStorage::UnfinishedHeaderMagicStr = "XMBu";
// Arbitrary version number - change this if we update the code and
// need to invalidate old users' caches
const u32 XMBStorage::XMBVersion = 4;
namespace
{
class XMBStorageWriter
{
public:
template<typename ...Args>
bool Load(WriteBuffer& writeBuffer, Args&&... args);
int GetElementName(const std::string& name) { return GetName(m_ElementSize, m_ElementIDs, name); }
int GetAttributeName(const std::string& name) { return GetName(m_AttributeSize, m_AttributeIDs, name); }
protected:
int GetName(int& totalSize, std::unordered_map<std::string, int>& names, const std::string& name)
{
int nameIdx = totalSize;
auto [iterator, inserted] = names.try_emplace(name, nameIdx);
if (inserted)
totalSize += name.size() + 5; // Add 1 for the null terminator & 4 for the size int.
return iterator->second;
}
void OutputNames(WriteBuffer& writeBuffer, const std::unordered_map<std::string, int>& names) const;
template<typename ...Args>
bool OutputElements(WriteBuffer&, Args...)
{
static_assert(sizeof...(Args) != sizeof...(Args), "OutputElements must be specialized.");
}
int m_ElementSize = 0;
int m_AttributeSize = 0;
std::unordered_map<std::string, int> m_ElementIDs;
std::unordered_map<std::string, int> m_AttributeIDs;
};
// Output text, prefixed by length in bytes (including null-terminator)
void WriteStringAndLineNumber(WriteBuffer& writeBuffer, const std::string& text, int lineNumber)
{
if (text.empty())
{
// No text; don't write much
writeBuffer.Append("\0\0\0\0", 4);
}
else
{
// Write length and line number and null-terminated text
u32 nodeLen = u32(4 + text.length() + 1);
writeBuffer.Append(&nodeLen, 4);
writeBuffer.Append(&lineNumber, 4);
writeBuffer.Append((void*)text.c_str(), nodeLen-4);
}
}
template<typename ...Args>
bool XMBStorageWriter::Load(WriteBuffer& writeBuffer, Args&&... args)
{
// Header
writeBuffer.Append(XMBStorage::UnfinishedHeaderMagicStr, 4);
// Version
writeBuffer.Append(&XMBStorage::XMBVersion, 4);
// Filled in below.
size_t elementPtr = writeBuffer.Size();
writeBuffer.Append("????????", 8);
// Likewise with attributes.
size_t attributePtr = writeBuffer.Size();
writeBuffer.Append("????????", 8);
if (!OutputElements<Args&&...>(writeBuffer, std::forward<Args>(args)...))
return false;
u32 data = writeBuffer.Size();
writeBuffer.Overwrite(&data, 4, elementPtr);
data = m_ElementIDs.size();
writeBuffer.Overwrite(&data, 4, elementPtr + 4);
OutputNames(writeBuffer, m_ElementIDs);
data = writeBuffer.Size();
writeBuffer.Overwrite(&data, 4, attributePtr);
data = m_AttributeIDs.size();
writeBuffer.Overwrite(&data, 4, attributePtr + 4);
OutputNames(writeBuffer, m_AttributeIDs);
// File is now valid, so insert correct magic string.
writeBuffer.Overwrite(XMBStorage::HeaderMagicStr, 4, 0);
return true;
}
void XMBStorageWriter::OutputNames(WriteBuffer& writeBuffer, const std::unordered_map<std::string, int>& names) const
{
std::vector<std::pair<std::string, int>> orderedElements;
for (const std::pair<std::string, int>& n : names)
orderedElements.emplace_back(n);
std::sort(orderedElements.begin(), orderedElements.end(), [](const auto& a, const auto&b) { return a.second < b.second; });
for (const std::pair<std::string, int>& n : orderedElements)
{
u32 textLen = (u32)n.first.length() + 1;
writeBuffer.Append(&textLen, 4);
writeBuffer.Append((void*)n.first.c_str(), textLen);
}
}
class JSNodeData
{
public:
JSNodeData(const ScriptInterface& s) : scriptInterface(s), rq(s) {}
bool Setup(XMBStorageWriter& xmb, JS::HandleValue value);
bool Output(WriteBuffer& writeBuffer, JS::HandleValue value) const;
std::vector<std::pair<u32, std::string>> m_Attributes;
std::vector<std::pair<u32, JS::Heap<JS::Value>>> m_Children;
const ScriptInterface& scriptInterface;
const ScriptRequest rq;
};
template<>
bool XMBStorageWriter::OutputElements<JSNodeData&, const u32&, JS::HandleValue&&>(WriteBuffer& writeBuffer, JSNodeData& data, const u32& nodeName, JS::HandleValue&& value)
{
// Set up variables.
if (!data.Setup(*this, value))
return false;
size_t posLength = writeBuffer.Size();
// Filled in later with the length of the element
writeBuffer.Append("????", 4);
writeBuffer.Append(&nodeName, 4);
u32 attrCount = static_cast<u32>(data.m_Attributes.size());
writeBuffer.Append(&attrCount, 4);
u32 childCount = data.m_Children.size();
writeBuffer.Append(&childCount, 4);
// Filled in later with the offset to the list of child elements
size_t posChildrenOffset = writeBuffer.Size();
writeBuffer.Append("????", 4);
data.Output(writeBuffer, value);
// Output attributes
for (const std::pair<u32, std::string> attr : data.m_Attributes)
{
writeBuffer.Append(&attr.first, 4);
u32 attrLen = u32(attr.second.size())+1;
writeBuffer.Append(&attrLen, 4);
writeBuffer.Append((void*)attr.second.c_str(), attrLen);
}
// Go back and fill in the child-element offset
u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4));
writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset);
// Output all child elements, making a copy since data will be overwritten.
std::vector<std::pair<u32, JS::Heap<JS::Value>>> children = data.m_Children;
for (const std::pair<u32, JS::Heap<JS::Value>>& child : children)
{
JS::RootedValue val(data.rq.cx, child.second);
if (!OutputElements<JSNodeData&, const u32&, JS::HandleValue&&>(writeBuffer, data, child.first, val))
return false;
}
// Go back and fill in the length
u32 length = (u32)(writeBuffer.Size() - posLength);
writeBuffer.Overwrite(&length, 4, posLength);
return true;
}
bool JSNodeData::Setup(XMBStorageWriter& xmb, JS::HandleValue value)
{
m_Attributes.clear();
m_Children.clear();
JSType valType = JS_TypeOfValue(rq.cx, value);
if (valType != JSTYPE_OBJECT)
return true;
std::vector<std::string> props;
if (!scriptInterface.EnumeratePropertyNames(value, true, props))
{
LOGERROR("Failed to enumerate component properties.");
return false;
}
for (const std::string& prop : props)
{
// Special 'value' key.
if (prop == "_string")
continue;
bool attrib = !prop.empty() && prop.front() == '@';
std::string_view name = prop;
if (!attrib && !prop.empty() && prop.back() == '@')
{
size_t idx = prop.substr(0, prop.size()-1).find_last_of('@');
if (idx == std::string::npos)
{
LOGERROR("Object key name cannot end with an '@' unless it is an index specifier.");
return false;
}
name = std::string_view(prop.c_str(), idx);
}
else if (attrib)
name = std::string_view(prop.c_str()+1, prop.length()-1);
JS::RootedValue child(rq.cx);
if (!scriptInterface.GetProperty(value, prop.c_str(), &child))
return false;
if (attrib)
{
std::string attrVal;
if (!ScriptInterface::FromJSVal(rq, child, attrVal))
{
LOGERROR("Attributes must be convertible to string");
return false;
}
m_Attributes.emplace_back(xmb.GetAttributeName(std::string(name)), attrVal);
continue;
}
bool isArray = false;
if (!JS::IsArrayObject(rq.cx, child, &isArray))
return false;
if (!isArray)
{
m_Children.emplace_back(xmb.GetElementName(std::string(name)), child);
continue;
}
// Parse each array object as a child.
JS::RootedObject obj(rq.cx);
JS_ValueToObject(rq.cx, child, &obj);
u32 length;
JS::GetArrayLength(rq.cx, obj, &length);
for (size_t i = 0; i < length; ++i)
{
JS::RootedValue arrayChild(rq.cx);
scriptInterface.GetPropertyInt(child, i, &arrayChild);
m_Children.emplace_back(xmb.GetElementName(std::string(name)), arrayChild);
}
}
return true;
}
bool JSNodeData::Output(WriteBuffer& writeBuffer, JS::HandleValue value) const
{
switch (JS_TypeOfValue(rq.cx, value))
{
case JSTYPE_UNDEFINED:
case JSTYPE_NULL:
{
writeBuffer.Append("\0\0\0\0", 4);
break;
}
case JSTYPE_OBJECT:
{
if (!scriptInterface.HasProperty(value, "_string"))
{
writeBuffer.Append("\0\0\0\0", 4);
break;
}
JS::RootedValue actualValue(rq.cx);
if (!scriptInterface.GetProperty(value, "_string", &actualValue))
return false;
std::string strVal;
if (!ScriptInterface::FromJSVal(rq, actualValue, strVal))
{
LOGERROR("'_string' value must be convertible to string");
return false;
}
WriteStringAndLineNumber(writeBuffer, strVal, 0);
break;
}
case JSTYPE_STRING:
case JSTYPE_NUMBER:
{
std::string strVal;
if (!ScriptInterface::FromJSVal(rq, value, strVal))
return false;
WriteStringAndLineNumber(writeBuffer, strVal, 0);
break;
}
default:
{
LOGERROR("Unsupported JS construct when parsing ParamNode");
return false;
}
}
return true;
}
template<>
bool XMBStorageWriter::OutputElements<xmlNodePtr&&>(WriteBuffer& writeBuffer, xmlNodePtr&& node)
{
// Filled in later with the length of the element
size_t posLength = writeBuffer.Size();
writeBuffer.Append("????", 4);
u32 name = GetElementName((const char*)node->name);
writeBuffer.Append(&name, 4);
u32 attrCount = 0;
for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
++attrCount;
writeBuffer.Append(&attrCount, 4);
u32 childCount = 0;
for (xmlNodePtr child = node->children; child; child = child->next)
if (child->type == XML_ELEMENT_NODE)
++childCount;
writeBuffer.Append(&childCount, 4);
// Filled in later with the offset to the list of child elements
size_t posChildrenOffset = writeBuffer.Size();
writeBuffer.Append("????", 4);
// Trim excess whitespace in the entity's text, while counting
// the number of newlines trimmed (so that JS error reporting
// can give the correct line number within the script)
std::string whitespace = " \t\r\n";
std::string text;
for (xmlNodePtr child = node->children; child; child = child->next)
{
if (child->type == XML_TEXT_NODE)
{
xmlChar* content = xmlNodeGetContent(child);
text += std::string((const char*)content);
xmlFree(content);
}
}
u32 linenum = xmlGetLineNo(node);
// Find the start of the non-whitespace section
size_t first = text.find_first_not_of(whitespace);
if (first == text.npos)
// Entirely whitespace - easy to handle
text = "";
else
{
// Count the number of \n being cut off,
// and add them to the line number
std::string trimmed (text.begin(), text.begin()+first);
linenum += std::count(trimmed.begin(), trimmed.end(), '\n');
// Find the end of the non-whitespace section,
// and trim off everything else
size_t last = text.find_last_not_of(whitespace);
text = text.substr(first, 1+last-first);
}
// Output text, prefixed by length in bytes
WriteStringAndLineNumber(writeBuffer, text, linenum);
// Output attributes
for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
{
u32 attrName = GetAttributeName((const char*)attr->name);
writeBuffer.Append(&attrName, 4);
xmlChar* value = xmlNodeGetContent(attr->children);
u32 attrLen = u32(xmlStrlen(value)+1);
writeBuffer.Append(&attrLen, 4);
writeBuffer.Append((void*)value, attrLen);
xmlFree(value);
}
// Go back and fill in the child-element offset
u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4));
writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset);
// Output all child elements
for (xmlNodePtr child = node->children; child; child = child->next)
if (child->type == XML_ELEMENT_NODE)
OutputElements<xmlNodePtr&&>(writeBuffer, std::move(child));
// Go back and fill in the length
u32 length = (u32)(writeBuffer.Size() - posLength);
writeBuffer.Overwrite(&length, 4, posLength);
return true;
}
} // anonymous namespace
bool XMBStorage::ReadFromFile(const PIVFS& vfs, const VfsPath& filename)
{
if(vfs->LoadFile(filename, m_Buffer, m_Size) < 0)
return false;
// if the game crashes during loading, (e.g. due to driver bugs),
// it sometimes leaves empty XMB files in the cache.
// reporting failure will cause our caller to re-generate the XMB.
if (m_Size == 0)
return false;
ENSURE(m_Size >= 4); // make sure it's at least got the initial header
return true;
}
bool XMBStorage::LoadXMLDoc(const xmlDocPtr doc)
{
WriteBuffer writeBuffer;
XMBStorageWriter writer;
if (!writer.Load(writeBuffer, std::move(xmlDocGetRootElement(doc))))
return false;
m_Buffer = writeBuffer.Data(); // add a reference
m_Size = writeBuffer.Size();
return true;
}
bool XMBStorage::LoadJSValue(const ScriptInterface& scriptInterface, JS::HandleValue value, const std::string& rootName)
{
WriteBuffer writeBuffer;
XMBStorageWriter writer;
const u32 name = writer.GetElementName(rootName);
JSNodeData data(scriptInterface);
if (!writer.Load(writeBuffer, data, name, std::move(value)))
return false;
m_Buffer = writeBuffer.Data(); // add a reference
m_Size = writeBuffer.Size();
return true;
}

110
source/ps/XMB/XMBStorage.h Normal file
View File

@ -0,0 +1,110 @@
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef INCLUDED_XMBSTORAGE
#define INCLUDED_XMBSTORAGE
#include "scriptinterface/ScriptForward.h"
#include <memory>
typedef struct _xmlDoc xmlDoc;
typedef xmlDoc* xmlDocPtr;
struct IVFS;
typedef std::shared_ptr<IVFS> PIVFS;
class Path;
typedef Path VfsPath;
/**
* Storage for XMBData
*/
class XMBStorage
{
public:
// File headers, to make sure it doesn't try loading anything other than an XMB
static const char* HeaderMagicStr;
static const char* UnfinishedHeaderMagicStr;
static const u32 XMBVersion;
XMBStorage() = default;
/**
* Read an XMB file on disk.
*/
bool ReadFromFile(const PIVFS& vfs, const VfsPath& filename);
/**
* Parse an XML document into XMB.
*
* Main limitations:
* - Can't correctly handle mixed text/elements inside elements -
* "<div> <b> Text </b> </div>" and "<div> Te<b/>xt </div>" are
* considered identical.
*/
bool LoadXMLDoc(const xmlDocPtr doc);
/**
* Parse a Javascript value into XMB.
* The syntax is similar to ParamNode, but supports multiple children with the same name, to match XML.
* You need to pass the name of the root object, as unlike XML this cannot be recovered from the value.
* The following JS object:
* {
* "a": 5,
* "b": "test",
* "x": {
* // Like ParamNode, _string is used for the value.
* "_string": "value",
* // Like ParamNode, attributes are prefixed with @.
* "@param": "something",
* "y": 3
* },
* // Every array item is parsed as a child.
* "object": [
* "a",
* "b",
* { "_string": "c" },
* { "child": "value" },
* ],
* // Same but without the array.
* "child@0@": 1,
* "child@1@": 2
* }
* will parse like the following XML:
* <a>5
* <b>test</b>
* <x param="something">value
* <y>3</y>
* </x>
* <object>a</object>
* <object>b</object>
* <object>c</object>
* <object><child>value</child></object>
* <child>1</child>
* <child>2</child>
* </a>
* See also tests for some other examples.
*/
bool LoadJSValue(const ScriptInterface& scriptInterface, JS::HandleValue value, const std::string& rootName);
std::shared_ptr<u8> m_Buffer;
size_t m_Size = 0;
};
#endif // INCLUDED_XMBSTORAGE

80
source/ps/XML/tests/test_XeroXMB.h → source/ps/XMB/tests/test_XMBData.h
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2015 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -18,41 +18,68 @@
#include "lib/self_test.h"
#include "ps/XML/Xeromyces.h"
#include "lib/file/io/write_buffer.h"
#include "ps/XMB/XMBStorage.h"
#include "scriptinterface/ScriptInterface.h"
#include <libxml/parser.h>
#include <memory>
class TestXeroXMB : public CxxTest::TestSuite
class TestXMBData : public CxxTest::TestSuite
{
private:
shared_ptr<u8> m_Buffer;
XMBFile parse(const char* doc)
std::unique_ptr<ScriptInterface> m_ScriptInterface;
CXeromyces parseXML(const char* doc)
{
xmlDocPtr xmlDoc = xmlReadMemory(doc, int(strlen(doc)), "", NULL,
XML_PARSE_NONET|XML_PARSE_NOCDATA);
WriteBuffer buffer;
PSRETURN ret = CXeromyces::CreateXMB(xmlDoc, buffer);
CXeromyces xmb;
bool ok = xmb.m_Data.LoadXMLDoc(xmlDoc);
xmlFreeDoc(xmlDoc);
TS_ASSERT_EQUALS(ok, true);
TS_ASSERT_EQUALS(ret, PSRETURN_OK);
XMBFile xmb;
m_Buffer = buffer.Data(); // hold a reference
TS_ASSERT(xmb.Initialise((const char*)m_Buffer.get()));
TS_ASSERT(xmb.Initialise(xmb.m_Data));
return xmb;
}
CXeromyces parseJS(const std::string rootName, const char* code)
{
ScriptRequest rq(*m_ScriptInterface);
JS::RootedValue val(rq.cx);
m_ScriptInterface->Eval(code, &val);
CXeromyces xmb;
bool ok = xmb.m_Data.LoadJSValue(*m_ScriptInterface, val, rootName);
TS_ASSERT_EQUALS(ok, true);
TS_ASSERT(xmb.Initialise(xmb.m_Data));
return xmb;
}
void setUp()
{
m_ScriptInterface = std::make_unique<ScriptInterface>("Test", "Test", g_ScriptContext);
}
void tearDown()
{
m_ScriptInterface.reset();
m_Buffer.reset();
}
public:
void test_basic()
{
XMBFile xmb (parse("<test>\n<foo x=' y '> bar </foo><foo>\n\n\nbar</foo>\n</test>"));
basic(parseXML("<test>\n<foo x=' y '> bar </foo><foo>\n\n\nbar</foo>\n</test>"), true);
// Array format for same-named elements.
basic(parseJS("test", "({ 'foo': [{ '@x': ' y ', '_string': 'bar' }, { '_string': '\\n\\n\\nbar' }] })"), false);
// Alternative format for same-named elements.
basic(parseJS("test", "({ 'foo@0@': { '@x': ' y ', '_string': 'bar' }, 'foo@1@': { '_string': '\\n\\n\\nbar' }})"), false);
}
void basic(const CXeromyces& xmb, bool checkLines)
{
TS_ASSERT_DIFFERS(xmb.GetElementID("test"), -1);
TS_ASSERT_DIFFERS(xmb.GetElementID("foo"), -1);
@ -68,17 +95,20 @@ public:
XMBElement root = xmb.GetRoot();
TS_ASSERT_EQUALS(root.GetNodeName(), el_test);
TS_ASSERT_EQUALS(root.GetLineNumber(), -1);
if (checkLines)
TS_ASSERT_EQUALS(root.GetLineNumber(), -1);
TS_ASSERT_EQUALS(CStr(root.GetText()), "");
TS_ASSERT_EQUALS(root.GetChildNodes().size(), 2);
XMBElement child = root.GetChildNodes()[0];
TS_ASSERT_EQUALS(child.GetNodeName(), el_foo);
TS_ASSERT_EQUALS(child.GetLineNumber(), 2);
if (checkLines)
TS_ASSERT_EQUALS(child.GetLineNumber(), 2);
TS_ASSERT_EQUALS(child.GetChildNodes().size(), 0);
TS_ASSERT_EQUALS(CStr(child.GetText()), "bar");
TS_ASSERT_EQUALS(root.GetChildNodes()[1].GetLineNumber(), 5);
if (checkLines)
TS_ASSERT_EQUALS(root.GetChildNodes()[1].GetLineNumber(), 5);
TS_ASSERT_EQUALS(child.GetAttributes().size(), 1);
XMBAttribute attr = child.GetAttributes()[0];
@ -88,8 +118,13 @@ public:
void test_GetFirstNamedItem()
{
XMBFile xmb (parse("<test> <x>A</x> <x>B</x> <y>C</y> <z>D</z> </test>"));
GetFirstNamedItem(parseXML("<test> <x>A</x> <x>B</x> <y>C</y> <z>D</z> </test>"), true);
GetFirstNamedItem(parseJS("test", "({ 'x': [{ '_string': 'A' }, 'B'], 'y': 'C', 'z': 'D' })"), false);
GetFirstNamedItem(parseJS("test", "({ 'x@0@': 'A', 'x@1@': 'B', 'y': 'C', 'z': 'D' })"), false);
}
void GetFirstNamedItem(const CXeromyces& xmb, bool checkLines)
{
XMBElement root = xmb.GetRoot();
TS_ASSERT_EQUALS(root.GetChildNodes().size(), 4);
@ -105,27 +140,28 @@ public:
TS_ASSERT_EQUALS(w.GetNodeName(), -1);
TS_ASSERT_EQUALS(CStr(w.GetText()), "");
TS_ASSERT_EQUALS(w.GetLineNumber(), -1);
if (checkLines)
TS_ASSERT_EQUALS(w.GetLineNumber(), -1);
TS_ASSERT_EQUALS(w.GetChildNodes().size(), 0);
TS_ASSERT_EQUALS(w.GetAttributes().size(), 0);
}
void test_doctype_ignored()
{
XMBFile xmb (parse("<!DOCTYPE foo SYSTEM \"file:///dev/urandom\"><foo/>"));
CXeromyces xmb (parseXML("<!DOCTYPE foo SYSTEM \"file:///dev/urandom\"><foo/>"));
TS_ASSERT_DIFFERS(xmb.GetElementID("foo"), -1);
}
void test_complex_parse()
{
XMBFile xmb (parse("<test>\t\n \tx &lt;>&amp;&quot;&apos;<![CDATA[foo]]>bar\n<x/>\nbaz<?cheese?>qux</test>"));
CXeromyces xmb (parseXML("<test>\t\n \tx &lt;>&amp;&quot;&apos;<![CDATA[foo]]>bar\n<x/>\nbaz<?cheese?>qux</test>"));
TS_ASSERT_EQUALS(CStr(xmb.GetRoot().GetText()), "x <>&\"'foobar\n\nbazqux");
}
void test_unicode()
{
XMBFile xmb (parse("<?xml version=\"1.0\" encoding=\"utf-8\"?><foo x='&#x1234;\xE1\x88\xB4'>&#x1234;\xE1\x88\xB4</foo>"));
CXeromyces xmb (parseXML("<?xml version=\"1.0\" encoding=\"utf-8\"?><foo x='&#x1234;\xE1\x88\xB4'>&#x1234;\xE1\x88\xB4</foo>"));
CStrW text;
text = xmb.GetRoot().GetText().FromUTF8();
@ -141,7 +177,7 @@ public:
void test_iso88591()
{
XMBFile xmb (parse("<?xml version=\"1.0\" encoding=\"iso-8859-1\"?><foo x='&#x1234;\xE1\x88\xB4'>&#x1234;\xE1\x88\xB4</foo>"));
CXeromyces xmb (parseXML("<?xml version=\"1.0\" encoding=\"iso-8859-1\"?><foo x='&#x1234;\xE1\x88\xB4'>&#x1234;\xE1\x88\xB4</foo>"));
CStrW text;
text = xmb.GetRoot().GetText().FromUTF8();

14
source/ps/XML/XMLWriter.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2013 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -114,20 +114,20 @@ const CStr& XMLWriter_File::GetOutput()
}
void XMLWriter_File::XMB(const XMBFile& file)
void XMLWriter_File::XMB(const XMBData& xmb)
{
ElementXMB(file, file.GetRoot());
ElementXMB(xmb, xmb.GetRoot());
}
void XMLWriter_File::ElementXMB(const XMBFile& file, XMBElement el)
void XMLWriter_File::ElementXMB(const XMBData& xmb, XMBElement el)
{
XMLWriter_Element writer(*this, file.GetElementString(el.GetNodeName()).c_str());
XMLWriter_Element writer(*this, xmb.GetElementString(el.GetNodeName()));
XERO_ITER_ATTR(el, attr)
writer.Attribute(file.GetAttributeString(attr.Name).c_str(), attr.Value);
writer.Attribute(xmb.GetAttributeString(attr.Name), attr.Value);
XERO_ITER_EL(el, child)
ElementXMB(file, child);
ElementXMB(xmb, child);
}
void XMLWriter_File::Comment(const char* text)

8
source/ps/XML/XMLWriter.h
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2019 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -65,7 +65,7 @@
#include "ps/CStr.h"
class XMBElement;
class XMBFile;
class XMBData;
class XMLWriter_Element;
class XMLWriter_File
@ -77,7 +77,7 @@ public:
void Comment(const char* text);
void XMB(const XMBFile& file);
void XMB(const XMBData& xmb);
bool StoreVFS(const PIVFS& vfs, const VfsPath& pathname);
const CStr8& GetOutput();
@ -86,7 +86,7 @@ private:
friend class XMLWriter_Element;
void ElementXMB(const XMBFile& file, XMBElement el);
void ElementXMB(const XMBData& xmb, XMBElement el);
void ElementStart(XMLWriter_Element* element, const char* name);
void ElementText(const char* text, bool cdata);

219
source/ps/XML/Xeromyces.cpp
View File

@ -28,6 +28,7 @@
#include "ps/CacheLoader.h"
#include "ps/CLogger.h"
#include "ps/Filesystem.h"
#include "RelaxNG.h"
#include "Xeromyces.h"
@ -120,13 +121,17 @@ PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename, const std::
validatorGrammarHash = GetValidator(validatorName).GetGrammarHash();
}
VfsPath xmbPath;
Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBVersion, xmbPath);
Status ret = cacheLoader.TryLoadingCached(filename, validatorGrammarHash, XMBStorage::XMBVersion, xmbPath);
if (ret == INFO::OK)
{
// Found a cached XMB - load it
if (ReadXMBFile(vfs, xmbPath))
if (m_Data.ReadFromFile(vfs, xmbPath))
{
if(!Initialise(m_Data))
return PSRETURN_Xeromyces_XMLParseError;
return PSRETURN_OK;
}
// If this fails then we'll continue and (re)create the loose cache -
// this failure legitimately happens due to partially-written XMB files.
}
@ -184,43 +189,20 @@ PSRETURN CXeromyces::ConvertFile(const PIVFS& vfs, const VfsPath& filename, cons
}
}
WriteBuffer writeBuffer;
CreateXMB(doc, writeBuffer);
m_Data.LoadXMLDoc(doc);
xmlFreeDoc(doc);
// Save the file to disk, so it can be loaded quickly next time.
// Don't save if invalid, because we want the syntax error every program start.
vfs->CreateFile(xmbPath, writeBuffer.Data(), writeBuffer.Size());
vfs->CreateFile(xmbPath, m_Data.m_Buffer, m_Data.m_Size);
m_XMBBuffer = writeBuffer.Data(); // add a reference
// Set up the XMBFile
const bool ok = Initialise((const char*)m_XMBBuffer.get());
// Set up the XMBData
const bool ok = Initialise(m_Data);
ENSURE(ok);
return PSRETURN_OK;
}
bool CXeromyces::ReadXMBFile(const PIVFS& vfs, const VfsPath& filename)
{
size_t size;
if(vfs->LoadFile(filename, m_XMBBuffer, size) < 0)
return false;
// if the game crashes during loading, (e.g. due to driver bugs),
// it sometimes leaves empty XMB files in the cache.
// reporting failure will cause our caller to re-generate the XMB.
if(size == 0)
return false;
ENSURE(size >= 4); // make sure it's at least got the initial header
// Set up the XMBFile
if(!Initialise((const char*)m_XMBBuffer.get()))
return false;
return true;
}
PSRETURN CXeromyces::LoadString(const char* xml, const std::string& validatorName /* = "" */)
{
ENSURE(g_XeromycesStarted);
@ -242,185 +224,12 @@ PSRETURN CXeromyces::LoadString(const char* xml, const std::string& validatorNam
}
}
WriteBuffer writeBuffer;
CreateXMB(doc, writeBuffer);
m_Data.LoadXMLDoc(doc);
xmlFreeDoc(doc);
m_XMBBuffer = writeBuffer.Data(); // add a reference
// Set up the XMBFile
const bool ok = Initialise((const char*)m_XMBBuffer.get());
// Set up the XMBData
const bool ok = Initialise(m_Data);
ENSURE(ok);
return PSRETURN_OK;
}
static void FindNames(const xmlNodePtr node, std::set<std::string>& elementNames, std::set<std::string>& attributeNames)
{
elementNames.insert((const char*)node->name);
for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
attributeNames.insert((const char*)attr->name);
for (xmlNodePtr child = node->children; child; child = child->next)
if (child->type == XML_ELEMENT_NODE)
FindNames(child, elementNames, attributeNames);
}
static void OutputElement(const xmlNodePtr node, WriteBuffer& writeBuffer,
std::map<std::string, u32>& elementIDs,
std::map<std::string, u32>& attributeIDs
)
{
// Filled in later with the length of the element
size_t posLength = writeBuffer.Size();
writeBuffer.Append("????", 4);
writeBuffer.Append(&elementIDs[(const char*)node->name], 4);
u32 attrCount = 0;
for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
++attrCount;
writeBuffer.Append(&attrCount, 4);
u32 childCount = 0;
for (xmlNodePtr child = node->children; child; child = child->next)
if (child->type == XML_ELEMENT_NODE)
++childCount;
writeBuffer.Append(&childCount, 4);
// Filled in later with the offset to the list of child elements
size_t posChildrenOffset = writeBuffer.Size();
writeBuffer.Append("????", 4);
// Trim excess whitespace in the entity's text, while counting
// the number of newlines trimmed (so that JS error reporting
// can give the correct line number within the script)
std::string whitespace = " \t\r\n";
std::string text;
for (xmlNodePtr child = node->children; child; child = child->next)
{
if (child->type == XML_TEXT_NODE)
{
xmlChar* content = xmlNodeGetContent(child);
text += std::string((const char*)content);
xmlFree(content);
}
}
u32 linenum = xmlGetLineNo(node);
// Find the start of the non-whitespace section
size_t first = text.find_first_not_of(whitespace);
if (first == text.npos)
// Entirely whitespace - easy to handle
text = "";
else
{
// Count the number of \n being cut off,
// and add them to the line number
std::string trimmed (text.begin(), text.begin()+first);
linenum += std::count(trimmed.begin(), trimmed.end(), '\n');
// Find the end of the non-whitespace section,
// and trim off everything else
size_t last = text.find_last_not_of(whitespace);
text = text.substr(first, 1+last-first);
}
// Output text, prefixed by length in bytes
if (text.length() == 0)
{
// No text; don't write much
writeBuffer.Append("\0\0\0\0", 4);
}
else
{
// Write length and line number and null-terminated text
u32 nodeLen = u32(4 + text.length()+1);
writeBuffer.Append(&nodeLen, 4);
writeBuffer.Append(&linenum, 4);
writeBuffer.Append((void*)text.c_str(), nodeLen-4);
}
// Output attributes
for (xmlAttrPtr attr = node->properties; attr; attr = attr->next)
{
writeBuffer.Append(&attributeIDs[(const char*)attr->name], 4);
xmlChar* value = xmlNodeGetContent(attr->children);
u32 attrLen = u32(xmlStrlen(value)+1);
writeBuffer.Append(&attrLen, 4);
writeBuffer.Append((void*)value, attrLen);
xmlFree(value);
}
// Go back and fill in the child-element offset
u32 childrenOffset = (u32)(writeBuffer.Size() - (posChildrenOffset+4));
writeBuffer.Overwrite(&childrenOffset, 4, posChildrenOffset);
// Output all child elements
for (xmlNodePtr child = node->children; child; child = child->next)
if (child->type == XML_ELEMENT_NODE)
OutputElement(child, writeBuffer, elementIDs, attributeIDs);
// Go back and fill in the length
u32 length = (u32)(writeBuffer.Size() - posLength);
writeBuffer.Overwrite(&length, 4, posLength);
}
PSRETURN CXeromyces::CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer)
{
// Header
writeBuffer.Append(UnfinishedHeaderMagicStr, 4);
// Version
writeBuffer.Append(&XMBVersion, 4);
u32 i;
// Find the unique element/attribute names
std::set<std::string> elementNames;
std::set<std::string> attributeNames;
FindNames(xmlDocGetRootElement(doc), elementNames, attributeNames);
std::map<std::string, u32> elementIDs;
std::map<std::string, u32> attributeIDs;
// Output element names
i = 0;
u32 elementCount = (u32)elementNames.size();
writeBuffer.Append(&elementCount, 4);
for (const std::string& n : elementNames)
{
u32 textLen = (u32)n.length()+1;
writeBuffer.Append(&textLen, 4);
writeBuffer.Append((void*)n.c_str(), textLen);
elementIDs[n] = i++;
}
// Output attribute names
i = 0;
u32 attributeCount = (u32)attributeNames.size();
writeBuffer.Append(&attributeCount, 4);
for (const std::string& n : attributeNames)
{
u32 textLen = (u32)n.length()+1;
writeBuffer.Append(&textLen, 4);
writeBuffer.Append((void*)n.c_str(), textLen);
attributeIDs[n] = i++;
}
OutputElement(xmlDocGetRootElement(doc), writeBuffer, elementIDs, attributeIDs);
// file is now valid, so insert correct magic string
writeBuffer.Overwrite(HeaderMagicStr, 4, 0);
return PSRETURN_OK;
}

18
source/ps/XML/Xeromyces.h
View File

@ -30,19 +30,17 @@ ERROR_TYPE(Xeromyces, XMLOpenFailed);
ERROR_TYPE(Xeromyces, XMLParseError);
ERROR_TYPE(Xeromyces, XMLValidationFailed);
#include "XeroXMB.h"
#include "ps/XMB/XMBData.h"
#include "ps/XMB/XMBStorage.h"
#include "lib/file/vfs/vfs.h"
class RelaxNGValidator;
class WriteBuffer;
typedef struct _xmlDoc xmlDoc;
typedef xmlDoc* xmlDocPtr;
class CXeromyces : public XMBFile
class CXeromyces : public XMBData
{
friend class TestXeroXMB;
friend class TestXMBData;
friend class XMBData;
public:
/**
* Load from an XML file (with invisible XMB caching).
@ -81,11 +79,7 @@ private:
PSRETURN ConvertFile(const PIVFS& vfs, const VfsPath& filename, const VfsPath& xmbPath, const std::string& validatorName);
bool ReadXMBFile(const PIVFS& vfs, const VfsPath& filename);
static PSRETURN CreateXMB(const xmlDocPtr doc, WriteBuffer& writeBuffer);
shared_ptr<u8> m_XMBBuffer;
XMBStorage m_Data;
};

10
source/simulation2/system/ParamNode.cpp
View File

@ -37,7 +37,7 @@ CParamNode::CParamNode(bool isOk) :
{
}
void CParamNode::LoadXML(CParamNode& ret, const XMBFile& xmb, const wchar_t* sourceIdentifier /*= NULL*/)
void CParamNode::LoadXML(CParamNode& ret, const XMBData& xmb, const wchar_t* sourceIdentifier /*= NULL*/)
{
ret.ApplyLayer(xmb, xmb.GetRoot(), sourceIdentifier);
}
@ -64,11 +64,11 @@ PSRETURN CParamNode::LoadXMLString(CParamNode& ret, const char* xml, const wchar
return PSRETURN_OK;
}
void CParamNode::ApplyLayer(const XMBFile& xmb, const XMBElement& element, const wchar_t* sourceIdentifier /*= NULL*/)
void CParamNode::ApplyLayer(const XMBData& xmb, const XMBElement& element, const wchar_t* sourceIdentifier /*= NULL*/)
{
ResetScriptVal();
std::string name = xmb.GetElementString(element.GetNodeName()); // TODO: is GetElementString inefficient?
std::string name = xmb.GetElementString(element.GetNodeName());
CStr value = element.GetText();
bool hasSetValue = false;
@ -224,8 +224,8 @@ void CParamNode::ApplyLayer(const XMBFile& xmb, const XMBElement& element, const
if (attr.Name == at_replace || attr.Name == at_op || attr.Name == at_merge || attr.Name == at_filtered)
continue;
// Add any others
std::string attrName = xmb.GetAttributeString(attr.Name);
node.m_Childs["@" + attrName].m_Value = attr.Value;
const char* attrName(xmb.GetAttributeString(attr.Name));
node.m_Childs[CStr("@") + attrName].m_Value = attr.Value;
}
}

6
source/simulation2/system/ParamNode.h
View File

@ -27,7 +27,7 @@
#include <map>
#include <string>
class XMBFile;
class XMBData;
class XMBElement;
class ScriptRequest;
@ -165,7 +165,7 @@ public:
* @param sourceIdentifier Optional; string you can pass along to indicate the source of
* the data getting loaded. Used for output to log messages if an error occurs.
*/
static void LoadXML(CParamNode& ret, const XMBFile& file, const wchar_t* sourceIdentifier = NULL);
static void LoadXML(CParamNode& ret, const XMBData& xmb, const wchar_t* sourceIdentifier = NULL);
/**
* Loads the XML data specified by @a path into the node @a ret.
@ -276,7 +276,7 @@ private:
* @param sourceIdentifier Optional; string you can pass along to indicate the source of
* the data getting applied. Used for output to log messages if an error occurs.
*/
void ApplyLayer(const XMBFile& xmb, const XMBElement& element, const wchar_t* sourceIdentifier = NULL);
void ApplyLayer(const XMBData& xmb, const XMBElement& element, const wchar_t* sourceIdentifier = NULL);
void ResetScriptVal();

2
source/soundmanager/scripting/SoundGroup.cpp
View File

@ -329,7 +329,7 @@ bool CSoundGroup::LoadSoundGroup(const VfsPath& pathnameXML)
if (root.GetNodeName() != el_soundgroup)
{
LOGERROR("Invalid SoundGroup format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName()).c_str());
LOGERROR("Invalid SoundGroup format (unrecognised root element '%s')", XeroFile.GetElementString(root.GetNodeName()));
return false;
}