Improve performance of full state hash computation, by skipping script components that are known to have no data.

Switch to much quicker (and less comprehensive) per-turn hashing in
multiplayer games.

This was SVN commit r9036.

binaries/data/mods/public/simulation/components/Armour.js

@@ -21,6 +21,8 @@ Armour.prototype.Init = function()
 {
 };
 
+Armour.prototype.Serialize = null; // we have no dynamic state to save
+
 Armour.prototype.TakeDamage = function(hack, pierce, crush)
 {
 	// Adjust damage values based on armour

binaries/data/mods/public/simulation/components/Attack.js

@@ -73,6 +73,12 @@ Attack.prototype.Schema =
 		"</element>" +
 	"</optional>";
 
+Attack.prototype.Init = function()
+{
+};
+
+Attack.prototype.Serialize = null; // we have no dynamic state to save
+
 /**
  * Return the type of the best attack.
  * TODO: this should probably depend on range, target, etc,

binaries/data/mods/public/simulation/components/Auras.js

@@ -37,4 +37,6 @@ Auras.prototype.Schema =
  * TODO: this all needs to be designed and implemented
  */
 
+Auras.prototype.Serialize = null; // we have no dynamic state to save
+
 Engine.RegisterComponentType(IID_Auras, "Auras", Auras);

binaries/data/mods/public/simulation/components/Builder.js

@@ -22,6 +22,8 @@ Builder.prototype.Init = function()
 {
 };
 
+Builder.prototype.Serialize = null; // we have no dynamic state to save
+
 Builder.prototype.GetEntitiesList = function()
 {
 	var string = this.template.Entities._string;

binaries/data/mods/public/simulation/components/Cost.js

@@ -31,6 +31,12 @@ Cost.prototype.Schema =
 		"</interleave>" +
 	"</element>";
 
+Cost.prototype.Init = function()
+{
+};
+
+Cost.prototype.Serialize = null; // we have no dynamic state to save
+
 Cost.prototype.GetPopCost = function()
 {
 	return +this.template.Population;

binaries/data/mods/public/simulation/components/Identity.js

@@ -106,6 +106,8 @@ Identity.prototype.Init = function()
 {
 };
 
+Identity.prototype.Serialize = null; // we have no dynamic state to save
+
 Identity.prototype.GetCiv = function()
 {
 	return this.template.Civ;

binaries/data/mods/public/simulation/components/Loot.js

@@ -21,4 +21,6 @@ Loot.prototype.Schema =
  * TODO: this all needs to be designed and implemented
  */
 
+Loot.prototype.Serialize = null; // we have no dynamic state to save
+
 Engine.RegisterComponentType(IID_Loot, "Loot", Loot);

binaries/data/mods/public/simulation/components/Sound.js

@@ -23,6 +23,8 @@ Sound.prototype.Init = function()
 {
 };
 
+Sound.prototype.Serialize = null; // we have no dynamic state to save
+
 Sound.prototype.GetSoundGroup = function(name)
 {
 	return this.template.SoundGroups[name] || "";

binaries/data/mods/public/simulation/components/StatusBars.js

@@ -16,12 +16,15 @@ StatusBars.prototype.Init = function()
 	this.enabled = false;
 };
 
-StatusBars.prototype.Serialize = function()
+// Because this is enabled directly by the GUI and is not
+// network-synchronised (it only affects local rendering),
+// we disable serialization in order to prevent OOS errors
+StatusBars.prototype.Serialize = null;
+
+StatusBars.prototype.Deserialize = function()
 {
-	// Because this is enabled directly by the GUI and is not
-	// network-synchronised (it only affects local rendering),
-	// return a dummy value to prevent OOS errors
-	return { "enabled": false };
+	// Use default initialisation
+	this.Init();
 };
 
 StatusBars.prototype.SetEnabled = function(enabled)

source/network/NetTurnManager.cpp

@@ -176,8 +176,9 @@ void CNetTurnManager::OnSyncError(u32 turn, const std::string& expectedHash)
 		return;
 	m_HasSyncError = true;
 
+	bool quick = !TurnNeedsFullHash(turn);
 	std::string hash;
-	bool ok = m_Simulation2.ComputeStateHash(hash);
+	bool ok = m_Simulation2.ComputeStateHash(hash, quick);
 	debug_assert(ok);
 
 	fs::wpath path (psLogDir()/L"oos_dump.txt");
@@ -226,6 +227,22 @@ void CNetTurnManager::FinishedAllCommands(u32 turn, u32 turnLength)
 	m_TurnLength = turnLength;
 }
 
+bool CNetTurnManager::TurnNeedsFullHash(u32 turn)
+{
+	// Check immediately for errors caused by e.g. inconsistent game versions
+	// (The hash is computed after the first sim update, so we start at turn == 1)
+	if (turn == 1)
+		return true;
+
+	// Otherwise check the full state every ~10 seconds in multiplayer games
+	// (TODO: should probably remove this when we're reasonably sure the game
+	// isn't too buggy, since the full hash is still pretty slow)
+	if (turn % 20 == 0)
+		return true;
+
+	return false;
+}
+
 void CNetTurnManager::EnableTimeWarpRecording(size_t numTurns)
 {
 	m_TimeWarpStates.clear();
@@ -286,16 +303,17 @@ void CNetClientTurnManager::NotifyFinishedOwnCommands(u32 turn)
 
 void CNetClientTurnManager::NotifyFinishedUpdate(u32 turn)
 {
+	bool quick = !TurnNeedsFullHash(turn);
 	std::string hash;
 	{
 		PROFILE("state hash check");
-		bool ok = m_Simulation2.ComputeStateHash(hash);
+		bool ok = m_Simulation2.ComputeStateHash(hash, quick);
 		debug_assert(ok);
 	}
 
 	NETTURN_LOG((L"NotifyFinishedUpdate(%d, %ls)\n", turn, Hexify(hash).c_str()));
 
-	m_Replay.Hash(hash);
+	m_Replay.Hash(hash, quick);
 
 	// Send message to the server
 	CSyncCheckMessage msg;

source/network/NetTurnManager.h

@@ -135,6 +135,12 @@ protected:
 	 */
 	virtual void NotifyFinishedUpdate(u32 turn) = 0;
 
+	/**
+	 * Returns whether we should compute a complete state hash for the given turn,
+	 * instead of a quick less-complete hash.
+	 */
+	bool TurnNeedsFullHash(u32 turn);
+
 	CSimulation2& m_Simulation2;
 
 	/// The turn that we have most recently executed

source/ps/Replay.cpp

@@ -93,9 +93,12 @@ void CReplayLogger::Turn(u32 n, u32 turnLength, const std::vector<SimulationComm
 	m_Stream->flush();
 }
 
-void CReplayLogger::Hash(const std::string& hash)
+void CReplayLogger::Hash(const std::string& hash, bool quick)
 {
-	*m_Stream << "hash " << Hexify(hash) << "\n";
+	if (quick)
+		*m_Stream << "hash-quick " << Hexify(hash) << "\n";
+	else
+		*m_Stream << "hash " << Hexify(hash) << "\n";
 }
 
 ////////////////////////////////////////////////////////////////
@@ -174,17 +177,19 @@ void CReplayPlayer::Replay()
 			SimulationCommand cmd = { player, data };
 			commands.push_back(cmd);
 		}
-		else if (type == "hash")
+		else if (type == "hash" || type == "hash-quick")
 		{
 			std::string replayHash;
 			*m_Stream >> replayHash;
 
+			bool quick = (type == "hash-quick");
+
 //			if (turn >= 1300)
 //			if (turn >= 0)
 			if (turn % 100 == 0)
 			{
 				std::string hash;
-				bool ok = game.GetSimulation2()->ComputeStateHash(hash);
+				bool ok = game.GetSimulation2()->ComputeStateHash(hash, quick);
 				debug_assert(ok);
 				std::string hexHash = Hexify(hash);
 				if (hexHash == replayHash)
@@ -199,7 +204,7 @@ void CReplayPlayer::Replay()
 			commands.clear();
 
 //			std::string hash;
-//			bool ok = game.GetSimulation2()->ComputeStateHash(hash);
+//			bool ok = game.GetSimulation2()->ComputeStateHash(hash, true);
 //			debug_assert(ok);
 //			debug_printf(L"%hs", Hexify(hash).c_str());
 
@@ -220,7 +225,7 @@ void CReplayPlayer::Replay()
 	}
 
 	std::string hash;
-	bool ok = game.GetSimulation2()->ComputeStateHash(hash);
+	bool ok = game.GetSimulation2()->ComputeStateHash(hash, false);
 	debug_assert(ok);
 	debug_printf(L"# Final state: %hs\n", Hexify(hash).c_str());
 

source/ps/Replay.h

@@ -45,7 +45,7 @@ public:
 	/**
 	 * Optional hash of simulation state (for sync checking).
 	 */
-	virtual void Hash(const std::string& hash) = 0;
+	virtual void Hash(const std::string& hash, bool quick) = 0;
 };
 
 /**
@@ -56,7 +56,7 @@ class CDummyReplayLogger : public IReplayLogger
 public:
 	virtual void StartGame(const CScriptValRooted& UNUSED(attribs)) { }
 	virtual void Turn(u32 UNUSED(n), u32 UNUSED(turnLength), const std::vector<SimulationCommand>& UNUSED(commands)) { }
-	virtual void Hash(const std::string& UNUSED(hash)) { }
+	virtual void Hash(const std::string& UNUSED(hash), bool UNUSED(quick)) { }
 };
 
 /**
@@ -71,7 +71,7 @@ public:
 
 	virtual void StartGame(const CScriptValRooted& attribs);
 	virtual void Turn(u32 n, u32 turnLength, const std::vector<SimulationCommand>& commands);
-	virtual void Hash(const std::string& hash);
+	virtual void Hash(const std::string& hash, bool quick);
 
 private:
 	ScriptInterface& m_ScriptInterface;

source/simulation2/Simulation2.cpp

@@ -295,7 +295,7 @@ void CSimulation2Impl::DumpState()
 
 	file << "State hash: " << std::hex;
 	std::string hashRaw;
-	m_ComponentManager.ComputeStateHash(hashRaw);
+	m_ComponentManager.ComputeStateHash(hashRaw, false);
 	for (size_t i = 0; i < hashRaw.size(); ++i)
 		file << std::setfill('0') << std::setw(2) << (int)(unsigned char)hashRaw[i];
 	file << std::dec << "\n";
@@ -493,9 +493,9 @@ void CSimulation2::ResetState(bool skipScriptedComponents, bool skipAI)
 	m->ResetState(skipScriptedComponents, skipAI);
 }
 
-bool CSimulation2::ComputeStateHash(std::string& outHash)
+bool CSimulation2::ComputeStateHash(std::string& outHash, bool quick)
 {
-	return m->m_ComponentManager.ComputeStateHash(outHash);
+	return m->m_ComponentManager.ComputeStateHash(outHash, quick);
 }
 
 bool CSimulation2::DumpDebugState(std::ostream& stream)

source/simulation2/Simulation2.h

@@ -195,7 +195,7 @@ public:
 	const CSimContext& GetSimContext() const;
 	ScriptInterface& GetScriptInterface() const;
 
-	bool ComputeStateHash(std::string& outHash);
+	bool ComputeStateHash(std::string& outHash, bool quick);
 	bool DumpDebugState(std::ostream& stream);
 	bool SerializeState(std::ostream& stream);
 	bool DeserializeState(std::istream& stream);

source/simulation2/scripting/ScriptComponent.cpp

@@ -28,6 +28,14 @@ CComponentTypeScript::CComponentTypeScript(ScriptInterface& scriptInterface, jsv
 	// Cache the property detection for efficiency
 	m_HasCustomSerialize = m_ScriptInterface.HasProperty(m_Instance.get(), "Serialize");
 	m_HasCustomDeserialize = m_ScriptInterface.HasProperty(m_Instance.get(), "Deserialize");
+
+	m_HasNullSerialize = false;
+	if (m_HasCustomSerialize)
+	{
+		CScriptVal val;
+		if (m_ScriptInterface.GetProperty(m_Instance.get(), "Serialize", val) && JSVAL_IS_NULL(val.get()))
+			m_HasNullSerialize = true;
+	}
 }
 
 void CComponentTypeScript::Init(const CParamNode& paramNode, entity_id_t ent)
@@ -54,6 +62,10 @@ void CComponentTypeScript::HandleMessage(const CMessage& msg, bool global)
 
 void CComponentTypeScript::Serialize(ISerializer& serialize)
 {
+	// If the component set Serialize = null, then do no work here
+	if (m_HasNullSerialize)
+		return;
+
 	// Support a custom "Serialize" function, which returns a new object that will be
 	// serialized instead of the component itself
 	if (m_HasCustomSerialize)
@@ -76,15 +88,22 @@ void CComponentTypeScript::Deserialize(const CParamNode& paramNode, IDeserialize
 	if (m_HasCustomDeserialize)
 	{
 		CScriptVal val;
-		deserialize.ScriptVal("object", val);
+
+		// If Serialize = null, we'll still call Deserialize but with undefined argument
+		if (!m_HasNullSerialize)
+			deserialize.ScriptVal("object", val);
+
 		if (!m_ScriptInterface.CallFunctionVoid(m_Instance.get(), "Deserialize", val))
 			LOGERROR(L"Script Deserialize call failed");
 	}
 	else
 	{
-		// Use ScriptObjectAppend so we don't lose the carefully-constructed
-		// prototype/parent of this object
-		deserialize.ScriptObjectAppend("object", m_Instance.getRef());
+		if (!m_HasNullSerialize)
+		{
+			// Use ScriptObjectAppend so we don't lose the carefully-constructed
+			// prototype/parent of this object
+			deserialize.ScriptObjectAppend("object", m_Instance.getRef());
+		}
 	}
 
 	m_ScriptInterface.SetProperty(m_Instance.get(), "entity", (int)ent, true, false);

source/simulation2/scripting/ScriptComponent.h

@@ -77,6 +77,7 @@ private:
 	CScriptValRooted m_Instance;
 	bool m_HasCustomSerialize;
 	bool m_HasCustomDeserialize;
+	bool m_HasNullSerialize;
 
 	NONCOPYABLE(CComponentTypeScript);
 };

source/simulation2/system/ComponentManager.h

@@ -207,7 +207,7 @@ public:
 	void ResetState();
 
 	// Various state serialization functions:
-	bool ComputeStateHash(std::string& outHash);
+	bool ComputeStateHash(std::string& outHash, bool quick);
 	bool DumpDebugState(std::ostream& stream);
 	// FlushDestroyedComponents must be called before SerializeState (since the destruction queue
 	// won't get serialized)

source/simulation2/system/ComponentManagerSerialization.cpp

@@ -100,11 +100,15 @@ bool CComponentManager::DumpDebugState(std::ostream& stream)
 	return true;
 }
 
-bool CComponentManager::ComputeStateHash(std::string& outHash)
+bool CComponentManager::ComputeStateHash(std::string& outHash, bool quick)
 {
 	// Hash serialization: this includes the minimal data necessary to detect
 	// differences in the state, and ignores things like counts and names
 
+	// If 'quick' is set, this checks even fewer things, so that it will
+	// be fast enough to run every turn but will typically detect any
+	// out-of-syncs fairly soon
+
 	CHashSerializer serializer(m_ScriptInterface);
 
 	serializer.StringASCII("rng", SerializeRNG(m_RNG), 0, 32);
@@ -116,6 +120,10 @@ bool CComponentManager::ComputeStateHash(std::string& outHash)
 		if (cit->second.empty())
 			continue;
 
+		// In quick mode, only check unit positions
+		if (quick && !(cit->first == CID_Position))
+			continue;
+
 		serializer.NumberI32_Unbounded("component type id", cit->first);
 
 		std::map<entity_id_t, IComponent*>::const_iterator eit = cit->second.begin();

source/simulation2/tests/test_ComponentManager.h

@@ -586,7 +586,7 @@ public:
 		);
 
 		std::string hash;
-		TS_ASSERT(man.ComputeStateHash(hash));
+		TS_ASSERT(man.ComputeStateHash(hash, false));
 		TS_ASSERT_EQUALS(hash.length(), (size_t)16);
 		TS_ASSERT_SAME_DATA(hash.data(), "\x1c\x45\x2b\x20\x1f\x0c\x00\x93\x60\x78\xe2\x63\xb1\x47\x08\x19", 16);
 		// echo -en "\x05\x00\x00\x0078606\x01\0\0\0\x01\0\0\0\xf8\x2a\0\0\x02\0\0\0\xd2\x04\0\0\x04\0\0\0\x01\0\0\0\x08\x52\0\0" | openssl md5 | perl -pe 's/(..)/\\x$1/g'

source/simulation2/tests/test_Serializer.h

@@ -509,7 +509,7 @@ public:
 			std::stringstream str;
 			std::string hash;
 			sim2.SerializeState(str);
-			sim2.ComputeStateHash(hash);
+			sim2.ComputeStateHash(hash, false);
 			debug_printf(L"\n");
 			debug_printf(L"# size = %d\n", (int)str.str().length());
 			debug_printf(L"# hash = ");
@@ -524,7 +524,7 @@ public:
 		for (size_t i = 0; i < reps; ++i)
 		{
 			std::string hash;
-			sim2.ComputeStateHash(hash);
+			sim2.ComputeStateHash(hash, false);
 		}
 		CALLGRIND_STOP_INSTRUMENTATION
 		t = timer_Time() - t;