Remember OOS on a per-client basis.

Change the OOS notification logic to remember the OOS-ness of each
client. Reset it on client leave.
The server will thus continue checking for OOS if the OOS-client leaves.
This is convenient to ignore observer OOS, or wait for an OOS player
without restarting the game.

Also add the turn number to the OOS dump, to fix #3348: particularly
following d4c2cf4430 the turn is likely to not be the same between
different clients.

Agree by: asterix
Differential Revision: https://code.wildfiregames.com/D3753
This was SVN commit r25170.

source/network/NetClientTurnManager.cpp

@@ -84,10 +84,6 @@ void CNetClientTurnManager::NotifyFinishedUpdate(u32 turn)
 
 	m_Replay.Hash(hash, quick);
 
-	// Don't send the hash if OOS
-	if (m_HasSyncError)
-		return;
-
 	// Send message to the server
 	CSyncCheckMessage msg;
 	msg.m_Turn = turn;
@@ -114,17 +110,13 @@ void CNetClientTurnManager::OnSyncError(u32 turn, const CStr& expectedHash, cons
 	CStr expectedHashHex(Hexify(expectedHash));
 	NETCLIENTTURN_LOG("OnSyncError(%d, %hs)\n", turn, expectedHashHex.c_str());
 
-	// Only complain the first time
-	if (m_HasSyncError)
-		return;
-
-	m_HasSyncError = true;
-
 	std::string hash;
 	ENSURE(m_Simulation2.ComputeStateHash(hash, !TurnNeedsFullHash(turn)));
 
 	OsPath oosdumpPath(psLogDir() / (L"oos_dump" + g_UniqueLogPostfix + L".txt"));
 	std::ofstream file (OsString(oosdumpPath).c_str(), std::ofstream::out | std::ofstream::trunc);
+	file << "oos turn: " << turn << std::endl;
+	file << "net client turn: " << m_CurrentTurn << std::endl;
 	m_Simulation2.DumpDebugState(file);
 	file.close();
 

source/network/NetServerTurnManager.cpp

@@ -35,7 +35,7 @@
 #endif
 
 CNetServerTurnManager::CNetServerTurnManager(CNetServerWorker& server)
-	: m_NetServer(server), m_ReadyTurn(COMMAND_DELAY_MP - 1), m_TurnLength(DEFAULT_TURN_LENGTH), m_HasSyncError(false)
+	: m_NetServer(server), m_ReadyTurn(COMMAND_DELAY_MP - 1), m_TurnLength(DEFAULT_TURN_LENGTH)
 {
 	// Turn 0 is not actually executed, store a dummy value.
 	m_SavedTurnLengths.push_back(0);
@@ -52,21 +52,21 @@ void CNetServerTurnManager::NotifyFinishedClientCommands(CNetServerSession& sess
 	NETSERVERTURN_LOG("NotifyFinishedClientCommands(client=%d, turn=%d)\n", client, turn);
 
 	// Must be a client we've already heard of
-	ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end());
+	ENSURE(m_ClientsData.find(client) != m_ClientsData.end());
 
 	// Clients must advance one turn at a time
-	if (turn != m_ClientsReady[client] + 1)
+	if (turn != m_ClientsData[client].readyTurn + 1)
 	{
 		LOGERROR("NotifyFinishedClientCommands: Client %d (%s) is ready for turn %d, but expected %d",
 			client,
 			utf8_from_wstring(session.GetUserName()).c_str(),
 			turn,
-			m_ClientsReady[client] + 1);
+			m_ClientsData[client].readyTurn + 1);
 
 		session.Disconnect(NDR_INCORRECT_READY_TURN_COMMANDS);
 	}
 
-	m_ClientsReady[client] = turn;
+	m_ClientsData[client].readyTurn = turn;
 
 	// Check whether this was the final client to become ready
 	CheckClientsReady();
@@ -80,13 +80,13 @@ void CNetServerTurnManager::CheckClientsReady()
 	max_observer_lag = max_observer_lag < 0 ? -1 : max_observer_lag > 10000 ? -1 : max_observer_lag;
 
 	// See if all clients (including self) are ready for a new turn
-	for (const std::pair<const int, u32>& clientReady : m_ClientsReady)
+	for (const std::pair<const int, Client>& clientData : m_ClientsData)
 	{
 		// Observers are allowed to lag more than regular clients.
-		if (m_ClientsObserver[clientReady.first] && (max_observer_lag == -1 || clientReady.second > m_ReadyTurn - max_observer_lag))
+		if (clientData.second.isObserver && (max_observer_lag == -1 || clientData.second.readyTurn > m_ReadyTurn - max_observer_lag))
 			continue;
 		NETSERVERTURN_LOG("  %d: %d <=? %d\n", clientReady.first, clientReady.second, m_ReadyTurn);
-		if (clientReady.second <= m_ReadyTurn)
+		if (clientData.second.readyTurn <= m_ReadyTurn)
 			return; // wasn't ready for m_ReadyTurn+1
 	}
 
@@ -111,31 +111,31 @@ void CNetServerTurnManager::NotifyFinishedClientUpdate(CNetServerSession& sessio
 	const CStrW& playername = session.GetUserName();
 
 	// Clients must advance one turn at a time
-	if (turn != m_ClientsSimulated[client] + 1)
+	if (turn != m_ClientsData[client].simulatedTurn + 1)
 	{
 		LOGERROR("NotifyFinishedClientUpdate: Client %d (%s) is ready for turn %d, but expected %d",
 			client,
 			utf8_from_wstring(playername).c_str(),
 			turn,
-			m_ClientsReady[client] + 1);
+			m_ClientsData[client].simulatedTurn + 1);
 
 		session.Disconnect(NDR_INCORRECT_READY_TURN_SIMULATED);
 	}
 
-	m_ClientsSimulated[client] = turn;
+	m_ClientsData[client].simulatedTurn = turn;
 
 	// Check for OOS only if in sync
 	if (m_HasSyncError)
 		return;
 
-	m_ClientPlayernames[client] = playername;
+	m_ClientsData[client].playerName = playername;
 	m_ClientStateHashes[turn][client] = hash;
 
 	// Find the newest turn which we know all clients have simulated
 	u32 newest = std::numeric_limits<u32>::max();
-	for (const std::pair<const int, u32>& clientSimulated : m_ClientsSimulated)
+	for (const std::pair<const int, Client>& clientData : m_ClientsData)
-		if (clientSimulated.second < newest)
+		if (clientData.second.simulatedTurn < newest)
-			newest = clientSimulated.second;
+			newest = clientData.second.simulatedTurn;
 
 	// For every set of state hashes that all clients have simulated, check for OOS
 	for (const std::pair<const u32, std::map<int, std::string>>& clientStateHash : m_ClientStateHashes)
@@ -155,7 +155,8 @@ void CNetServerTurnManager::NotifyFinishedClientUpdate(CNetServerSession& sessio
 			{
 				// Oh no, out of sync
 				m_HasSyncError = true;
-				OOSPlayerNames.push_back(m_ClientPlayernames[hashPair.first]);
+				m_ClientsData[hashPair.first].isOOS = true;
+				OOSPlayerNames.push_back(m_ClientsData[hashPair.first].playerName);
 			}
 		}
 
@@ -184,24 +185,33 @@ void CNetServerTurnManager::InitialiseClient(int client, u32 turn, bool observer
 {
 	NETSERVERTURN_LOG("InitialiseClient(client=%d, turn=%d)\n", client, turn);
 
-	ENSURE(m_ClientsReady.find(client) == m_ClientsReady.end());
+	ENSURE(m_ClientsData.find(client) == m_ClientsData.end());
-	m_ClientsReady[client] = turn + COMMAND_DELAY_MP - 1;
+	Client& data = m_ClientsData[client];
-	m_ClientsSimulated[client] = turn;
+	data.readyTurn = turn + COMMAND_DELAY_MP - 1;
-	m_ClientsObserver[client] = observer;
+	data.simulatedTurn = turn;
+	data.isObserver = observer;
 }
 
 void CNetServerTurnManager::UninitialiseClient(int client)
 {
 	NETSERVERTURN_LOG("UninitialiseClient(client=%d)\n", client);
 
-	ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end());
+	ENSURE(m_ClientsData.find(client) != m_ClientsData.end());
-	m_ClientsReady.erase(client);
+	bool checkOOS = m_ClientsData[client].isOOS;
-	m_ClientsSimulated.erase(client);
+	m_ClientsData.erase(client);
-	m_ClientsObserver.erase(client);
 
 	// Check whether we're ready for the next turn now that we're not
 	// waiting for this client any more
 	CheckClientsReady();
+
+	// Check whether we're still OOS.
+	if (checkOOS)
+	{
+		for (const std::pair<const int, Client>& clientData : m_ClientsData)
+			if (clientData.second.isOOS)
+				return;
+		m_HasSyncError = false;
+	}
 }
 
 void CNetServerTurnManager::SetTurnLength(u32 msecs)

source/network/NetServerTurnManager.h

@@ -72,24 +72,27 @@ public:
 private:
 	void CheckClientsReady();
 
-	/// The latest turn for which we have received all commands from all clients
+	struct Client
-	u32 m_ReadyTurn;
+	{
+		CStrW playerName;
+		// Latest turn for which all commands have been received.
+		u32 readyTurn;
+		// Last known simulated turn.
+		u32 simulatedTurn;
+		bool isObserver;
+		bool isOOS = false;
+	};
 
-	// Client ID -> whether they are only an observer.
+	std::unordered_map<int, Client> m_ClientsData;
-	std::unordered_map<int, bool> m_ClientsObserver;
 
-	// Client ID -> ready turn number (the latest turn for which all commands have been received from that client)
+	// Cached value - is any client OOS? This is reset when the OOS client leaves.
-	std::map<int, u32> m_ClientsReady;
+	bool m_HasSyncError = false;
-
-	// Client ID -> last known simulated turn number (for which we have the state hash)
-	// (the client has reached the start of this turn, not done the update for it yet)
-	std::map<int, u32> m_ClientsSimulated;
 
 	// Map of turn -> {Client ID -> state hash}; old indexes <= min(m_ClientsSimulated) are deleted
 	std::map<u32, std::map<int, std::string>> m_ClientStateHashes;
 
-	// Map of client ID -> playername
+	/// The latest turn for which we have received all commands from all clients
-	std::map<int, CStrW> m_ClientPlayernames;
+	u32 m_ReadyTurn;
 
 	// Current turn length
 	u32 m_TurnLength;
@@ -98,8 +101,6 @@ private:
 	std::vector<u32> m_SavedTurnLengths;
 
 	CNetServerWorker& m_NetServer;
-
-	bool m_HasSyncError;
 };
 
 #endif // INCLUDED_NETSERVERTURNMANAGER

source/simulation2/Simulation2.cpp

@@ -885,6 +885,7 @@ bool CSimulation2::ComputeStateHash(std::string& outHash, bool quick)
 
 bool CSimulation2::DumpDebugState(std::ostream& stream)
 {
+	stream << "sim turn: " << m->m_TurnNumber << std::endl;
 	return m->m_ComponentManager.DumpDebugState(stream, true);
 }
 

source/simulation2/system/ReplayTurnManager.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -44,6 +44,8 @@ private:
 	static const CStr EventNameReplayFinished;
 	static const CStr EventNameReplayOutOfSync;
 
+	bool m_HasSyncError = false;
+
 	// Contains the commands of every player on each turn
 	std::map<u32, std::vector<std::pair<player_id_t, std::string>>> m_ReplayCommands;
 

source/simulation2/system/TurnManager.cpp

@@ -40,7 +40,7 @@ const CStr CTurnManager::EventNameSavegameLoaded = "SavegameLoaded";
 
 CTurnManager::CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, u32 commandDelay, int clientId, IReplayLogger& replay)
 	: m_Simulation2(simulation), m_CurrentTurn(0), m_CommandDelay(commandDelay), m_ReadyTurn(commandDelay - 1), m_TurnLength(defaultTurnLength),
-	m_PlayerId(-1), m_ClientId(clientId), m_DeltaSimTime(0), m_HasSyncError(false), m_Replay(replay),
+	m_PlayerId(-1), m_ClientId(clientId), m_DeltaSimTime(0), m_Replay(replay),
 	m_FinalTurn(std::numeric_limits<u32>::max()), m_TimeWarpNumTurns(0),
 	m_QuickSaveMetadata(m_Simulation2.GetScriptInterface().GetGeneralJSContext())
 {

source/simulation2/system/TurnManager.h

@@ -209,8 +209,6 @@ protected:
 	/// add elapsed time increments to until we reach 0).
 	float m_DeltaSimTime;
 
-	bool m_HasSyncError;
-
 	IReplayLogger& m_Replay;
 
 	// The number of the last turn that is allowed to be executed (used for replays)