/* Copyright (C) 2016 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 2 of the License, or * (at your option) any later version. * * 0 A.D. is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with 0 A.D. If not, see . */ #include "precompiled.h" #include "NetTurnManager.h" #include "NetMessage.h" #include "network/NetServer.h" #include "network/NetClient.h" #include "network/NetMessage.h" #include "gui/GUIManager.h" #include "maths/MathUtil.h" #include "ps/CLogger.h" #include "ps/Profile.h" #include "ps/Pyrogenesis.h" #include "ps/Replay.h" #include "ps/SavedGame.h" #include "scriptinterface/ScriptInterface.h" #include "simulation2/Simulation2.h" #include #include #include const u32 DEFAULT_TURN_LENGTH_MP = 500; const u32 DEFAULT_TURN_LENGTH_SP = 200; static const int COMMAND_DELAY = 2; #if 0 #define NETTURN_LOG(args) debug_printf args #else #define NETTURN_LOG(args) #endif static std::string Hexify(const std::string& s) { std::stringstream str; str << std::hex; for (size_t i = 0; i < s.size(); ++i) str << std::setfill('0') << std::setw(2) << (int)(unsigned char)s[i]; return str.str(); } CNetTurnManager::CNetTurnManager(CSimulation2& simulation, u32 defaultTurnLength, int clientId, IReplayLogger& replay) : m_Simulation2(simulation), m_CurrentTurn(0), m_ReadyTurn(1), m_TurnLength(defaultTurnLength), m_DeltaSimTime(0), m_PlayerId(-1), m_ClientId(clientId), m_HasSyncError(false), m_Replay(replay), m_TimeWarpNumTurns(0), m_FinalTurn(std::numeric_limits::max()) { // When we are on turn n, we schedule new commands for n+2. // We know that all other clients have finished scheduling commands for n (else we couldn't have got here). // We know we have not yet finished scheduling commands for n+2. // Hence other clients can be on turn n-1, n, n+1, and no other. // So they can be sending us commands scheduled for n+1, n+2, n+3. // So we need a 3-element buffer: m_QueuedCommands.resize(COMMAND_DELAY + 1); } void CNetTurnManager::ResetState(u32 newCurrentTurn, u32 newReadyTurn) { m_CurrentTurn = newCurrentTurn; m_ReadyTurn = newReadyTurn; m_DeltaSimTime = 0; size_t queuedCommandsSize = m_QueuedCommands.size(); m_QueuedCommands.clear(); m_QueuedCommands.resize(queuedCommandsSize); } void CNetTurnManager::SetPlayerID(int playerId) { m_PlayerId = playerId; } bool CNetTurnManager::WillUpdate(float simFrameLength) { // Keep this in sync with the return value of Update() if (m_CurrentTurn > m_FinalTurn) return false; if (m_DeltaSimTime + simFrameLength < 0) return false; if (m_ReadyTurn <= m_CurrentTurn) return false; return true; } bool CNetTurnManager::Update(float simFrameLength, size_t maxTurns) { if (m_CurrentTurn > m_FinalTurn) return false; m_DeltaSimTime += simFrameLength; // If the game becomes laggy, m_DeltaSimTime increases progressively. // The engine will fast forward accordingly to catch up. // To keep the game playable, stop fast forwarding after 2 turn lengths. m_DeltaSimTime = std::min(m_DeltaSimTime, 2.0f * m_TurnLength / 1000.0f); // If we haven't reached the next turn yet, do nothing if (m_DeltaSimTime < 0) return false; NETTURN_LOG((L"Update current=%d ready=%d\n", m_CurrentTurn, m_ReadyTurn)); // Check that the next turn is ready for execution if (m_ReadyTurn <= m_CurrentTurn) { // Oops, we wanted to start the next turn but it's not ready yet - // there must be too much network lag. // TODO: complain to the user. // TODO: send feedback to the server to increase the turn length. // Reset the next-turn timer to 0 so we try again next update but // so we don't rush to catch up in subsequent turns. // TODO: we should do clever rate adjustment instead of just pausing like this. m_DeltaSimTime = 0; return false; } maxTurns = std::max((size_t)1, maxTurns); // always do at least one turn for (size_t i = 0; i < maxTurns; ++i) { // Check that we've reached the i'th next turn if (m_DeltaSimTime < 0) break; // Check that the i'th next turn is still ready if (m_ReadyTurn <= m_CurrentTurn) break; NotifyFinishedOwnCommands(m_CurrentTurn + COMMAND_DELAY); // Increase now, so Update can send new commands for a subsequent turn ++m_CurrentTurn; // Clean up any destroyed entities since the last turn (e.g. placement previews // or rally point flags generated by the GUI). (Must do this before the time warp // serialization.) m_Simulation2.FlushDestroyedEntities(); // Save the current state for rewinding, if enabled if (m_TimeWarpNumTurns && (m_CurrentTurn % m_TimeWarpNumTurns) == 0) { PROFILE3("time warp serialization"); std::stringstream stream; m_Simulation2.SerializeState(stream); m_TimeWarpStates.push_back(stream.str()); } // Put all the client commands into a single list, in a globally consistent order std::vector commands; for (std::map >::iterator it = m_QueuedCommands[0].begin(); it != m_QueuedCommands[0].end(); ++it) { commands.insert(commands.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); } m_QueuedCommands.pop_front(); m_QueuedCommands.resize(m_QueuedCommands.size() + 1); m_Replay.Turn(m_CurrentTurn-1, m_TurnLength, commands); NETTURN_LOG((L"Running %d cmds\n", commands.size())); m_Simulation2.Update(m_TurnLength, commands); NotifyFinishedUpdate(m_CurrentTurn); // Set the time for the next turn update m_DeltaSimTime -= m_TurnLength / 1000.f; } return true; } bool CNetTurnManager::UpdateFastForward() { m_DeltaSimTime = 0; NETTURN_LOG((L"UpdateFastForward current=%d ready=%d\n", m_CurrentTurn, m_ReadyTurn)); // Check that the next turn is ready for execution if (m_ReadyTurn <= m_CurrentTurn) return false; while (m_ReadyTurn > m_CurrentTurn) { // TODO: It would be nice to remove some of the duplication with Update() // (This is similar but doesn't call any Notify functions or update DeltaTime, // it just updates the simulation state) ++m_CurrentTurn; m_Simulation2.FlushDestroyedEntities(); // Put all the client commands into a single list, in a globally consistent order std::vector commands; for (std::map >::iterator it = m_QueuedCommands[0].begin(); it != m_QueuedCommands[0].end(); ++it) { commands.insert(commands.end(), std::make_move_iterator(it->second.begin()), std::make_move_iterator(it->second.end())); } m_QueuedCommands.pop_front(); m_QueuedCommands.resize(m_QueuedCommands.size() + 1); m_Replay.Turn(m_CurrentTurn-1, m_TurnLength, commands); NETTURN_LOG((L"Running %d cmds\n", commands.size())); m_Simulation2.Update(m_TurnLength, commands); } return true; } void CNetTurnManager::OnSyncError(u32 turn, const CStr& expectedHash, std::vector& playerNames) { NETTURN_LOG((L"OnSyncError(%d, %hs)\n", turn, Hexify(expectedHash).c_str())); // Only complain the first time if (m_HasSyncError) return; bool quick = !TurnNeedsFullHash(turn); std::string hash; ENSURE(m_Simulation2.ComputeStateHash(hash, quick)); OsPath path = psLogDir() / "oos_dump.txt"; std::ofstream file (OsString(path).c_str(), std::ofstream::out | std::ofstream::trunc); m_Simulation2.DumpDebugState(file); file.close(); hash = Hexify(hash); const std::string& expectedHashHex = Hexify(expectedHash); DisplayOOSError(turn, hash, expectedHashHex, false, &playerNames, &path); } void CNetTurnManager::DisplayOOSError(u32 turn, const CStr& hash, const CStr& expectedHash, bool isReplay, std::vector* playerNames = NULL, OsPath* path = NULL) { m_HasSyncError = true; std::stringstream msg; msg << "Out of sync on turn " << turn; if (playerNames) for (size_t i = 0; i < playerNames->size(); ++i) msg << (i == 0 ? "\nPlayers: " : ", ") << utf8_from_wstring((*playerNames)[i].m_Name); if (isReplay) msg << "\n\n" << "The current game state is different from the original game state."; else msg << "\n\n" << "Your game state is " << (expectedHash == hash ? "identical to" : "different from") << " the hosts game state."; if (path) msg << "\n\n" << "Dumping current state to " << CStr(path->string8()).EscapeToPrintableASCII(); LOGERROR("%s", msg.str()); if (g_GUI) g_GUI->DisplayMessageBox(600, 350, L"Sync error", wstring_from_utf8(msg.str())); } void CNetTurnManager::Interpolate(float simFrameLength, float realFrameLength) { // TODO: using m_TurnLength might be a bit dodgy when length changes - maybe // we need to save the previous turn length? float offset = clamp(m_DeltaSimTime / (m_TurnLength / 1000.f) + 1.0, 0.0, 1.0); // Stop animations while still updating the selection highlight if (m_CurrentTurn > m_FinalTurn) simFrameLength = 0; m_Simulation2.Interpolate(simFrameLength, offset, realFrameLength); } void CNetTurnManager::AddCommand(int client, int player, JS::HandleValue data, u32 turn) { NETTURN_LOG((L"AddCommand(client=%d player=%d turn=%d)\n", client, player, turn)); if (!(m_CurrentTurn < turn && turn <= m_CurrentTurn + COMMAND_DELAY + 1)) { debug_warn(L"Received command for invalid turn"); return; } m_Simulation2.GetScriptInterface().FreezeObject(data, true); m_QueuedCommands[turn - (m_CurrentTurn+1)][client].emplace_back(player, m_Simulation2.GetScriptInterface().GetContext(), data); } void CNetTurnManager::FinishedAllCommands(u32 turn, u32 turnLength) { NETTURN_LOG((L"FinishedAllCommands(%d, %d)\n", turn, turnLength)); ENSURE(turn == m_ReadyTurn + 1); m_ReadyTurn = turn; m_TurnLength = turnLength; } bool CNetTurnManager::TurnNeedsFullHash(u32 turn) { // Check immediately for errors caused by e.g. inconsistent game versions // (The hash is computed after the first sim update, so we start at turn == 1) if (turn == 1) return true; // Otherwise check the full state every ~10 seconds in multiplayer games // (TODO: should probably remove this when we're reasonably sure the game // isn't too buggy, since the full hash is still pretty slow) if (turn % 20 == 0) return true; return false; } void CNetTurnManager::EnableTimeWarpRecording(size_t numTurns) { m_TimeWarpStates.clear(); m_TimeWarpNumTurns = numTurns; } void CNetTurnManager::RewindTimeWarp() { if (m_TimeWarpStates.empty()) return; std::stringstream stream(m_TimeWarpStates.back()); m_Simulation2.DeserializeState(stream); m_TimeWarpStates.pop_back(); // Reset the turn manager state, so we won't execute stray commands and // won't do the next snapshot until the appropriate time. // (Ideally we ought to serialise the turn manager state and restore it // here, but this is simpler for now.) ResetState(0, 1); } void CNetTurnManager::QuickSave() { TIMER(L"QuickSave"); std::stringstream stream; if (!m_Simulation2.SerializeState(stream)) { LOGERROR("Failed to quicksave game"); return; } m_QuickSaveState = stream.str(); if (g_GUI) m_QuickSaveMetadata = g_GUI->GetSavedGameData(); else m_QuickSaveMetadata = std::string(); LOGMESSAGERENDER("Quicksaved game"); } void CNetTurnManager::QuickLoad() { TIMER(L"QuickLoad"); if (m_QuickSaveState.empty()) { LOGERROR("Cannot quickload game - no game was quicksaved"); return; } std::stringstream stream(m_QuickSaveState); if (!m_Simulation2.DeserializeState(stream)) { LOGERROR("Failed to quickload game"); return; } if (g_GUI && !m_QuickSaveMetadata.empty()) g_GUI->RestoreSavedGameData(m_QuickSaveMetadata); LOGMESSAGERENDER("Quickloaded game"); // See RewindTimeWarp ResetState(0, 1); } CNetClientTurnManager::CNetClientTurnManager(CSimulation2& simulation, CNetClient& client, int clientId, IReplayLogger& replay) : CNetTurnManager(simulation, DEFAULT_TURN_LENGTH_MP, clientId, replay), m_NetClient(client) { } void CNetClientTurnManager::PostCommand(JS::HandleValue data) { NETTURN_LOG((L"PostCommand()\n")); // Transmit command to server CSimulationMessage msg(m_Simulation2.GetScriptInterface(), m_ClientId, m_PlayerId, m_CurrentTurn + COMMAND_DELAY, data); m_NetClient.SendMessage(&msg); // Add to our local queue //AddCommand(m_ClientId, m_PlayerId, data, m_CurrentTurn + COMMAND_DELAY); // TODO: we should do this when the server stops sending our commands back to us } void CNetClientTurnManager::NotifyFinishedOwnCommands(u32 turn) { NETTURN_LOG((L"NotifyFinishedOwnCommands(%d)\n", turn)); // Send message to the server CEndCommandBatchMessage msg; msg.m_TurnLength = DEFAULT_TURN_LENGTH_MP; // TODO: why do we send this? msg.m_Turn = turn; m_NetClient.SendMessage(&msg); } void CNetClientTurnManager::NotifyFinishedUpdate(u32 turn) { bool quick = !TurnNeedsFullHash(turn); std::string hash; { PROFILE3("state hash check"); ENSURE(m_Simulation2.ComputeStateHash(hash, quick)); } NETTURN_LOG((L"NotifyFinishedUpdate(%d, %hs)\n", turn, Hexify(hash).c_str())); m_Replay.Hash(hash, quick); // Don't send the hash if OOS if (m_HasSyncError) return; // Send message to the server CSyncCheckMessage msg; msg.m_Turn = turn; msg.m_Hash = hash; m_NetClient.SendMessage(&msg); } void CNetClientTurnManager::OnDestroyConnection() { NotifyFinishedOwnCommands(m_CurrentTurn + COMMAND_DELAY); } void CNetClientTurnManager::OnSimulationMessage(CSimulationMessage* msg) { // Command received from the server - store it for later execution AddCommand(msg->m_Client, msg->m_Player, msg->m_Data, msg->m_Turn); } CNetLocalTurnManager::CNetLocalTurnManager(CSimulation2& simulation, IReplayLogger& replay) : CNetTurnManager(simulation, DEFAULT_TURN_LENGTH_SP, 0, replay) { } void CNetLocalTurnManager::PostCommand(JS::HandleValue data) { // Add directly to the next turn, ignoring COMMAND_DELAY, // because we don't need to compensate for network latency AddCommand(m_ClientId, m_PlayerId, data, m_CurrentTurn + 1); } void CNetLocalTurnManager::NotifyFinishedOwnCommands(u32 turn) { FinishedAllCommands(turn, m_TurnLength); } void CNetLocalTurnManager::NotifyFinishedUpdate(u32 UNUSED(turn)) { #if 0 // this hurts performance and is only useful for verifying log replays std::string hash; { PROFILE3("state hash check"); ENSURE(m_Simulation2.ComputeStateHash(hash)); } m_Replay.Hash(hash); #endif } void CNetLocalTurnManager::OnSimulationMessage(CSimulationMessage* UNUSED(msg)) { debug_warn(L"This should never be called"); } CNetReplayTurnManager::CNetReplayTurnManager(CSimulation2& simulation, IReplayLogger& replay) : CNetLocalTurnManager(simulation, replay) { } void CNetReplayTurnManager::StoreReplayCommand(u32 turn, int player, const std::string& command) { // Using the pair we make sure that commands per turn will be processed in the correct order m_ReplayCommands[turn].emplace_back(player, command); } void CNetReplayTurnManager::StoreReplayHash(u32 turn, const std::string& hash, bool quick) { m_ReplayHash[turn] = std::make_pair(hash, quick); } void CNetReplayTurnManager::StoreReplayTurnLength(u32 turn, u32 turnLength) { m_ReplayTurnLengths[turn] = turnLength; // Initialize turn length if (turn == 0) m_TurnLength = m_ReplayTurnLengths[0]; } void CNetReplayTurnManager::StoreFinalReplayTurn(u32 turn) { m_FinalTurn = turn; } void CNetReplayTurnManager::NotifyFinishedUpdate(u32 turn) { if (turn == 1 && m_FinalTurn == 0) g_GUI->SendEventToAll("ReplayFinished"); if (turn > m_FinalTurn) return; DoTurn(turn); // Compare hash if it exists in the replay and if we didn't have an OOS already if (m_HasSyncError || m_ReplayHash.find(turn) == m_ReplayHash.end()) return; std::string expectedHash = m_ReplayHash[turn].first; bool quickHash = m_ReplayHash[turn].second; // Compute hash std::string hash; ENSURE(m_Simulation2.ComputeStateHash(hash, quickHash)); hash = Hexify(hash); if (hash != expectedHash) DisplayOOSError(turn, hash, expectedHash, true); } void CNetReplayTurnManager::DoTurn(u32 turn) { debug_printf("Executing turn %u of %u\n", turn, m_FinalTurn); m_TurnLength = m_ReplayTurnLengths[turn]; // Simulate commands for that turn for (const std::pair& pair : m_ReplayCommands[turn]) { JS::RootedValue command(m_Simulation2.GetScriptInterface().GetContext()); m_Simulation2.GetScriptInterface().ParseJSON(pair.second, &command); AddCommand(m_ClientId, pair.first, command, m_CurrentTurn + 1); } if (turn == m_FinalTurn) g_GUI->SendEventToAll("ReplayFinished"); } CNetServerTurnManager::CNetServerTurnManager(CNetServerWorker& server) : m_NetServer(server), m_ReadyTurn(1), m_TurnLength(DEFAULT_TURN_LENGTH_MP), m_HasSyncError(false) { // The first turn we will actually execute is number 2, // so store dummy values into the saved lengths list m_SavedTurnLengths.push_back(0); m_SavedTurnLengths.push_back(0); } void CNetServerTurnManager::NotifyFinishedClientCommands(int client, u32 turn) { NETTURN_LOG((L"NotifyFinishedClientCommands(client=%d, turn=%d)\n", client, turn)); // Must be a client we've already heard of ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end()); // Clients must advance one turn at a time ENSURE(turn == m_ClientsReady[client] + 1); m_ClientsReady[client] = turn; // Check whether this was the final client to become ready CheckClientsReady(); } void CNetServerTurnManager::CheckClientsReady() { // See if all clients (including self) are ready for a new turn for (std::map::iterator it = m_ClientsReady.begin(); it != m_ClientsReady.end(); ++it) { NETTURN_LOG((L" %d: %d <=? %d\n", it->first, it->second, m_ReadyTurn)); if (it->second <= m_ReadyTurn) return; // wasn't ready for m_ReadyTurn+1 } // Advance the turn ++m_ReadyTurn; NETTURN_LOG((L"CheckClientsReady: ready for turn %d\n", m_ReadyTurn)); // Tell all clients that the next turn is ready CEndCommandBatchMessage msg; msg.m_TurnLength = m_TurnLength; msg.m_Turn = m_ReadyTurn; m_NetServer.Broadcast(&msg); // Save the turn length in case it's needed later ENSURE(m_SavedTurnLengths.size() == m_ReadyTurn); m_SavedTurnLengths.push_back(m_TurnLength); } void CNetServerTurnManager::NotifyFinishedClientUpdate(int client, const CStrW& playername, u32 turn, const CStr& hash) { // Clients must advance one turn at a time ENSURE(turn == m_ClientsSimulated[client] + 1); m_ClientsSimulated[client] = turn; // Check for OOS only if in sync if (m_HasSyncError) return; m_ClientPlayernames[client] = playername; m_ClientStateHashes[turn][client] = hash; // Find the newest turn which we know all clients have simulated u32 newest = std::numeric_limits::max(); for (std::map::iterator it = m_ClientsSimulated.begin(); it != m_ClientsSimulated.end(); ++it) { if (it->second < newest) newest = it->second; } // For every set of state hashes that all clients have simulated, check for OOS for (std::map >::iterator it = m_ClientStateHashes.begin(); it != m_ClientStateHashes.end(); ++it) { if (it->first > newest) break; // Assume the host is correct (maybe we should choose the most common instead to help debugging) std::string expected = it->second.begin()->second; // Find all players that are OOS on that turn std::vector OOSPlayerNames; for (std::map::iterator cit = it->second.begin(); cit != it->second.end(); ++cit) { NETTURN_LOG((L"sync check %d: %d = %hs\n", it->first, cit->first, Hexify(cit->second).c_str())); if (cit->second != expected) { // Oh no, out of sync m_HasSyncError = true; OOSPlayerNames.push_back(m_ClientPlayernames[cit->first]); } } // Tell everyone about it if (m_HasSyncError) { CSyncErrorMessage msg; msg.m_Turn = it->first; msg.m_HashExpected = expected; for (const CStrW& playername : OOSPlayerNames) { CSyncErrorMessage::S_m_PlayerNames h; h.m_Name = playername; msg.m_PlayerNames.push_back(h); } m_NetServer.Broadcast(&msg); break; } } // Delete the saved hashes for all turns that we've already verified m_ClientStateHashes.erase(m_ClientStateHashes.begin(), m_ClientStateHashes.lower_bound(newest+1)); } void CNetServerTurnManager::InitialiseClient(int client, u32 turn) { NETTURN_LOG((L"InitialiseClient(client=%d, turn=%d)\n", client, turn)); ENSURE(m_ClientsReady.find(client) == m_ClientsReady.end()); m_ClientsReady[client] = turn + 1; m_ClientsSimulated[client] = turn; } void CNetServerTurnManager::UninitialiseClient(int client) { NETTURN_LOG((L"UninitialiseClient(client=%d)\n", client)); ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end()); m_ClientsReady.erase(client); m_ClientsSimulated.erase(client); // Check whether we're ready for the next turn now that we're not // waiting for this client any more CheckClientsReady(); } void CNetServerTurnManager::SetTurnLength(u32 msecs) { m_TurnLength = msecs; } u32 CNetServerTurnManager::GetSavedTurnLength(u32 turn) { ENSURE(turn <= m_ReadyTurn); return m_SavedTurnLengths.at(turn); }