Netcode: allow observers to lag behind the live game.

Observers no longer lag the game for players. There is still some time
to serialise the game when sending it to a joining observer, and
depending on the chosen 'max lag' the game may stop while observers
sufficiently catch up, but this impact too is reduced.

- Make the NetServerTurnManager ignore players marked as 'observers' for
the purpose of ending a turn, effectively making it possible for
observers to lag without it affecting the players in any way.
- Add a config option (network.observermaxlag) that specifies how many
turns behind the live game observers are allowed to be. Default to 10
turns, or 2 seconds, to keep them 'largely live'.
- The controller is not treated as an observer.
- Implement a simple UI to show this delay & allow the game to speed up
automatically to try and catch up. This can be deactivated via
network.autocatchup.
- Move network options to the renamed 'Network / Lobby' options page.
- Do not debug_warn/crash when receiving commands from the past -
instead warn and carry on, to avoid DOS and "coop play" issues.

Refs #5903, Refs #4210

Differential Revision: https://code.wildfiregames.com/D3737
This was SVN commit r25156.
This commit is contained in:
wraitii 2021-03-29 07:53:06 +00:00
parent e3a254225a
commit 5ebf2020b0
12 changed files with 188 additions and 60 deletions

View File

@ -486,6 +486,8 @@ name_id = "0ad"
duplicateplayernames = false ; Rename joining player to "User (2)" if "User" is already connected, otherwise prohibit join.
lateobservers = everyone ; Allow observers to join the game after it started. Possible values: everyone, buddies, disabled.
observerlimit = 8 ; Prevent further observer joins in running games if this limit is reached
observermaxlag = 10 ; Make clients wait for observers if they lag more than X turns behind. -1 means "never wait for observers".
autocatchup = true ; Auto-accelerate the sim rate if lagging behind (as an observer).
[overlay]
fps = "false" ; Show frames per second in top right corner

View File

@ -27,12 +27,6 @@
"tooltip": "If you disable it, the welcome screen will still appear once, each time a new version is available. You can always launch it from the main menu.",
"config": "gui.splashscreen.enable"
},
{
"type": "boolean",
"label": "Network warnings",
"tooltip": "Show which player has a bad connection in multiplayer games.",
"config": "overlay.netwarnings"
},
{
"type": "boolean",
"label": "FPS overlay",
@ -57,25 +51,6 @@
"tooltip": "Always show the remaining ceasefire time.",
"config": "gui.session.ceasefirecounter"
},
{
"type": "dropdown",
"label": "Late observer joins",
"tooltip": "Allow everybody or buddies only to join the game as observer after it started.",
"config": "network.lateobservers",
"list": [
{ "value": "everyone", "label": "Everyone" },
{ "value": "buddies", "label": "Buddies" },
{ "value": "disabled", "label": "Disabled" }
]
},
{
"type": "number",
"label": "Observer limit",
"tooltip": "Prevent further observers from joining if the limit is reached.",
"config": "network.observerlimit",
"min": 0,
"max": 32
},
{
"type": "boolean",
"label": "Chat timestamp",
@ -425,7 +400,7 @@
]
},
{
"label": "Lobby",
"label": "Networking / Lobby",
"tooltip": "These settings only affect the multiplayer.",
"options":
[
@ -447,6 +422,45 @@
"label": "Game rating column",
"tooltip": "Show the average rating of the participating players in a column of the gamelist.",
"config": "lobby.columns.gamerating"
},
{
"type": "boolean",
"label": "Network warnings",
"tooltip": "Show which player has a bad connection in multiplayer games.",
"config": "overlay.netwarnings"
},
{
"type": "dropdown",
"label": "Late observer joins",
"tooltip": "Allow everybody or buddies only to join the game as observer after it started.",
"config": "network.lateobservers",
"list": [
{ "value": "everyone", "label": "Everyone" },
{ "value": "buddies", "label": "Buddies" },
{ "value": "disabled", "label": "Disabled" }
]
},
{
"type": "number",
"label": "Observer limit",
"tooltip": "Prevent further observers from joining if the limit is reached.",
"config": "network.observerlimit",
"min": 0,
"max": 32
},
{
"type": "number",
"label": "Max lag for observers",
"tooltip": "When hosting, pause the game if observers are lagging more than this many turns. If set to -1, observers are ignored.",
"config": "network.observermaxlag",
"min": -1,
"max": 10000
},
{
"type": "boolean",
"label": "(Observer) Speed up when lagging.",
"tooltip": "When observing a game, automatically speed up if you start lagging, to catch up with the live match.",
"config": "network.autocatchup"
}
]
},

View File

@ -0,0 +1,63 @@
/**
* Shows an overlay if the game is lagging behind the net server.
*/
class NetworkDelayOverlay
{
constructor()
{
this.netDelayOverlay = Engine.GetGUIObjectByName("netDelayOverlay");
this.netDelayOverlay.caption="toto";
this.caption = translate(this.Caption);
this.sprintfData = {};
this.initialSimRate = Engine.GetSimRate();
this.currentSimRate = this.initialSimRate;
setTimeout(() => this.CheckDelay(), 1000);
}
CheckDelay()
{
setTimeout(() => this.CheckDelay(), 1000);
let delay = +(Engine.HasNetClient() && Engine.GetPendingTurns());
if (g_IsObserver && Engine.ConfigDB_GetValue("user", "network.autocatchup"))
{
if (delay > this.MAX_LIVE_DELAY && this.currentSimRate <= this.initialSimRate)
{
this.currentSimRate = this.initialSimRate * 1.1;
Engine.SetSimRate(this.currentSimRate);
}
else if (delay <= this.NORMAL_DELAY && this.currentSimRate > this.initialSimRate)
{
this.currentSimRate = this.initialSimRate;
Engine.SetSimRate(this.currentSimRate);
}
}
if (delay < this.MAX_LIVE_DELAY)
{
this.netDelayOverlay.hidden = true;
return;
}
this.netDelayOverlay.hidden = false;
this.sprintfData.delay = (delay / this.TURNS_PER_SECOND);
this.sprintfData.delay = this.sprintfData.delay.toFixed(this.sprintfData.delay < 5 ? 1 : 0);
this.netDelayOverlay.caption = sprintf(this.caption, this.sprintfData);
}
}
/**
* Because of command delay, we can still be several turns behind the 'ready' turn and not
* particularly late. This should be kept in sync with the command delay.
*/
NetworkDelayOverlay.prototype.NORMAL_DELAY = 3;
NetworkDelayOverlay.prototype.MAX_LIVE_DELAY = 6;
/**
* This needs to be kept in sync with the turn length.
*/
NetworkDelayOverlay.prototype.TURNS_PER_SECOND = 5;
NetworkDelayOverlay.prototype.Caption = translate("Delay to live stream: %(delay)ss");

View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Put this under the regular gamestateNotifications. -->
<object name="netDelayOverlay"
type="text"
hidden="false"
ghost="true"
z="199"
size="100%-300 60 100%-110 80"
font="mono-10"
textcolor="white"
text_align="right"
text_valign="top"
sprite="color: 0 0 0 100"
/>

View File

@ -20,6 +20,7 @@ var g_GameSpeedControl;
var g_Menu;
var g_MiniMapPanel;
var g_NetworkStatusOverlay;
var g_NetworkDelayOverlay;
var g_ObjectivesDialog;
var g_OutOfSyncNetwork;
var g_OutOfSyncReplay;
@ -289,6 +290,7 @@ function init(initData, hotloadData)
g_Menu = new Menu(g_PauseControl, g_PlayerViewControl, g_Chat);
g_MiniMapPanel = new MiniMapPanel(g_PlayerViewControl, g_DiplomacyColors, g_WorkerTypes);
g_NetworkStatusOverlay = new NetworkStatusOverlay();
g_NetworkDelayOverlay = new NetworkDelayOverlay();
g_ObjectivesDialog = new ObjectivesDialog(g_PlayerViewControl, mapCache);
g_OutOfSyncNetwork = new OutOfSyncNetwork();
g_OutOfSyncReplay = new OutOfSyncReplay();

View File

@ -38,6 +38,7 @@
<include directory="gui/session/hotkeys/"/>
<include file="gui/session/NetworkStatusOverlay.xml"/>
<include file="gui/session/NetworkDelayOverlay.xml"/>
<include file="gui/session/PauseOverlay.xml"/>
<include file="gui/session/TimeNotificationOverlay.xml"/>

View File

@ -747,7 +747,7 @@ void CNetServerWorker::OnUserLeave(CNetServerSession* session)
RemovePlayer(session->GetGUID());
if (m_ServerTurnManager && session->GetCurrState() != NSS_JOIN_SYNCING)
m_ServerTurnManager->UninitialiseClient(session->GetHostID()); // TODO: only for non-observers
m_ServerTurnManager->UninitialiseClient(session->GetHostID());
// TODO: ought to switch the player controlled by that client
// back to AI control, or something?
@ -1402,7 +1402,9 @@ bool CNetServerWorker::OnJoinSyncingLoadedGame(void* context, CFsmEvent* event)
}
// Tell the turn manager to expect commands from this new client
server.m_ServerTurnManager->InitialiseClient(session->GetHostID(), readyTurn);
// Special case: the controller shouldn't be treated as an observer in any case.
bool isObserver = server.m_PlayerAssignments[session->GetGUID()].m_PlayerID == -1 && server.m_ControllerGUID != session->GetGUID();
server.m_ServerTurnManager->InitialiseClient(session->GetHostID(), readyTurn, isObserver);
// Tell the client that everything has finished loading and it should start now
CLoadedGameMessage loaded;
@ -1530,7 +1532,11 @@ void CNetServerWorker::StartGame(const CStr& initAttribs)
m_ServerTurnManager = new CNetServerTurnManager(*this);
for (CNetServerSession* session : m_Sessions)
m_ServerTurnManager->InitialiseClient(session->GetHostID(), 0); // TODO: only for non-observers
{
// Special case: the controller shouldn't be treated as an observer in any case.
bool isObserver = m_PlayerAssignments[session->GetGUID()].m_PlayerID == -1 && m_ControllerGUID != session->GetGUID();
m_ServerTurnManager->InitialiseClient(session->GetHostID(), 0, isObserver);
}
m_State = SERVER_STATE_LOADING;

View File

@ -24,6 +24,7 @@
#include "lib/utf8.h"
#include "ps/CLogger.h"
#include "ps/ConfigDB.h"
#include "simulation2/system/TurnManager.h"
#if 0
@ -73,9 +74,17 @@ void CNetServerTurnManager::NotifyFinishedClientCommands(CNetServerSession& sess
void CNetServerTurnManager::CheckClientsReady()
{
int max_observer_lag = -1;
CFG_GET_VAL("network.observermaxlag", max_observer_lag);
// Clamp to 0-10000 turns, below/above that is no limit.
max_observer_lag = max_observer_lag < 0 ? -1 : max_observer_lag > 10000 ? -1 : max_observer_lag;
// See if all clients (including self) are ready for a new turn
for (const std::pair<const int, u32>& clientReady : m_ClientsReady)
{
// Observers are allowed to lag more than regular clients.
if (m_ClientsObserver[clientReady.first] && (max_observer_lag == -1 || clientReady.second > m_ReadyTurn - max_observer_lag))
continue;
NETSERVERTURN_LOG(" %d: %d <=? %d\n", clientReady.first, clientReady.second, m_ReadyTurn);
if (clientReady.second <= m_ReadyTurn)
return; // wasn't ready for m_ReadyTurn+1
@ -171,13 +180,14 @@ void CNetServerTurnManager::NotifyFinishedClientUpdate(CNetServerSession& sessio
m_ClientStateHashes.erase(m_ClientStateHashes.begin(), m_ClientStateHashes.lower_bound(newest+1));
}
void CNetServerTurnManager::InitialiseClient(int client, u32 turn)
void CNetServerTurnManager::InitialiseClient(int client, u32 turn, bool observer)
{
NETSERVERTURN_LOG("InitialiseClient(client=%d, turn=%d)\n", client, turn);
ENSURE(m_ClientsReady.find(client) == m_ClientsReady.end());
m_ClientsReady[client] = turn + COMMAND_DELAY_MP - 1;
m_ClientsSimulated[client] = turn;
m_ClientsObserver[client] = observer;
}
void CNetServerTurnManager::UninitialiseClient(int client)
@ -187,6 +197,7 @@ void CNetServerTurnManager::UninitialiseClient(int client)
ENSURE(m_ClientsReady.find(client) != m_ClientsReady.end());
m_ClientsReady.erase(client);
m_ClientsSimulated.erase(client);
m_ClientsObserver.erase(client);
// Check whether we're ready for the next turn now that we're not
// waiting for this client any more

View File

@ -1,4 +1,4 @@
/* Copyright (C) 2018 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -18,16 +18,18 @@
#ifndef INCLUDED_NETSERVERTURNMANAGER
#define INCLUDED_NETSERVERTURNMANAGER
#include <map>
#include "ps/CStr.h"
#include <map>
#include <unordered_map>
class CNetServerWorker;
class CNetServerSession;
/**
* The server-side counterpart to CNetClientTurnManager.
* Records the turn state of each client, and sends turn advancement messages
* when all clients are ready.
* when clients are ready.
*
* Thread-safety:
* - This is constructed and used by CNetServerWorker in the network server thread.
@ -43,13 +45,13 @@ public:
void NotifyFinishedClientUpdate(CNetServerSession& session, u32 turn, const CStr& hash);
/**
* Inform the turn manager of a new client who will be sending commands.
* Inform the turn manager of a new client
* @param observer - whether this client is an observer.
*/
void InitialiseClient(int client, u32 turn);
void InitialiseClient(int client, u32 turn, bool observer);
/**
* Inform the turn manager that a previously-initialised client has left the game
* and will no longer be sending commands.
* Inform the turn manager that a previously-initialised client has left the game.
*/
void UninitialiseClient(int client);
@ -73,6 +75,9 @@ private:
/// The latest turn for which we have received all commands from all clients
u32 m_ReadyTurn;
// Client ID -> whether they are only an observer.
std::unordered_map<int, bool> m_ClientsObserver;
// Client ID -> ready turn number (the latest turn for which all commands have been received from that client)
std::map<int, u32> m_ClientsReady;
@ -84,7 +89,7 @@ private:
std::map<u32, std::map<int, std::string>> m_ClientStateHashes;
// Map of client ID -> playername
std::map<u32, CStrW> m_ClientPlayernames;
std::map<int, CStrW> m_ClientPlayernames;
// Current turn length
u32 m_TurnLength;

View File

@ -98,6 +98,17 @@ void SetSimRate(float rate)
g_Game->SetSimRate(rate);
}
int GetPendingTurns(const ScriptRequest& rq)
{
if (!g_Game || !g_Game->GetTurnManager())
{
ScriptException::Raise(rq, "Game is not started");
return 0;
}
return g_Game->GetTurnManager()->GetPendingTurns();
}
bool IsPaused(const ScriptRequest& rq)
{
if (!g_Game)
@ -179,6 +190,7 @@ void RegisterScriptFunctions(const ScriptRequest& rq)
ScriptFunction::Register<&SetViewedPlayer>(rq, "SetViewedPlayer");
ScriptFunction::Register<&GetSimRate>(rq, "GetSimRate");
ScriptFunction::Register<&SetSimRate>(rq, "SetSimRate");
ScriptFunction::Register<&GetPendingTurns>(rq, "GetPendingTurns");
ScriptFunction::Register<&IsPaused>(rq, "IsPaused");
ScriptFunction::Register<&SetPaused>(rq, "SetPaused");
ScriptFunction::Register<&IsVisualReplay>(rq, "IsVisualReplay");

View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -36,20 +36,6 @@
#define NETTURN_LOG(...)
#endif
/**
* Maximum number of turns between two clients.
* When we are on turn n, we schedule new commands for n+COMMAND_DELAY.
* We know that all other clients have finished scheduling commands for n,
* else we couldn't have got here, which means they're at least on turn n-COMMAND_DELAY+1.
* We know we have not yet finished scheduling commands for n+COMMAND_DELAY, so no client can be there.
* Hence other clients can be on turns [n-COMMAND_DELAY+1, ..., n+COMMAND_DELAY-1], and no other,
* hence any two clients can only be this many turns apart.
*/
constexpr int MaxClientTurnDelta(int commandDelay)
{
return 2 * (commandDelay - 1);
}
const CStr CTurnManager::EventNameSavegameLoaded = "SavegameLoaded";
CTurnManager::CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, u32 commandDelay, int clientId, IReplayLogger& replay)
@ -58,8 +44,7 @@ CTurnManager::CTurnManager(CSimulation2& simulation, u32 defaultTurnLength, u32
m_FinalTurn(std::numeric_limits<u32>::max()), m_TimeWarpNumTurns(0),
m_QuickSaveMetadata(m_Simulation2.GetScriptInterface().GetGeneralJSContext())
{
// Lag between any two clients is bounded. Add 1 for inclusive bounds.
m_QueuedCommands.resize(MaxClientTurnDelta(m_CommandDelay) + 1);
m_QueuedCommands.resize(1);
}
void CTurnManager::ResetState(u32 newCurrentTurn, u32 newReadyTurn)
@ -237,16 +222,23 @@ void CTurnManager::AddCommand(int client, int player, JS::HandleValue data, u32
{
NETTURN_LOG("AddCommand(client=%d player=%d turn=%d current=%d, ready=%d)\n", client, player, turn, m_CurrentTurn, m_ReadyTurn);
// Reject commands for turns that we should not be able to compute (in the past or too far future).
if (m_CurrentTurn >= turn || turn > m_CurrentTurn + MaxClientTurnDelta(m_CommandDelay) + 1)
// Reject commands for turns that we should not be able to compute (in the past).
if (m_CurrentTurn >= turn)
{
debug_warn(L"Received command for invalid turn");
// The most likely explanation is that an observer that's lagging behind is sending commands,
// which is possible when cheats are enabled. Report & ignore.
// It seems a bad idea to error out too badly here:
// nefarious clients could try and send broken commands to DOS.
LOGWARNING("Received command for invalid turn %i (current turn is %i)", turn, m_CurrentTurn);
return;
}
m_Simulation2.GetScriptInterface().FreezeObject(data, true);
ScriptRequest rq(m_Simulation2.GetScriptInterface());
size_t command_in_turns = turn - (m_CurrentTurn+1);
if (m_QueuedCommands.size() <= command_in_turns)
m_QueuedCommands.resize(command_in_turns+1);
m_QueuedCommands[turn - (m_CurrentTurn+1)][client].emplace_back(player, rq.cx, data);
}

View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 Wildfire Games.
/* Copyright (C) 2021 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
@ -155,7 +155,13 @@ public:
void QuickSave(JS::HandleValue GUIMetadata);
void QuickLoad();
u32 GetCurrentTurn() { return m_CurrentTurn; }
u32 GetCurrentTurn() const { return m_CurrentTurn; }
/**
* @return how many turns are ready to be computed.
* (used to detect players/observers that fall behind the live game.
*/
u32 GetPendingTurns() const { return m_ReadyTurn - m_CurrentTurn; }
protected:
/**