Rate limit handling for Dropbox (#6416)

* Added rate limit handling to dropbox

+ some refactor to make the whole file one object

* Added error messages on rate limit reached

+ refactored some popup code to reduce repetition

* Fixed merge error

* Made variables private

* Fixed file upload not working

because of missing override flag for dropbox

* Stop multiplayer refresher if rate limit reached

* Fixed typo

* Various code changes/fixes

- ErrorResponse var name has to be `error` because that's how DropBox's json property is named
- Change FileStorageRateLimitReached exception to store the seconds remaining as its own property instead of in the message
- Use toIntOrNull to avoid setting defaults in two places

* Fixed missed exception message

Co-authored-by: Azzurite <azzurite@gmail.com>
This commit is contained in:
Leonard Günther
2022-05-13 08:16:52 +02:00
committed by GitHub
parent 4ab7d56c14
commit 09b4e82589
10 changed files with 211 additions and 125 deletions

View File

@ -578,6 +578,7 @@ Current Turn: [civName] since [time] [timeUnit] ago =
Minutes =
Hours =
Days =
Server limit reached! Please wait for [time] seconds =
# Save game menu

View File

@ -15,6 +15,7 @@ import androidx.work.*
import com.badlogic.gdx.backends.android.AndroidApplication
import com.unciv.logic.GameInfo
import com.unciv.logic.GameSaver
import com.unciv.logic.multiplayer.FileStorageRateLimitReached
import com.unciv.models.metadata.GameSettings
import com.unciv.logic.multiplayer.OnlineMultiplayer
import java.io.FileNotFoundException
@ -269,6 +270,9 @@ class MultiplayerTurnCheckWorker(appContext: Context, workerParams: WorkerParame
foundGame = Pair(gameNames[arrayIndex], gameIds[arrayIndex])
}
arrayIndex++
} catch (ex: FileStorageRateLimitReached) {
// We just break here as configuredDelay is probably enough to wait for the rate limit anyway
break
} catch (ex: FileNotFoundException){
// FileNotFoundException is thrown by OnlineMultiplayer().tryDownloadGamePreview(gameId)
// and indicates that there is no game preview present for this game

View File

@ -8,10 +8,17 @@ import java.net.URL
import java.nio.charset.Charset
import java.util.*
import kotlin.collections.ArrayList
import kotlin.concurrent.timer
object DropBox {
fun dropboxApi(url: String, data: String = "", contentType: String = "", dropboxApiArg: String = ""): InputStream? {
object DropBox: IFileStorage {
private var remainingRateLimitSeconds = 0
private var rateLimitTimer: Timer? = null
private fun dropboxApi(url: String, data: String = "", contentType: String = "", dropboxApiArg: String = ""): InputStream? {
if (remainingRateLimitSeconds > 0)
throw FileStorageRateLimitReached(remainingRateLimitSeconds)
with(URL(url).openConnection() as HttpURLConnection) {
requestMethod = "POST" // default is GET
@ -40,11 +47,13 @@ object DropBox {
val responseString = reader.readText()
println(responseString)
val error = json().fromJson(ErrorResponse::class.java, responseString)
// Throw Exceptions based on the HTTP response from dropbox
if (responseString.contains("path/not_found/"))
throw FileNotFoundException()
if (responseString.contains("path/conflict/file"))
throw FileStorageConflictException()
when {
error.error_summary.startsWith("too_many_requests/") -> triggerRateLimit(error)
error.error_summary.startsWith("path/not_found/") -> throw FileNotFoundException()
error.error_summary.startsWith("path/conflict/file") -> throw FileStorageConflictException()
}
return null
} catch (error: Error) {
@ -56,8 +65,67 @@ object DropBox {
}
}
fun getFolderList(folder: String): ArrayList<DropboxMetaData> {
val folderList = ArrayList<DropboxMetaData>()
// This is the location in Dropbox only
private fun getLocalGameLocation(fileName: String) = "/MultiplayerGames/$fileName"
override fun deleteFile(fileName: String){
dropboxApi(
url="https://api.dropboxapi.com/2/files/delete_v2",
data="{\"path\":\"${getLocalGameLocation(fileName)}\"}",
contentType="application/json"
)
}
override fun getFileMetaData(fileName: String): IFileMetaData {
val stream = dropboxApi(
url="https://api.dropboxapi.com/2/files/get_metadata",
data="{\"path\":\"${getLocalGameLocation(fileName)}\"}",
contentType="application/json"
)!!
val reader = BufferedReader(InputStreamReader(stream))
return json().fromJson(MetaData::class.java, reader.readText())
}
override fun saveFileData(fileName: String, data: String, overwrite: Boolean) {
val overwriteModeString = if(!overwrite) "" else ""","mode":{".tag":"overwrite"}"""
dropboxApi(
url="https://content.dropboxapi.com/2/files/upload",
data=data,
contentType="application/octet-stream",
dropboxApiArg = """{"path":"${getLocalGameLocation(fileName)}"$overwriteModeString}"""
)
}
override fun loadFileData(fileName: String): String {
val inputStream = downloadFile(getLocalGameLocation(fileName))
return BufferedReader(InputStreamReader(inputStream)).readText()
}
fun downloadFile(fileName: String): InputStream {
val response = dropboxApi("https://content.dropboxapi.com/2/files/download",
contentType = "text/plain", dropboxApiArg = "{\"path\":\"$fileName\"}")
return response!!
}
/**
* If the dropbox rate limit is reached for this bearer token we strictly have to wait for the
* specified retry_after seconds before trying again. If non is supplied or can not be parsed
* the default value of 5 minutes will be used.
* Any attempt before the rate limit is dropped again will also contribute to the rate limit
*/
private fun triggerRateLimit(response: ErrorResponse) {
remainingRateLimitSeconds = response.error?.retry_after?.toIntOrNull() ?: 300
rateLimitTimer = timer("RateLimitTimer", true, 0, 1000) {
remainingRateLimitSeconds--
if (remainingRateLimitSeconds == 0)
rateLimitTimer?.cancel()
}
throw FileStorageRateLimitReached(remainingRateLimitSeconds)
}
fun getFolderList(folder: String): ArrayList<IFileMetaData> {
val folderList = ArrayList<IFileMetaData>()
// The DropBox API returns only partial file listings from one request. list_folder and
// list_folder/continue return similar responses, but list_folder/continue requires a cursor
// instead of the path.
@ -74,33 +142,6 @@ object DropBox {
return folderList
}
fun downloadFile(fileName: String): InputStream {
val response = dropboxApi("https://content.dropboxapi.com/2/files/download",
contentType = "text/plain", dropboxApiArg = "{\"path\":\"$fileName\"}")
return response!!
}
fun downloadFileAsString(fileName: String): String {
val inputStream = downloadFile(fileName)
return BufferedReader(InputStreamReader(inputStream)).readText()
}
/**
* @param overwrite set to true to avoid DropBoxFileConflictException
* @throws DropBoxFileConflictException when overwrite is false and a file with the
* same name already exists
*/
fun uploadFile(fileName: String, data: String, overwrite: Boolean = false) {
val overwriteModeString = if(!overwrite) "" else ""","mode":{".tag":"overwrite"}"""
dropboxApi("https://content.dropboxapi.com/2/files/upload",
data, "application/octet-stream", """{"path":"$fileName"$overwriteModeString}""")
}
fun deleteFile(fileName: String){
dropboxApi("https://api.dropboxapi.com/2/files/delete_v2",
"{\"path\":\"$fileName\"}", "application/json")
}
fun fileExists(fileName: String): Boolean {
try {
dropboxApi("https://api.dropboxapi.com/2/files/get_metadata",
@ -111,13 +152,6 @@ object DropBox {
}
}
fun getFileMetaData(fileName: String): IFileMetaData {
val stream = dropboxApi("https://api.dropboxapi.com/2/files/get_metadata",
"{\"path\":\"$fileName\"}", "application/json")!!
val reader = BufferedReader(InputStreamReader(stream))
return json().fromJson(DropboxMetaData::class.java, reader.readText())
}
//
// fun createTemplate(): String {
// val result = dropboxApi("https://api.dropboxapi.com/2/file_properties/templates/add_for_user",
@ -127,14 +161,14 @@ object DropBox {
// }
@Suppress("PropertyName")
class FolderList{
var entries = ArrayList<DropboxMetaData>()
private class FolderList{
var entries = ArrayList<MetaData>()
var cursor = ""
var has_more = false
}
@Suppress("PropertyName")
class DropboxMetaData: IFileMetaData {
private class MetaData: IFileMetaData {
var name = ""
private var server_modified = ""
@ -142,27 +176,14 @@ object DropBox {
return server_modified.parseDate()
}
}
}
class DropboxFileStorage: IFileStorage {
// This is the location in Dropbox only
fun getLocalGameLocation(fileName: String) = "/MultiplayerGames/$fileName"
override fun saveFileData(fileName: String, data: String) {
val fileLocationDropbox = getLocalGameLocation(fileName)
DropBox.uploadFile(fileLocationDropbox, data, true)
}
override fun loadFileData(fileName: String): String {
return DropBox.downloadFileAsString(getLocalGameLocation(fileName))
}
override fun getFileMetaData(fileName: String): IFileMetaData {
return DropBox.getFileMetaData(getLocalGameLocation(fileName))
}
override fun deleteFile(fileName: String) {
DropBox.deleteFile(getLocalGameLocation(fileName))
}
@Suppress("PropertyName")
private class ErrorResponse {
var error_summary = ""
var error: Details? = null
class Details {
var retry_after = ""
}
}
}

View File

@ -6,12 +6,29 @@ import com.unciv.UncivGame
import com.unciv.logic.GameInfo
import com.unciv.logic.GameInfoPreview
import com.unciv.logic.GameSaver
import java.io.FileNotFoundException
import java.util.*
interface IFileStorage {
fun saveFileData(fileName: String, data: String)
/**
* @throws FileStorageRateLimitReached if the file storage backend can't handle any additional actions for a time
* @throws FileStorageConflictException if the file already exists and [overwrite] is false
*/
fun saveFileData(fileName: String, data: String, overwrite: Boolean)
/**
* @throws FileStorageRateLimitReached if the file storage backend can't handle any additional actions for a time
* @throws FileNotFoundException if the file can't be found
*/
fun loadFileData(fileName: String): String
/**
* @throws FileStorageRateLimitReached if the file storage backend can't handle any additional actions for a time
* @throws FileNotFoundException if the file can't be found
*/
fun getFileMetaData(fileName: String): IFileMetaData
/**
* @throws FileStorageRateLimitReached if the file storage backend can't handle any additional actions for a time
* @throws FileNotFoundException if the file can't be found
*/
fun deleteFile(fileName: String)
}
@ -22,7 +39,7 @@ interface IFileMetaData {
class UncivServerFileStorage(val serverUrl:String):IFileStorage {
override fun saveFileData(fileName: String, data: String) {
override fun saveFileData(fileName: String, data: String, overwrite: Boolean) {
SimpleHttp.sendRequest(Net.HttpMethods.PUT, "$serverUrl/files/$fileName", data){
success: Boolean, result: String ->
if (!success) {
@ -59,6 +76,7 @@ class UncivServerFileStorage(val serverUrl:String):IFileStorage {
}
class FileStorageConflictException: Exception()
class FileStorageRateLimitReached(val limitRemainingSeconds: Int): Exception()
/**
* Allows access to games stored on a server for multiplayer purposes.
@ -74,7 +92,7 @@ class OnlineMultiplayer(var fileStorageIdentifier: String? = null) {
if (fileStorageIdentifier == null)
fileStorageIdentifier = UncivGame.Current.settings.multiplayerServer
fileStorage = if (fileStorageIdentifier == Constants.dropboxMultiplayerServer)
DropboxFileStorage()
DropBox
else UncivServerFileStorage(fileStorageIdentifier!!)
}
@ -86,7 +104,7 @@ class OnlineMultiplayer(var fileStorageIdentifier: String? = null) {
}
val zippedGameInfo = GameSaver.gameInfoToString(gameInfo, forceZip = true)
fileStorage.saveFileData(gameInfo.gameId, zippedGameInfo)
fileStorage.saveFileData(gameInfo.gameId, zippedGameInfo, true)
}
/**
@ -97,7 +115,7 @@ class OnlineMultiplayer(var fileStorageIdentifier: String? = null) {
*/
fun tryUploadGamePreview(gameInfo: GameInfoPreview) {
val zippedGameInfo = GameSaver.gameInfoToString(gameInfo)
fileStorage.saveFileData("${gameInfo.gameId}_Preview", zippedGameInfo)
fileStorage.saveFileData("${gameInfo.gameId}_Preview", zippedGameInfo, true)
}
fun tryDownloadGame(gameId: String): GameInfo {

View File

@ -3,11 +3,8 @@ package com.unciv.logic.multiplayer
import com.unciv.json.json
import com.unciv.logic.GameInfo
import com.unciv.logic.GameInfoPreview
import com.unciv.logic.GameSaver
import com.unciv.ui.saves.Gzip
import java.io.BufferedReader
import java.io.FileNotFoundException
import java.io.InputStreamReader
import java.util.*
import kotlin.math.pow
@ -68,7 +65,7 @@ class ServerMutex(val gameInfo: GameInfoPreview) {
}
try {
OnlineMultiplayer().fileStorage.saveFileData(fileName, Gzip.zip(json().toJson(LockFile())))
OnlineMultiplayer().fileStorage.saveFileData(fileName, Gzip.zip(json().toJson(LockFile())), false)
} catch (ex: FileStorageConflictException) {
return locked
}

View File

@ -5,6 +5,7 @@ import com.badlogic.gdx.scenes.scene2d.ui.TextField
import com.unciv.logic.GameInfoPreview
import com.unciv.logic.GameSaver
import com.unciv.logic.civilization.PlayerType
import com.unciv.logic.multiplayer.FileStorageRateLimitReached
import com.unciv.models.translations.tr
import com.unciv.ui.pickerscreens.PickerScreen
import com.unciv.ui.utils.*
@ -117,18 +118,16 @@ class EditMultiplayerGameInfoScreen(val gameInfo: GameInfoPreview?, gameName: St
}
} else {
postCrashHandlingRunnable {
//change popup text
popup.innerTable.clear()
popup.addGoodSizedLabel("You can only resign if it's your turn").row()
popup.addCloseButton()
popup.reuseWith("You can only resign if it's your turn", true)
}
}
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
popup.reuseWith("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds", true)
}
} catch (ex: Exception) {
postCrashHandlingRunnable {
//change popup text
popup.innerTable.clear()
popup.addGoodSizedLabel("Could not upload game!").row()
popup.addCloseButton()
popup.reuseWith("Could not upload game!", true)
}
}
}

View File

@ -4,6 +4,7 @@ import com.badlogic.gdx.Gdx
import com.badlogic.gdx.files.FileHandle
import com.badlogic.gdx.scenes.scene2d.ui.*
import com.unciv.logic.*
import com.unciv.logic.multiplayer.FileStorageRateLimitReached
import com.unciv.models.translations.tr
import com.unciv.ui.pickerscreens.PickerScreen
import com.unciv.ui.utils.*
@ -126,20 +127,20 @@ class MultiplayerScreen(previousScreen: BaseScreen) : PickerScreen() {
//Adds a new Multiplayer game to the List
//gameId must be nullable because clipboard content could be null
fun addMultiplayerGame(gameId: String?, gameName: String = "") {
val popup = Popup(this)
popup.addGoodSizedLabel("Working...")
popup.open()
try {
//since the gameId is a String it can contain anything and has to be checked
UUID.fromString(IdChecker.checkAndReturnGameUuid(gameId!!))
} catch (ex: Exception) {
val errorPopup = Popup(this)
errorPopup.addGoodSizedLabel("Invalid game ID!")
errorPopup.row()
errorPopup.addCloseButton()
errorPopup.open()
popup.reuseWith("Invalid game ID!", true)
return
}
if (gameIsAlreadySavedAsMultiplayer(gameId)) {
ToastPopup("Game is already added", this)
popup.reuseWith("Game is already added", true)
return
}
@ -168,20 +169,16 @@ class MultiplayerScreen(previousScreen: BaseScreen) : PickerScreen() {
postCrashHandlingRunnable { reloadGameListUI() }
} catch (ex: Exception) {
postCrashHandlingRunnable {
val errorPopup = Popup(this)
errorPopup.addGoodSizedLabel("Could not download game!")
errorPopup.row()
errorPopup.addCloseButton()
errorPopup.open()
popup.reuseWith("Could not download game!", true)
}
}
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
popup.reuseWith("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds", true)
}
} catch (ex: Exception) {
postCrashHandlingRunnable {
val errorPopup = Popup(this)
errorPopup.addGoodSizedLabel("Could not download game!")
errorPopup.row()
errorPopup.addCloseButton()
errorPopup.open()
popup.reuseWith("Could not download game!", true)
}
}
postCrashHandlingRunnable {
@ -202,14 +199,13 @@ class MultiplayerScreen(previousScreen: BaseScreen) : PickerScreen() {
val gameId = multiplayerGames[selectedGameFile]!!.gameId
val gameInfo = OnlineMultiplayer().tryDownloadGame(gameId)
postCrashHandlingRunnable { game.loadGame(gameInfo) }
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
loadingGamePopup.reuseWith("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds", true)
}
} catch (ex: Exception) {
postCrashHandlingRunnable {
loadingGamePopup.close()
val errorPopup = Popup(this)
errorPopup.addGoodSizedLabel("Could not download game!")
errorPopup.row()
errorPopup.addCloseButton()
errorPopup.open()
loadingGamePopup.reuseWith("Could not download game!", true)
}
}
}
@ -356,6 +352,11 @@ class MultiplayerScreen(previousScreen: BaseScreen) : PickerScreen() {
ToastPopup("Could not download game!" + " ${fileHandle.name()}", this)
}
}
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
ToastPopup("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds", this)
}
break // No need to keep trying if rate limit is reached
} catch (ex: Exception) {
//skipping one is not fatal
//Trying to use as many prev. used strings as possible

View File

@ -11,6 +11,7 @@ import com.unciv.UncivGame
import com.unciv.logic.*
import com.unciv.logic.civilization.PlayerType
import com.unciv.logic.map.MapType
import com.unciv.logic.multiplayer.FileStorageRateLimitReached
import com.unciv.logic.multiplayer.OnlineMultiplayer
import com.unciv.models.metadata.GameSetupInfo
import com.unciv.models.ruleset.RulesetCache
@ -45,7 +46,7 @@ class NewGameScreen(
if (gameSetupInfo.gameParameters.victoryTypes.isEmpty())
gameSetupInfo.gameParameters.victoryTypes.addAll(ruleset.victories.keys)
playerPickerTable = PlayerPickerTable(
this, gameSetupInfo.gameParameters,
if (isNarrowerThan4to3()) stage.width - 20f else 0f
@ -107,7 +108,7 @@ class NewGameScreen(
noHumanPlayersPopup.open()
return@onClick
}
if (gameSetupInfo.gameParameters.victoryTypes.isEmpty()) {
val noVictoryTypesPopup = Popup(this)
noVictoryTypesPopup.addGoodSizedLabel("No victory conditions were selected!".tr()).row()
@ -226,17 +227,23 @@ class NewGameScreen(
}
private fun newGameThread() {
val popup = Popup(this)
postCrashHandlingRunnable {
popup.addGoodSizedLabel("Working...").row()
popup.open()
}
val newGame:GameInfo
try {
newGame = GameStarter.startNewGame(gameSetupInfo)
} catch (exception: Exception) {
exception.printStackTrace()
postCrashHandlingRunnable {
Popup(this).apply {
addGoodSizedLabel("It looks like we can't make a map with the parameters you requested!".tr()).row()
addGoodSizedLabel("Maybe you put too many players into too small a map?".tr()).row()
popup.apply {
reuseWith("It looks like we can't make a map with the parameters you requested!")
row()
addGoodSizedLabel("Maybe you put too many players into too small a map?").row()
addCloseButton()
open()
}
Gdx.input.inputProcessor = stage
rightSideButton.enable()
@ -255,15 +262,21 @@ class NewGameScreen(
// Saved as Multiplayer game to show up in the session browser
val newGamePreview = newGame.asPreview()
GameSaver.saveGame(newGamePreview, newGamePreview.gameId)
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
popup.reuseWith("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds", true)
}
Gdx.input.inputProcessor = stage
rightSideButton.enable()
rightSideButton.setText("Start game!".tr())
return
} catch (ex: Exception) {
postCrashHandlingRunnable {
Popup(this).apply {
addGoodSizedLabel("Could not upload game!").row()
Gdx.input.inputProcessor = stage
addCloseButton()
open()
}
popup.reuseWith("Could not upload game!", true)
}
Gdx.input.inputProcessor = stage
rightSideButton.enable()
rightSideButton.setText("Start game!".tr())
return
}
}

View File

@ -184,6 +184,20 @@ open class Popup(val screen: BaseScreen): Table(BaseScreen.skin) {
cell2.minWidth(cell1.actor.width)
}
/**
* Reuse this popup as an error/info popup with a new message.
* Removes everything from the popup to replace it with the message
* and a close button if requested
*/
fun reuseWith(newText: String, withCloseButton: Boolean = false) {
innerTable.clear()
addGoodSizedLabel(newText)
if (withCloseButton) {
row()
addCloseButton()
}
}
/**
* Sets or retrieves the [Actor] that currently has keyboard focus.
*

View File

@ -21,6 +21,7 @@ import com.unciv.logic.civilization.CivilizationInfo
import com.unciv.logic.civilization.ReligionState
import com.unciv.logic.civilization.diplomacy.DiplomaticStatus
import com.unciv.logic.map.MapVisualization
import com.unciv.logic.multiplayer.FileStorageRateLimitReached
import com.unciv.logic.trade.TradeEvaluation
import com.unciv.models.Tutorial
import com.unciv.models.UncivSound
@ -370,14 +371,24 @@ class WorldScreen(val gameInfo: GameInfo, val viewingCiv:CivilizationInfo) : Bas
postCrashHandlingRunnable { createNewWorldScreen(latestGame) }
}
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
loadingGamePopup.reuseWith("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds", true)
}
// stop refresher to not spam user with "Server limit reached!"
// popups and restart after limit timer is over
stopMultiPlayerRefresher()
val restartAfter : Long = ex.limitRemainingSeconds.toLong() * 1000
timer("RestartTimerTimer", true, restartAfter, 0 ) {
multiPlayerRefresher = timer("multiPlayerRefresh", true, period = 10000) {
loadLatestMultiplayerState()
}
}
} catch (ex: Throwable) {
postCrashHandlingRunnable {
val couldntDownloadLatestGame = Popup(this)
couldntDownloadLatestGame.addGoodSizedLabel("Couldn't download the latest game state!").row()
couldntDownloadLatestGame.addCloseButton()
couldntDownloadLatestGame.addAction(Actions.delay(5f, Actions.run { couldntDownloadLatestGame.close() }))
loadingGamePopup.close()
couldntDownloadLatestGame.open()
loadingGamePopup.reuseWith("Couldn't download the latest game state!", true)
loadingGamePopup.addAction(Actions.delay(5f, Actions.run { loadingGamePopup.close() }))
}
}
}
@ -664,6 +675,13 @@ class WorldScreen(val gameInfo: GameInfo, val viewingCiv:CivilizationInfo) : Bas
if (originalGameInfo.gameParameters.isOnlineMultiplayer) {
try {
OnlineMultiplayer().tryUploadGame(gameInfoClone, withPreview = true)
} catch (ex: FileStorageRateLimitReached) {
postCrashHandlingRunnable {
val cantUploadNewGamePopup = Popup(this)
cantUploadNewGamePopup.addGoodSizedLabel("Server limit reached! Please wait for [${ex.limitRemainingSeconds}] seconds").row()
cantUploadNewGamePopup.addCloseButton()
cantUploadNewGamePopup.open()
}
} catch (ex: Exception) {
postCrashHandlingRunnable { // Since we're changing the UI, that should be done on the main thread
val cantUploadNewGamePopup = Popup(this)