From 3d9c5bcc34c26fcac867b6339220515e3de68d2b Mon Sep 17 00:00:00 2001 From: SomeTroglodyte <63000004+SomeTroglodyte@users.noreply.github.com> Date: Thu, 16 Sep 2021 19:59:50 +0200 Subject: [PATCH] Mod TranslationFileWriter re-work (#5219) * Mod TranslationFileWriter re-work * Mod TranslationFileWriter re-work - style and existing in base treatment * Mod TranslationFileWriter re-work - style and existing in base treatment --- .../translations/TranslationFileWriter.kt | 97 +++++++++++-------- .../unciv/models/translations/Translations.kt | 66 +++++++------ .../ui/worldscreen/mainmenu/OptionsPopup.kt | 6 +- 3 files changed, 96 insertions(+), 73 deletions(-) diff --git a/core/src/com/unciv/models/translations/TranslationFileWriter.kt b/core/src/com/unciv/models/translations/TranslationFileWriter.kt index 13452e7d34..f00ee5afa1 100644 --- a/core/src/com/unciv/models/translations/TranslationFileWriter.kt +++ b/core/src/com/unciv/models/translations/TranslationFileWriter.kt @@ -24,31 +24,49 @@ object TranslationFileWriter { const val templateFileLocation = "jsons/translations/template.properties" private const val languageFileLocation = "jsons/translations/%s.properties" - fun writeNewTranslationFiles(translations: Translations) { + fun writeNewTranslationFiles(): String { + try { + val translations = Translations() + translations.readAllLanguagesTranslation() - val percentages = generateTranslationFiles(translations) - writeLanguagePercentages(percentages) + val percentages = generateTranslationFiles(translations) + writeLanguagePercentages(percentages) - // try to do the same for the mods - for (modFolder in Gdx.files.local("mods").list().filter { it.isDirectory }) - generateTranslationFiles(translations, modFolder) - // write percentages is not needed: for an individual mod it makes no sense + // See #5168 for some background on this + for ((modName, modTranslations) in translations.modsWithTranslations) { + val modFolder = Gdx.files.local("mods").child(modName) + val modPercentages = generateTranslationFiles(modTranslations, modFolder, translations) + writeLanguagePercentages(modPercentages, modFolder) // unused by the game but maybe helpful for the mod developer + } + return "Translation files are generated successfully." + } catch (ex: Throwable) { + return ex.localizedMessage + } } private fun getFileHandle(modFolder: FileHandle?, fileLocation: String) = if (modFolder != null) modFolder.child(fileLocation) else Gdx.files.local(fileLocation) - private fun generateTranslationFiles(translations: Translations, modFolder: FileHandle? = null): HashMap { + /** + * Writes new language files per Mod or for BaseRuleset - only each language that exists in [translations]. + * @param baseTranslations For a mod, pass the base translations here so strings already existing there can be seen + * @return a map with the percentages of translated lines per language + */ + private fun generateTranslationFiles( + translations: Translations, + modFolder: FileHandle? = null, + baseTranslations: Translations? = null + ): HashMap { val fileNameToGeneratedStrings = LinkedHashMap>() - val linesFromTemplates = mutableListOf() + val linesToTranslate = mutableListOf() if (modFolder == null) { // base game val templateFile = getFileHandle(modFolder, templateFileLocation) // read the template if (templateFile.exists()) - linesFromTemplates.addAll(templateFile.reader(TranslationFileReader.charset).readLines()) + linesToTranslate.addAll(templateFile.reader(TranslationFileReader.charset).readLines()) for (baseRuleset in BaseRuleset.values()) { val generatedStringsFromBaseRuleset = @@ -58,28 +76,27 @@ object TranslationFileWriter { } fileNameToGeneratedStrings["Tutorials"] = generateTutorialsStrings() - } else fileNameToGeneratedStrings.putAll(generateStringsFromJSONs(modFolder)) + } else { + fileNameToGeneratedStrings.putAll(generateStringsFromJSONs(modFolder.child("jsons"))) + } - // Tutorials are a bit special - if (modFolder == null) // this is for base only, not mods - - for (key in fileNameToGeneratedStrings.keys) { - linesFromTemplates.add("\n#################### Lines from $key ####################\n") - linesFromTemplates.addAll(fileNameToGeneratedStrings.getValue(key)) - } + for (key in fileNameToGeneratedStrings.keys) { + linesToTranslate.add("\n#################### Lines from $key ####################\n") + linesToTranslate.addAll(fileNameToGeneratedStrings.getValue(key)) + } var countOfTranslatableLines = 0 val countOfTranslatedLines = HashMap() // iterate through all available languages - for (language in translations.getLanguages()) { + for ((languageIndex, language) in translations.getLanguages().withIndex()) { var translationsOfThisLanguage = 0 val stringBuilder = StringBuilder() // This is so we don't add the same keys twice if we have the same value in both Vanilla and G&K val existingTranslationKeys = HashSet() - for (line in linesFromTemplates) { + for (line in linesToTranslate) { if (!line.contains(" = ")) { // small hack to insert empty lines if (line.startsWith(specialNewLineCode)) { @@ -99,16 +116,21 @@ object TranslationFileWriter { if (existingTranslationKeys.contains(hashMapKey)) continue // don't add it twice existingTranslationKeys.add(hashMapKey) - // count translatable lines only once (e.g. for English) - if (language == "English") countOfTranslatableLines++ + // count translatable lines only once + if (languageIndex == 0) countOfTranslatableLines++ - var translationValue = "" - - val translationEntry = translations[hashMapKey] - if (translationEntry != null && translationEntry.containsKey(language)) { - translationValue = translationEntry[language]!! + val existingTranslation = translations[hashMapKey] + var translationValue = if (existingTranslation != null && language in existingTranslation){ translationsOfThisLanguage++ - } else stringBuilder.appendLine(" # Requires translation!") + existingTranslation[language]!! + } else if (baseTranslations?.get(hashMapKey)?.containsKey(language) == true) { + // String is used in the mod but also exists in base - ignore + continue + } else { + // String is not translated either here or in base + stringBuilder.appendLine(" # Requires translation!") + "" + } // THE PROBLEM // When we come to change params written in the TranslationFileWriter, @@ -141,22 +163,19 @@ object TranslationFileWriter { // Calculate the percentages of translations // It should be done after the loop of languages, since the countOfTranslatableLines is not known in the 1st iteration - for (key in countOfTranslatedLines.keys) - countOfTranslatedLines[key] = if (countOfTranslatableLines > 0) countOfTranslatedLines.getValue(key) * 100 / countOfTranslatableLines - else 100 + for (entry in countOfTranslatedLines) + entry.setValue(if (countOfTranslatableLines <= 0) 100 else entry.value * 100 / countOfTranslatableLines) return countOfTranslatedLines } - private fun writeLanguagePercentages(percentages: HashMap) { - val stringBuilder = StringBuilder() - for (entry in percentages) { - stringBuilder.appendLine(entry.key + " = " + entry.value) - } - Gdx.files.local(TranslationFileReader.percentagesFileLocation).writeString(stringBuilder.toString(), false) + private fun writeLanguagePercentages(percentages: HashMap, modFolder: FileHandle? = null) { + val output = percentages.asSequence() + .joinToString("\n", postfix = "\n") { "${it.key} = ${it.value}" } + getFileHandle(modFolder, TranslationFileReader.percentagesFileLocation) + .writeString(output, false) } - private fun generateTutorialsStrings(): MutableSet { val tutorialsStrings = mutableSetOf() @@ -395,4 +414,4 @@ object TranslationFileWriter { } } -} \ No newline at end of file +} diff --git a/core/src/com/unciv/models/translations/Translations.kt b/core/src/com/unciv/models/translations/Translations.kt index 3033be2e39..229aca944a 100644 --- a/core/src/com/unciv/models/translations/Translations.kt +++ b/core/src/com/unciv/models/translations/Translations.kt @@ -5,6 +5,7 @@ import com.unciv.UncivGame import com.unciv.models.stats.Stats import java.util.* import kotlin.collections.HashMap +import kotlin.collections.LinkedHashSet /** * This collection holds all translations for the game. @@ -30,7 +31,7 @@ class Translations : LinkedHashMap(){ var percentCompleteOfLanguages = HashMap() .apply { put("English",100) } // So even if we don't manage to load the percentages, we can still pass the language screen - private var modsWithTranslations: HashMap = hashMapOf() // key == mod name + internal var modsWithTranslations: HashMap = hashMapOf() // key == mod name // used by tr() whenever GameInfo not initialized (allowing new game screen to use mod translations) var translationActiveMods = LinkedHashSet() @@ -64,17 +65,16 @@ class Translations : LinkedHashMap(){ return get(text, language, activeMods)?.get(language) ?: text } - fun getLanguages(): List { - val toReturn = mutableListOf() - - for(entry in values) - for(languageName in entry.keys) - if(!toReturn.contains(languageName)) toReturn.add(languageName) - - return toReturn - } - + /** Get all languages present in `this`, used for [TranslationFileWriter] and `TranslationTests` */ + fun getLanguages() = linkedSetOf().apply { + for (entry in values) + for (languageName in entry.keys) + add(languageName) + } + /** This reads all translations for a specific language, including _all_ installed mods. + * Vanilla translations go into `this` instance, mod translations into [modsWithTranslations]. + */ private fun tryReadTranslationForLanguage(language: String, printOutput: Boolean) { val translationStart = System.currentTimeMillis() @@ -86,6 +86,7 @@ class Translations : LinkedHashMap(){ // which is super odd because everyone should support UTF-8 languageTranslations = TranslationFileReader.read(Gdx.files.internal(translationFileName)) } catch (ex: Exception) { + println("Exception reading translations for $language: ${ex.message}") return } @@ -93,33 +94,36 @@ class Translations : LinkedHashMap(){ for (modFolder in Gdx.files.local("mods").list()) { val modTranslationFile = modFolder.child(translationFileName) if (modTranslationFile.exists()) { - val translationsForMod = Translations() - createTranslations(language, TranslationFileReader.read(modTranslationFile), translationsForMod) - - modsWithTranslations[modFolder.name()] = translationsForMod + var translationsForMod = modsWithTranslations[modFolder.name()] + if (translationsForMod == null) { + translationsForMod = Translations() + modsWithTranslations[modFolder.name()] = translationsForMod + } + try { + translationsForMod.createTranslations(language, TranslationFileReader.read(modTranslationFile)) + } catch (ex: Exception) { + println("Exception reading translations for ${modFolder.name()} $language: ${ex.message}") + } } } createTranslations(language, languageTranslations) val translationFilesTime = System.currentTimeMillis() - translationStart - if(printOutput) println("Loading translation file for $language - " + translationFilesTime + "ms") + if (printOutput) println("Loading translation file for $language - " + translationFilesTime + "ms") } - private fun createTranslations(language: String, - languageTranslations: HashMap, - targetTranslations: Translations = this) { + private fun createTranslations(language: String, languageTranslations: HashMap) { for (translation in languageTranslations) { val hashKey = if (translation.key.contains('[')) translation.key.getPlaceholderText() else translation.key - if (!containsKey(hashKey)) - targetTranslations[hashKey] = TranslationEntry(translation.key) - - // why not in one line, Because there were actual crashes. - // I'm pretty sure I solved this already, but hey double-checking doesn't cost anything. - val entry = targetTranslations[hashKey] - if (entry != null) entry[language] = translation.value + var entry = this[hashKey] + if (entry == null) { + entry = TranslationEntry(translation.key) + this[hashKey] = entry + } + entry[language] = translation.value } } @@ -127,6 +131,7 @@ class Translations : LinkedHashMap(){ tryReadTranslationForLanguage(UncivGame.Current.settings.language, false) } + /** Get a list of supported languages for [readAllLanguagesTranslation] */ // This function is too strange for me, however, let's keep it "as is" for now. - JackRainy private fun getLanguagesWithTranslationFile(): List { @@ -134,10 +139,10 @@ class Translations : LinkedHashMap(){ // So apparently the Locales don't work for everyone, which is horrendous // So for those players, which seem to be Android-y, we try to see what files exist directly...yeah =/ try{ - for(file in Gdx.files.internal("jsons/translations").list()) + for (file in Gdx.files.internal("jsons/translations").list()) languages.add(file.nameWithoutExtension()) } - catch (ex:Exception){} // Iterating on internal files will not work when running from a .jar + catch (ex:Exception) {} // Iterating on internal files will not work when running from a .jar languages.addAll(Locale.getAvailableLocales() // And this should work for Desktop, meaning from a .jar .map { it.getDisplayName(Locale.ENGLISH) }) // Maybe THIS is the problem, that the DISPLAY locale wasn't english @@ -156,6 +161,7 @@ class Translations : LinkedHashMap(){ .filter { Gdx.files.internal("jsons/translations/$it.properties").exists() } } + /** Ensure _all_ languages are loaded, used by [TranslationFileWriter] and `TranslationTests` */ fun readAllLanguagesTranslation(printOutput:Boolean=false) { // Apparently you can't iterate over the files in a directory when running out of a .jar... // https://www.badlogicgames.com/forum/viewtopic.php?f=11&t=27250 @@ -168,7 +174,7 @@ class Translations : LinkedHashMap(){ } val translationFilesTime = System.currentTimeMillis() - translationStart - if(printOutput) println("Loading translation files - "+translationFilesTime+"ms") + if(printOutput) println("Loading translation files - ${translationFilesTime}ms") } fun loadPercentageCompleteOfLanguages(){ @@ -177,7 +183,7 @@ class Translations : LinkedHashMap(){ percentCompleteOfLanguages = TranslationFileReader.readLanguagePercentages() val translationFilesTime = System.currentTimeMillis() - startTime - println("Loading percent complete of languages - "+translationFilesTime+"ms") + println("Loading percent complete of languages - ${translationFilesTime}ms") } } diff --git a/core/src/com/unciv/ui/worldscreen/mainmenu/OptionsPopup.kt b/core/src/com/unciv/ui/worldscreen/mainmenu/OptionsPopup.kt index 39b20bd313..ad52310dd3 100644 --- a/core/src/com/unciv/ui/worldscreen/mainmenu/OptionsPopup.kt +++ b/core/src/com/unciv/ui/worldscreen/mainmenu/OptionsPopup.kt @@ -512,11 +512,9 @@ class OptionsPopup(val previousScreen: CameraStageBaseScreen) : Popup(previousSc if (Gdx.app.type == Application.ApplicationType.Desktop) { val generateTranslationsButton = "Generate translation files".toTextButton() val generateAction = { - val translations = Translations() - translations.readAllLanguagesTranslation() - TranslationFileWriter.writeNewTranslationFiles(translations) + val result = TranslationFileWriter.writeNewTranslationFiles() // notify about completion - generateTranslationsButton.setText("Translation files are generated successfully.".tr()) + generateTranslationsButton.setText(result.tr()) generateTranslationsButton.disable() } generateTranslationsButton.onClick(generateAction)