xone47/0ad
1
0
Fork 0
forked from 0ad/0ad
Code Pull Requests Activity

Generate nightly SVN builds for end users, fixes #1819.

Browse Source
This commit is contained in:
Nicolas Auvray 2024-08-20 19:15:56 +02:00
parent b698558102
commit e9d1cb6cc7
14 changed files with 226 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -64,7 +64,8 @@ test_*.cpp
stub_*.cpp
!test_setup.cpp
# Binary files generated in nightly builds
# Translation files
binaries/data/mods/public/gui/credits/texts/translators.json
*.po
*.pot
@ -86,6 +87,9 @@ stub_*.cpp
binaries/data/mods/**/*.bmp
binaries/data/mods/**/*.jpg
# Vulkan SPIR-V shaders
binaries/data/mods/*/shaders/spirv
# Windows specific data
desktop.ini
Thumbs.db

3
binaries/data/mods/public/gui/credits/credits.js
View File

@ -37,6 +37,9 @@ function init()
let json = Engine.ReadJSONFile("gui/credits/texts/" + category + ".json");
if (!json || !json.Content)
{
if (category == "translators")
warn("Translators credits are not present, pull translations from the nightly build to get them.")
else
error("Could not load credits for " + category + "!");
continue;
}

20
build/jenkins/dockerfiles/translations.Dockerfile
View File

@ -1,20 +0,0 @@
FROM debian:buster
ARG DEBIAN_FRONTEND=noninteractive
ARG DEBCONF_NOWARNINGS="yes"
RUN useradd -ms /bin/bash --uid 1006 builder
RUN apt-get -qq update && apt-get install -qqy --no-install-recommends \
curl \
python3-dev \
python3-pip \
git \
subversion \
&& apt-get clean
ENV SHELL /bin/bash
RUN pip3 install setuptools wheel
RUN pip3 install lxml babel
RUN curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
RUN install tx /usr/bin/tx
USER builder
COPY --chown=builder transifexrc /home/builder/.transifexrc

35
build/jenkins/lint-translations.sh
View File

@ -1,35 +0,0 @@
#!/bin/sh
# This script uses the Dennis PO(T) linter to find issues.
# See http://dennis.readthedocs.io/en/latest/index.html for
# installation instructions.
set +e # Lint everything without failing
# Move to the root of the repository (this script is in build/jenkins/)
cd "$(dirname $0)"/../../
# Configuration for the linter
# Ignore
# - W302: Translated string is identical to source string
parameters='--excluderules W302'
# Run lint and output to a file that will be posted on Phabricator
echo "Running Dennis..."
{
echo "Linting templates..."
echo "Engine"
dennis-cmd lint ${parameters} binaries/data/l10n/*.pot
echo "Mod mod"
dennis-cmd lint ${parameters} binaries/data/mods/mod/l10n/*.pot
echo "Public mod"
dennis-cmd lint ${parameters} binaries/data/mods/public/l10n/*.pot
echo "Linting translations..."
echo "Engine"
dennis-cmd lint ${parameters} binaries/data/l10n/*.po
echo "Mod mod"
dennis-cmd lint ${parameters} binaries/data/mods/mod/l10n/*.po
echo "Public mod"
dennis-cmd lint ${parameters} binaries/data/mods/public/l10n/*.po
} > .phabricator-comment

76
build/jenkins/pipelines/docker-translations.Jenkinsfile
View File

@ -1,76 +0,0 @@
/* Copyright (C) 2023 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
*/
// This pipeline is used to update translations to and from Transifex.
pipeline {
agent {
node {
label 'LinuxSlave'
customWorkspace '/zpool0/trunk'
}
}
stages {
stage("Prepare volume") {
steps {
sh "sudo zfs clone zpool0/trunk@latest zpool0/translations"
ws('/zpool0/translations') {
sh "svn revert . -R"
sh "svn st | cut -c 9- | xargs rm -rf"
sh "svn up"
}
}
}
stage("Update translations") {
steps {
ws('/zpool0/translations') {
withDockerContainer("0ad-translations:latest") {
sh "python3 source/tools/i18n/updateTemplates.py"
withCredentials([string(credentialsId: 'redacted', variable: 'token')]) {
sh 'TX_TOKEN="$token" python3 source/tools/i18n/pullTranslations.py'
}
dir("source/tools/i18n/") {
sh "python3 generateDebugTranslation.py --long"
}
sh "python3 source/tools/i18n/cleanTranslationFiles.py"
sh "python3 source/tools/i18n/checkDiff.py --verbose"
sh "python3 source/tools/i18n/creditTranslators.py"
}
}
}
}
stage("Commit") {
steps {
ws('/zpool0/translations') {
withCredentials([usernamePassword(credentialsId: 'redacted', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) {
sh 'svn relocate --username $SVNUSER --password $SVNPASS --no-auth-cache https://svn.wildfiregames.com/svn/ps/trunk'
sh 'svn commit --username $SVNUSER --password $SVNPASS --no-auth-cache --non-interactive -m "[i18n] Updated POT and PO files."'
}
}
}
}
}
post {
always {
sleep 10
sh "sudo zfs destroy zpool0/translations"
}
}
}

164
build/jenkins/pipelines/nightly-build.Jenkinsfile Normal file
View File

@ -0,0 +1,164 @@
/* Copyright (C) 2024 Wildfire Games.
* This file is part of 0 A.D.
*
* 0 A.D. is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* 0 A.D. is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with 0 A.D. If not, see <http://www.gnu.org/licenses/>.
*/
// This pipeline is used to generate the nightly builds.
def visualStudioPath = "\"C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe\""
def buildOptions = "/p:PlatformToolset=v141_xp /p:XPDeprecationWarning=false /t:pyrogenesis /t:AtlasUI /m:2 /nologo -clp:Warningsonly -clp:ErrorsOnly"
def gitHash = ""
def buildSPIRV = false
pipeline {
agent {
node {
label 'WindowsAgent'
customWorkspace 'workspace/nightly-build'
}
}
parameters {
booleanParam(name: 'NEW_REPO', defaultValue: false, description: 'If a brand new nightly repo is being generated, do not attempt to identify unchanged translations.')
stashedFile(name: 'spirv_rules', description: 'rules.json file for generation of SPIR-V shaders. Needed for a new repo, else the existing rules files will be used. Uploading a new rules file will force-rebuild the shaders.')
}
stages {
stage("Generate build version") {
steps {
checkout scmGit(branches: [[name: "${GIT_BRANCH}"]], extensions: [localBranch()])
script { gitHash = bat(script:"@git rev-parse --short HEAD", returnStdout: true ).trim() }
bat "cd build\\build_version && build_version.bat"
}
}
stage("Pull game assets") {
steps {
bat "git lfs pull"
}
}
stage("Check for shader changes") {
when {
changeset 'binaries/data/mods/**/shaders/**/*.xml'
}
steps {
script { buildSPIRV = true }
}
}
stage ("Pre-build") {
steps {
bat "cd libraries && get-windows-libs.bat"
bat "(robocopy C:\\wxwidgets3.0.4\\lib libraries\\win32\\wxwidgets\\lib /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0"
bat "(robocopy C:\\wxwidgets3.0.4\\include libraries\\win32\\wxwidgets\\include /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0"
bat "cd build\\workspaces && update-workspaces.bat --atlas --without-pch --without-tests"
}
}
stage ("Build") {
steps {
bat("cd build\\workspaces\\vs2017 && ${visualStudioPath} pyrogenesis.sln /p:Configuration=Release ${buildOptions}")
}
}
stage("Check-in SPIRV generation rules") {
when {
expression { env.spirv_rules_FILENAME }
}
steps {
unstash 'spirv_rules'
bat "move spirv_rules source\\tools\\spirv\\rules.json"
script { buildSPIRV = true }
}
}
stage("Mirror to SVN") {
steps {
ws("workspace/nightly-svn") {
svn(url: "https://svn.wildfiregames.com/nightly-build/trunk", changelog: false)
bat "svn revert -R . && svn cleanup"
script { env.NIGHTLY_PATH = env.WORKSPACE }
}
bat """
(robocopy . %NIGHTLY_PATH% ^
/XD .git ^
/XF .gitattributes ^
/XF .gitignore ^
/XD %cd%\\binaries\\system ^
/XD %cd%\\build\\workspaces\\vs2017 ^
/XD %cd%\\libraries\\source\\.svn ^
/XD %cd%\\libraries\\win32\\.svn ^
/XD %cd%\\libraries\\win32\\wxwidgets\\include ^
/XD %cd%\\libraries\\win32\\wxwidgets\\lib ^
/XD .svn ^
/XD %NIGHTLY_PATH%\\binaries\\data\\mods\\mod\\shaders\\spirv ^
/XD %NIGHTLY_PATH%\\binaries\\data\\mods\\public\\shaders\\spirv ^
/MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0
"""
bat """
(robocopy binaries\\system ..\\nightly-svn\\binaries\\system ^
/XF *.exp ^
/XF *.lib ^
/MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0
"""
}
}
stage("Recompile SPIR-V shaders") {
when {
expression { buildSPIRV }
}
steps {
ws("workspace/nightly-svn") {
bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/mod source/tools/spirv/rules.json binaries/data/mods/mod"
bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/public source/tools/spirv/rules.json binaries/data/mods/public"
}
}
}
stage("Update translations") {
steps {
ws("workspace/nightly-svn") {
bat "cd source\\tools\\i18n && python updateTemplates.py"
withCredentials([string(credentialsId: 'TX_TOKEN', variable: 'TX_TOKEN')]) {
bat "cd source\\tools\\i18n && python pullTranslations.py"
}
bat "cd source\\tools\\i18n && python generateDebugTranslation.py --long"
bat "cd source\\tools\\i18n && python cleanTranslationFiles.py"
script { if (!params.NEW_REPO) {
bat "python source\\tools\\i18n\\checkDiff.py --verbose"
}}
bat "cd source\\tools\\i18n && python creditTranslators.py"
}
}
}
stage("Commit") {
steps {
ws("workspace/nightly-svn") {
bat "svn add --force ."
bat "(for /F \"tokens=* delims=! \" %%A in ('svn status ^| findstr /R \"^!\"') do (svn delete %%A)) || (echo No deleted files found) ^& exit 0"
withCredentials([usernamePassword(credentialsId: 'nightly-autobuild', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) {
script { env.GITHASH = gitHash
bat 'svn commit --username %SVNUSER% --password %SVNPASS% --no-auth-cache --non-interactive -m "Nightly build for %GITHASH% (%DATE%)"'
}
}
}
}
}
}
}

18
source/tools/i18n/checkDiff.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
# Copyright (C) 2021 Wildfire Games.
# Copyright (C) 2024 Wildfire Games.
# This file is part of 0 A.D.
#
# 0 A.D. is free software: you can redistribute it and/or modify
@ -29,9 +29,9 @@ def get_diff():
diff_process = subprocess.run(["svn", "diff", "binaries"], capture_output=True)
if diff_process.returncode != 0:
print(f"Error running svn diff: {diff_process.stderr.decode()}. Exiting.")
print(f"Error running svn diff: {diff_process.stderr.decode('utf-8')}. Exiting.")
return
return io.StringIO(diff_process.stdout.decode())
return io.StringIO(diff_process.stdout.decode('utf-8'))
def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
"""Run through a diff of .po files and check that some of the changes
@ -45,10 +45,10 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
l = diff.readline()
while l:
if l.startswith("Index: binaries"):
if not l.endswith(".pot\n") and not l.endswith(".po\n"):
if not l.strip().endswith(".pot") and not l.strip().endswith(".po"):
curfile = None
else:
curfile = l[7:-1]
curfile = l[7:].strip()
files.add(curfile)
# skip patch header
diff.readline()
@ -60,7 +60,7 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
if l[0] != '-' and l[0] != '+':
l = diff.readline()
continue
if l[1] == '\n' or (l[1] == '#' and l[2] == ":"):
if l[1:].strip() == "" or (l[1] == '#' and l[2] == ':'):
l = diff.readline()
continue
if "# Copyright (C)" in l or "POT-Creation-Date:" in l or "PO-Revision-Date" in l or "Last-Translator" in l:
@ -78,7 +78,7 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
def revert_files(files: List[str], verbose = False):
revert_process = subprocess.run(["svn", "revert"] + files, capture_output=True)
if revert_process.returncode != 0:
print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode()}")
print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode('utf-8')}")
if verbose:
for file in files:
print(f"Reverted {file}")
@ -96,9 +96,9 @@ def add_untracked(verbose = False):
continue
# Ignore non PO files. This is important so that the translator credits
# correctly be updated, note however the script assumes a pristine SVN otherwise.
if not line.endswith(".po") and not line.endswith(".pot"):
continue
file = line[1:].strip()
if not file.endswith(".po") and not file.endswith(".pot"):
continue
add_process = subprocess.run(["svn", "add", file, "--parents"], capture_output=True)
if add_process.stderr != b'':
print(f"Warning: file {file} could not be added.")

2
source/tools/i18n/cleanTranslationFiles.py
View File

@ -43,7 +43,7 @@ def main():
for file in files:
usernames = []
reached = False
for line in fileinput.input(file.replace("\\", "/"), inplace=True):
for line in fileinput.input(file.replace("\\", "/"), inplace=True, encoding="utf-8"):
if reached:
if line == "# \n":
line = ""

11
source/tools/i18n/get-nightly-translations.bat Normal file
View File

@ -0,0 +1,11 @@
rem **Download translations from the latest nightly build**
rem **This will overwrite any uncommitted changes to messages.json files**
svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/l10n ..\..\..\binaries\data\l10n
for %%m in (mod public) do (
svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/%%m/l10n ..\..\..\binaries\data\mods\%%m\l10n
)
svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/public/gui/credits/texts/translators.json ..\..\..\binaries\data\mods\public\gui\credits\texts\translators.json

15
source/tools/i18n/get-nightly-translations.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/sh
set -ev
# Download translations from the latest nightly build
# This will overwrite any uncommitted changes to messages.json files
cd "$(dirname $0)"
svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/l10n ../../../binaries/data/l10n
for m in "mod public"; do
svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/${m}/l10n ../../../binaries/data/mods/${m}/l10n
done
svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/public/gui/credits/texts/translators.json ../../../binaries/data/mods/public/gui/credits/texts/translators.json

32
source/tools/i18n/maintenanceTasks.sh
View File

@ -1,32 +0,0 @@
#!/bin/sh
# Regenerates the POT files, downloads the latest translations from Transifex,
# and prepares the commit of the updated POT and PO files.
SCRIPT_PATH="`dirname \"$0\"`"
# POT Generation ##############################################################
echo ":: Regenerating the translation templates…"
python3 "${SCRIPT_PATH}/updateTemplates.py"
# PO Download #################################################################
echo ":: Downloading translations from Transifex…"
python3 "${SCRIPT_PATH}/pullTranslations.py"
# Pre-Commit Cleanup #########################################################
echo ":: Removing unneeded data from the .po files…"
python3 "${SCRIPT_PATH}/cleanTranslationFiles.py"
echo ":: Reverting unnecessary changes…"
python3 "${SCRIPT_PATH}/checkDiff.py"
# Commit ######################################################################
echo ":: Done"
echo " Now you can commit your changes to the server."

4
source/tools/i18n/pullTranslations.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
# Copyright (C) 2023 Wildfire Games.
# Copyright (C) 2024 Wildfire Games.
# This file is part of 0 A.D.
#
# 0 A.D. is free software: you can redistribute it and/or modify
@ -29,7 +29,7 @@ def main():
path = os.path.join(root, folder)
os.chdir(path)
print(f"INFO: Starting to pull translations in {path}...")
subprocess.run(["tx", "pull", "-f"])
subprocess.run(["tx", "pull", "-a", "-f"])
if __name__ == "__main__":

5
source/tools/spirv/get-nightly-shaders.bat Normal file
View File

@ -0,0 +1,5 @@
rem **Download SPIR-V shaders from the latest nightly build**
for %%m in (mod public) do (
svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/%%m/shaders/spirv ..\..\..\binaries\data\mods\%%m\shaders\spirv
)

10
source/tools/spirv/get-nightly-shaders.sh Normal file
View File

@ -0,0 +1,10 @@
#!/bin/sh
set -ev
# Download SPIR-V shaders from the latest nightly build
cd "$(dirname $0)"
for m in "mod public"; do
svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/${m}/shaders/spirv ../../../binaries/data/mods/${m}/shaders/spirv
done