1
0
forked from 0ad/0ad

Update tinygettext. Fixes #2522.

Updates to upstream changeset f71a8e5e84bffadd61ac93768b332262989844fd,
with some additions
that will be submitted upstream (without the addition of precompiled.h
and win32 dirent).

This was SVN commit r16462.
This commit is contained in:
leper 2015-03-24 17:47:08 +00:00
parent 28abcb3bcc
commit c73c9370ee
17 changed files with 171 additions and 101 deletions

View File

@ -342,12 +342,14 @@ extern_lib_defs = {
if os.is("windows") then
add_default_include_paths("iconv")
defines { "HAVE_ICONV_CONST" }
defines { "ICONV_CONST=const" }
defines { "LIBICONV_STATIC" }
elseif os.is("macosx") then
add_default_include_paths("iconv")
defines { "LIBICONV_STATIC" }
elseif os.getversion().description == "FreeBSD" then
defines { "HAVE_ICONV_CONST" }
defines { "ICONV_CONST=const" }
end
end,
link_settings = function()

View File

@ -44,6 +44,9 @@ private:
std::string translate(const Entries& dict, const std::string& msgid);
std::string translate_plural(const Entries& dict, const std::string& msgid, const std::string& msgidplural, int num);
bool m_has_fallback;
Dictionary* m_fallback;
public:
/** Constructs a dictionary converting to the specified \a charset (default UTF-8) */
Dictionary(const std::string& charset = "UTF-8");
@ -80,7 +83,7 @@ public:
translate(). */
void add_translation(const std::string& msgid, const std::string& msgid_plural,
const std::vector<std::string>& msgstrs);
void add_translation(const std::string& msgctxt,
void add_translation(const std::string& msgctxt,
const std::string& msgid, const std::string& msgid_plural,
const std::vector<std::string>& msgstrs);
@ -92,7 +95,7 @@ public:
/** Iterate over all messages, Func is of type:
void func(const std::string& msgid, const std::vector<std::string>& msgstrs) */
template<class Func>
Func foreach(Func func)
Func foreach(Func func)
{
for(Entries::iterator i = entries.begin(); i != entries.end(); ++i)
{
@ -101,10 +104,16 @@ public:
return func;
}
void addFallback(Dictionary* fallback)
{
m_has_fallback = true;
m_fallback = fallback;
}
/** Iterate over all messages with a context, Func is of type:
void func(const std::string& ctxt, const std::string& msgid, const std::vector<std::string>& msgstrs) */
template<class Func>
Func foreach_ctxt(Func func)
Func foreach_ctxt(Func func)
{
for(CtxtEntries::iterator i = ctxt_entries.begin(); i != ctxt_entries.end(); ++i)
{
@ -115,6 +124,10 @@ public:
}
return func;
}
private:
Dictionary(const Dictionary&) = delete;
Dictionary& operator=(const Dictionary&) = delete;
};
} // namespace tinygettext

View File

@ -45,7 +45,7 @@ private:
std::string charset;
bool use_fuzzy;
Language current_language;
Dictionary* current_dict;
@ -86,6 +86,7 @@ public:
std::set<Language> get_languages();
void set_filesystem(std::unique_ptr<FileSystem> filesystem);
std::string convertFilename2Language(const std::string &s_in) const;
private:
DictionaryManager (const DictionaryManager&);

View File

@ -25,7 +25,7 @@
namespace tinygettext {
class FileSystem
class FileSystem
{
public:
virtual ~FileSystem() {}

View File

@ -5,12 +5,12 @@
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -26,20 +26,20 @@
# define tinygettext_iconv_t SDL_iconv_t
# define tinygettext_iconv SDL_iconv
# define tinygettext_iconv_open SDL_iconv_open
# define tinygettext_iconv_close SDL_iconv_close
# define tinygettext_iconv_close SDL_iconv_close
#else
# include <iconv.h>
# ifdef HAVE_ICONV_CONST
# define tinygettext_ICONV_CONST const
# define tinygettext_ICONV_CONST ICONV_CONST
# else
# define tinygettext_ICONV_CONST
# define tinygettext_ICONV_CONST
# endif
# define tinygettext_iconv_t iconv_t
# define tinygettext_iconv iconv
# define tinygettext_iconv_open iconv_open
# define tinygettext_iconv_close iconv_close
# define tinygettext_iconv_close iconv_close
#endif
namespace tinygettext {

View File

@ -36,12 +36,12 @@ private:
public:
/** Create a language from language and country code:
Example: Languge("de", "DE"); */
static Language from_spec(const std::string& language,
const std::string& country = std::string(),
static Language from_spec(const std::string& language,
const std::string& country = std::string(),
const std::string& modifier = std::string());
/** Create a language from language and country code:
Example: Languge("deutsch");
Example: Languge("deutsch");
Example: Languge("de_DE"); */
static Language from_name(const std::string& str);
@ -55,8 +55,8 @@ public:
/** Create an undefined Language object */
Language();
operator bool() const { return language_spec; }
explicit operator bool() const { return language_spec != NULL; }
/** Returns the language code (i.e. de, en, fr) */
std::string get_language() const;

View File

@ -45,7 +45,7 @@ private:
public:
Log(log_callback_t callback);
~Log();
std::ostream& get();
};

View File

@ -39,7 +39,7 @@ public:
{}
PluralForms(unsigned int nplural_, PluralFunc plural_)
: nplural(nplural_),
: nplural(nplural_),
plural(plural_)
{}
@ -49,13 +49,13 @@ public:
bool operator==(const PluralForms& other) { return nplural == other.nplural && plural == other.plural; }
bool operator!=(const PluralForms& other) { return !(*this == other); }
operator bool() const {
return plural;
explicit operator bool() const {
return plural != NULL;
}
};
} // namespace tinygettext
#endif
#endif
/* EOF */

View File

@ -42,7 +42,7 @@ private:
std::string current_line;
IConv conv;
POParser(const std::string& filename, std::istream& in_, Dictionary& dict_, bool use_fuzzy = true);
~POParser();
@ -53,7 +53,11 @@ private:
void get_string_line(std::ostringstream& str, size_t skip);
bool is_empty_line();
bool prefix(const char* );
#ifdef _WIN32
void error(const std::string& msg);
#else
void error(const std::string& msg) __attribute__((__noreturn__));
#endif
void warning(const std::string& msg);
public:

View File

@ -23,12 +23,14 @@
#include "tinygettext/dictionary.hpp"
namespace tinygettext {
Dictionary::Dictionary(const std::string& charset_) :
entries(),
ctxt_entries(),
charset(charset_),
plural_forms()
plural_forms(),
m_has_fallback(false),
m_fallback()
{
}
@ -70,12 +72,13 @@ Dictionary::translate_plural(const Entries& dict, const std::string& msgid, cons
{
unsigned int n = 0;
n = plural_forms.get_plural(count);
if(n >= msgstrs.size())
if (n >= msgstrs.size())
{
log_error << "Plural translation not available (and not set to empty): '" << msgid << "'" << std::endl;
log_error << "Missing plural form: " << n << std::endl;
return msgid;
}
}
if (!msgstrs[n].empty())
return msgstrs[n];
else
@ -115,8 +118,10 @@ Dictionary::translate(const Entries& dict, const std::string& msgid)
else
{
log_info << "Couldn't translate: " << msgid << std::endl;
return msgid;
}
if (m_has_fallback) return m_fallback->translate(msgid);
else return msgid;
}
}
std::string
@ -135,7 +140,7 @@ Dictionary::translate_ctxt(const std::string& msgctxt, const std::string& msgid)
}
std::string
Dictionary::translate_ctxt_plural(const std::string& msgctxt,
Dictionary::translate_ctxt_plural(const std::string& msgctxt,
const std::string& msgid, const std::string& msgidplural, int num)
{
CtxtEntries::iterator i = ctxt_entries.find(msgctxt);
@ -152,7 +157,7 @@ Dictionary::translate_ctxt_plural(const std::string& msgctxt,
return msgid;
}
}
void
Dictionary::add_translation(const std::string& msgid, const std::string& ,
const std::vector<std::string>& msgstrs)
@ -172,14 +177,14 @@ Dictionary::add_translation(const std::string& msgid, const std::string& msgstr)
}
else if (vec[0] != msgstr)
{
log_warning << "collision in add_translation: '"
log_warning << "collision in add_translation: '"
<< msgid << "' -> '" << msgstr << "' vs '" << vec[0] << "'" << std::endl;
vec[0] = msgstr;
}
}
void
Dictionary::add_translation(const std::string& msgctxt,
Dictionary::add_translation(const std::string& msgctxt,
const std::string& msgid, const std::string& msgid_plural,
const std::vector<std::string>& msgstrs)
{
@ -209,7 +214,7 @@ Dictionary::add_translation(const std::string& msgctxt, const std::string& msgid
vec[0] = msgstr;
}
}
} // namespace tinygettext
/* EOF */

View File

@ -77,7 +77,7 @@ DictionaryManager::get_dictionary()
{
if (current_dict)
{
return *current_dict;
return *current_dict;
}
else
{
@ -100,7 +100,7 @@ DictionaryManager::get_dictionary(const Language& language)
//log_debug << "...normalized as \"" << lang << "\"" << std::endl;
assert(language);
Dictionaries::iterator i = dictionaries.find(language);
Dictionaries::iterator i = dictionaries.find(language);
if (i != dictionaries.end())
{
return *i->second;
@ -124,7 +124,8 @@ DictionaryManager::get_dictionary(const Language& language)
// check if filename matches requested language
if (has_suffix(*filename, ".po"))
{ // ignore anything that isn't a .po file
Language po_language = Language::from_env(filename->substr(0, filename->size()-3));
Language po_language = Language::from_env(convertFilename2Language(*filename));
if (!po_language)
{
@ -133,7 +134,7 @@ DictionaryManager::get_dictionary(const Language& language)
else
{
int score = Language::match(language, po_language);
if (score > best_score)
{
best_score = score;
@ -142,11 +143,11 @@ DictionaryManager::get_dictionary(const Language& language)
}
}
}
if (!best_filename.empty())
{
std::string pofile = *p + "/" + best_filename;
try
try
{
std::unique_ptr<std::istream> in = filesystem->open_file(pofile);
if (!in.get())
@ -158,7 +159,7 @@ DictionaryManager::get_dictionary(const Language& language)
POParser::parse(pofile, *in, *dict);
}
}
catch(std::exception& e)
catch(std::exception& e)
{
log_error << "error: failure parsing: " << pofile << std::endl;
log_error << e.what() << "" << std::endl;
@ -166,6 +167,11 @@ DictionaryManager::get_dictionary(const Language& language)
}
}
if (language.get_country().size() > 0)
{
printf("Adding language fallback %s\n", language.get_language().c_str());
dict->addFallback( &get_dictionary(Language::from_spec(language.get_language())) );
}
return *dict;
}
}
@ -181,7 +187,7 @@ DictionaryManager::get_languages()
for(std::vector<std::string>::iterator file = files.begin(); file != files.end(); ++file)
{
if (has_suffix(*file, ".po"))
if (has_suffix(*file, ".po"))
{
languages.insert(Language::from_env(file->substr(0, file->size()-3)));
}
@ -238,7 +244,43 @@ DictionaryManager::set_filesystem(std::unique_ptr<FileSystem> filesystem_)
{
filesystem = std::move(filesystem_);
}
// ----------------------------------------------------------------------------
/** This function converts a .po filename (e.g. zh_TW.po) into a language
* specification (zh_TW). On case insensitive file systems (think windows)
* the filename and therefore the country specification is lower case
* (zh_tw). It Converts the lower case characters of the country back to
* upper case, otherwise tinygettext does not identify the country
* correctly.
*/
std::string DictionaryManager::convertFilename2Language(const std::string &s_in) const
{
std::string s;
if(s_in.substr(s_in.size()-3, 3)==".po")
s = s_in.substr(0, s_in.size()-3);
else
s = s_in;
bool underscore_found = false;
for(unsigned int i=0; i<s.size(); i++)
{
if(underscore_found)
{
// If we get a non-alphanumerical character/
// we are done (en_GB.UTF-8) - only convert
// the 'gb' part ... if we ever get this kind
// of filename.
if(!::isalpha(s[i]))
break;
s[i] = static_cast<char>(::toupper(s[i]));
}
else
underscore_found = s[i]=='_';
}
return s;
} // convertFilename2Language
} // namespace tinygettext
/* EOF */

View File

@ -31,15 +31,15 @@
namespace tinygettext {
#ifndef tinygettext_ICONV_CONST
# define tinygettext_ICONV_CONST
# define tinygettext_ICONV_CONST
#endif
IConv::IConv()
IConv::IConv()
: to_charset(),
from_charset(),
cd(0)
{}
IConv::IConv(const std::string& from_charset_, const std::string& to_charset_)
: to_charset(),
from_charset(),
@ -47,13 +47,13 @@ IConv::IConv(const std::string& from_charset_, const std::string& to_charset_)
{
set_charsets(from_charset_, to_charset_);
}
IConv::~IConv()
{
if (cd)
tinygettext_iconv_close(cd);
}
void
IConv::set_charsets(const std::string& from_charset_, const std::string& to_charset_)
{
@ -112,8 +112,8 @@ IConv::convert(const std::string& text)
// a std::string
tinygettext_ICONV_CONST char* inbuf = const_cast<char*>(&text[0]);
std::string result(outbytesleft, 'X');
char* outbuf = &result[0];
char* outbuf = &result[0];
// Try to convert the text.
size_t ret = tinygettext_iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
if (ret == static_cast<size_t>(-1))

View File

@ -22,9 +22,10 @@
#include <assert.h>
#include <unordered_map>
#include <vector>
#include <algorithm>
namespace tinygettext {
struct LanguageSpec {
/** Language code: "de", "en", ... */
const char* language;
@ -38,7 +39,7 @@ struct LanguageSpec {
/** Language name: "German", "English", "French", ... */
const char* name;
};
/** Language Definitions */
//*{
static const LanguageSpec languages[] = {
@ -213,7 +214,7 @@ static const LanguageSpec languages[] = {
{ "pl", "PL", 0, "Polish (Poland)" },
{ "ps", 0, 0, "Pashto" },
{ "pt", 0, 0, "Portuguese" },
{ "pt", "BR", 0, "Brazilian" },
{ "pt", "BR", 0, "Portuguese (Brazil)" },
{ "pt", "PT", 0, "Portuguese (Portugal)" },
{ "qu", 0, 0, "Quechua" },
{ "rm", 0, 0, "Rhaeto-Romance" },
@ -283,7 +284,7 @@ static const LanguageSpec languages[] = {
{ NULL, 0, 0, NULL }
};
//*}
std::string
resolve_language_alias(const std::string& name)
{
@ -352,7 +353,7 @@ resolve_language_alias(const std::string& name)
name_lowercase[i] = static_cast<char>(tolower(name[i]));
Aliases::iterator i = language_aliases.find(name_lowercase);
if (i != language_aliases.end())
if (i != language_aliases.end())
{
return i->second;
}
@ -361,19 +362,20 @@ resolve_language_alias(const std::string& name)
return name;
}
}
Language
Language::from_spec(const std::string& language, const std::string& country, const std::string& modifier)
{
static std::unordered_map<std::string, std::vector<const LanguageSpec*> > language_map;
typedef std::unordered_map<std::string, std::vector<const LanguageSpec*> > LanguageSpecMap;
static LanguageSpecMap language_map;
if (language_map.empty())
{ // Init language_map
for(int i = 0; languages[i].language != NULL; ++i)
language_map[languages[i].language].push_back(&languages[i]);
}
std::unordered_map<std::string, std::vector<const LanguageSpec*> >::iterator i = language_map.find(language);
LanguageSpecMap::iterator i = language_map.find(language);
if (i != language_map.end())
{
std::vector<const LanguageSpec*>& lst = i->second;
@ -383,7 +385,7 @@ Language::from_spec(const std::string& language, const std::string& country, con
tmpspec.country = country.c_str();
tmpspec.modifier = modifier.c_str();
Language tmplang(&tmpspec);
const LanguageSpec* best_match = 0;
int best_match_score = 0;
for(std::vector<const LanguageSpec*>::iterator j = lst.begin(); j != lst.end(); ++j)
@ -445,7 +447,7 @@ Language::from_env(const std::string& env)
return from_spec(language, country, modifier);
}
Language::Language(const LanguageSpec* language_spec_)
: language_spec(language_spec_)
{
@ -471,7 +473,7 @@ Language::match(const Language& lhs, const Language& rhs)
{ 7, 6, 3 }, // country wildcard
{ 4, 2, 1 }, // country miss
};
int c;
if (lhs.get_country() == rhs.get_country())
c = 0;
@ -479,7 +481,7 @@ Language::match(const Language& lhs, const Language& rhs)
c = 1;
else
c = 2;
int m;
if (lhs.get_modifier() == rhs.get_modifier())
m = 0;
@ -565,7 +567,7 @@ Language::operator!=(const Language& rhs) const
{
return language_spec != rhs.language_spec;
}
} // namespace tinygettext
/* EOF */

View File

@ -21,11 +21,11 @@
#include "tinygettext/log.hpp"
namespace tinygettext {
Log::log_callback_t Log::log_info_callback = &Log::default_log_callback;
Log::log_callback_t Log::log_warning_callback = &Log::default_log_callback;
Log::log_callback_t Log::log_error_callback = &Log::default_log_callback;
void
Log::default_log_callback(const std::string& str)
{
@ -49,24 +49,24 @@ Log::set_log_error_callback(log_callback_t callback)
{
log_error_callback = callback;
}
Log::Log(log_callback_t callback_) :
callback(callback_),
out()
{
}
Log::~Log()
Log::~Log()
{
callback(out.str());
}
std::ostream&
Log::get()
Log::get()
{
return out;
return out;
}
} // namespace tinygettext
/* EOF */

View File

@ -22,15 +22,15 @@
#include <unordered_map>
namespace tinygettext {
/**
/**
* Plural functions are used to select a string that matches a given
* count. \a n is the count and the return value is the string index
* used in the .po file, for example:
*
*
* msgstr[0] = "You got %d error";
* msgstr[1] = "You got %d errors";
* ^-- return value of plural function
* msgstr[1] = "You got %d errors";
* ^-- return value of plural function
*/
unsigned int plural1(int ) { return 0; }
unsigned int plural2_1(int n) { return (n != 1); }
@ -45,12 +45,13 @@ unsigned int plural3_pl(int n) { return static_cast<unsigned int>(n==1 ? 0 : n%1
unsigned int plural3_sl(int n) { return static_cast<unsigned int>(n%100==1 ? 0 : n%100==2 ? 1 : n%100==3 || n%100==4 ? 2 : 3); }
unsigned int plural4_ar(int n) { return static_cast<unsigned int>( n==1 ? 0 : n==2 ? 1 : n>=3 && n<=10 ? 2 : 3 ); }
unsigned int plural4_gd(int n) { return static_cast<unsigned int>( n==1 || n==11) ? 0 : (n==2 || n==12) ? 1 : (n > 2 && n < 20) ? 2 : 3; }
PluralForms
PluralForms::from_string(const std::string& str)
{
static std::unordered_map<std::string, PluralForms> plural_forms;
typedef std::unordered_map<std::string, PluralForms> PluralFormsMap;
static PluralFormsMap plural_forms;
if (plural_forms.empty())
{
// Note that the plural forms here shouldn't contain any spaces
@ -70,14 +71,14 @@ PluralForms::from_string(const std::string& str)
plural_forms["Plural-Forms:nplurals=4;plural=n==1?0:n==2?1:n>=3&&n<=10?2:3;"]=PluralForms(4, plural4_ar);
plural_forms["Plural-Forms:nplurals=4;plural=(n==1||n==11)?0:(n==2||n==12)?1:(n>2&&n<20)?2:3;"]=PluralForms(4, plural4_gd);
}
// Remove spaces from string before lookup
std::string space_less_str;
for(std::string::size_type i = 0; i < str.size(); ++i)
if (!isspace(str[i]))
space_less_str += str[i];
std::unordered_map<std::string, PluralForms>::const_iterator it= plural_forms.find(space_less_str);
PluralFormsMap::const_iterator it= plural_forms.find(space_less_str);
if (it != plural_forms.end())
{
return it->second;

View File

@ -36,26 +36,26 @@
namespace tinygettext {
bool POParser::pedantic = true;
void
POParser::parse(const std::string& filename, std::istream& in, Dictionary& dict)
{
POParser parser(filename, in, dict);
parser.parse();
}
class POParserError {};
POParser::POParser(const std::string& filename_, std::istream& in_, Dictionary& dict_, bool use_fuzzy_) :
filename(filename_),
in(in_),
dict(dict_),
filename(filename_),
in(in_),
dict(dict_),
use_fuzzy(use_fuzzy_),
running(false),
eof(false),
running(false),
eof(false),
big5(false),
line_number(0),
current_line(),
line_number(0),
current_line(),
conv()
{
}
@ -100,7 +100,7 @@ POParser::get_string_line(std::ostringstream& out, size_t skip)
if (current_line[skip] != '"')
error("expected start of string '\"'");
std::string::size_type i;
for(i = skip+1; current_line[i] != '\"'; ++i)
{
@ -109,10 +109,10 @@ POParser::get_string_line(std::ostringstream& out, size_t skip)
out << current_line[i];
i += 1;
if (i >= current_line.size())
error("invalid big5 encoding");
out << current_line[i];
}
else if (i >= current_line.size())
@ -136,7 +136,7 @@ POParser::get_string_line(std::ostringstream& out, size_t skip)
case 'r': out << '\r'; break;
case '"': out << '"'; break;
case '\\': out << '\\'; break;
default:
default:
std::ostringstream err;
err << "unhandled escape '\\" << current_line[i] << "'";
warning(err.str());
@ -198,7 +198,7 @@ POParser::get_string(unsigned int skip)
skip += 1;
}
}
next:
next_line();
for(std::string::size_type i = 0; i < current_line.size(); ++i)
@ -247,7 +247,7 @@ POParser::parse_header(const std::string& header)
if (has_prefix(line, "Content-Type:"))
{
// from_charset = line.substr(len);
unsigned int len = strlen("Content-Type: text/plain; charset=");
size_t len = strlen("Content-Type: text/plain; charset=");
if (line.compare(0, len, "Content-Type: text/plain; charset=") == 0)
{
from_charset = line.substr(len);
@ -348,7 +348,7 @@ POParser::parse()
// Parser structure
while(!eof)
{
try
try
{
bool fuzzy = false;
bool has_msgctxt = false;
@ -393,7 +393,7 @@ POParser::parse()
error("expected 'msgstr[N] (0 <= N <= 9)'");
}
else if (prefix("msgstr[") &&
current_line.size() > 8 &&
current_line.size() > 8 &&
isdigit(current_line[7]) && current_line[8] == ']')
{
unsigned int number = static_cast<unsigned int>(current_line[7] - '0');
@ -408,7 +408,7 @@ POParser::parse()
msgstr_num[number] = conv.convert(msgstr);
goto next;
}
else
else
{
error("expected 'msgstr[N]'");
}
@ -481,14 +481,14 @@ POParser::parse()
error("expected 'msgstr' or 'msgid_plural'");
}
}
if (!is_empty_line())
error("expected empty line");
next_line();
}
catch(POParserError&)
{
{
}
}
}

View File

@ -59,7 +59,7 @@ UnixFileSystem::open_directory(const std::string& pathname)
return files;
}
}
std::unique_ptr<std::istream>
UnixFileSystem::open_file(const std::string& filename)
{