diff --git a/mk/linux/mg_setsvnprops.sh b/mk/linux/mg_setsvnprops.sh index a1bd5b767..c8a8e26a8 100755 --- a/mk/linux/mg_setsvnprops.sh +++ b/mk/linux/mg_setsvnprops.sh @@ -6,6 +6,14 @@ CURRENTDIR="$(dirname $(readlink -f $0))" +# cpp /.c / .h files +find ${CURRENTDIR}/../../source/ -iname '*.cpp' -exec svn propset svn:mime-type text/plain '{}' \; +find ${CURRENTDIR}/../../source/ -iname '*.cpp' -exec svn propset svn:eol-style native '{}' \; +find ${CURRENTDIR}/../../source/ -iname '*.c' -exec svn propset svn:mime-type text/plain '{}' \; +find ${CURRENTDIR}/../../source/ -iname '*.c' -exec svn propset svn:eol-style native '{}' \; +find ${CURRENTDIR}/../../source/ -iname '*.h' -exec svn propset svn:mime-type text/plain '{}' \; +find ${CURRENTDIR}/../../source/ -iname '*.h' -exec svn propset svn:eol-style native '{}' \; + # LNG files #find ${CURRENTDIR}/../../data/glest_game/ -name "*\.lng" -exec echo {} \; find ${CURRENTDIR}/../../data/glest_game/ -iname '*.lng' -exec svn propset svn:mime-type text/plain '{}' \; diff --git a/source/glest_game/global/lang.cpp b/source/glest_game/global/lang.cpp index 0de4ea167..4f95b5cb8 100644 --- a/source/glest_game/global/lang.cpp +++ b/source/glest_game/global/lang.cpp @@ -101,6 +101,10 @@ void Lang::loadGameStrings(string uselanguage, bool loadFonts, Font::fontIsRightToLeft = strToBool(lang.getString("FONT_RIGHTTOLEFT")); } + if( lang.hasString("FONT_RIGHTTOLEFT_MIXED_SUPPORT")) { + Font::fontSupportMixedRightToLeft = strToBool(lang.getString("FONT_RIGHTTOLEFT_MIXED_SUPPORT")); + } + if( lang.hasString("MEGAGLEST_FONT")) { //setenv("MEGAGLEST_FONT","/usr/share/fonts/truetype/ttf-japanese-gothic.ttf",0); // Japanese #if defined(WIN32) diff --git a/source/glest_game/main/main.cpp b/source/glest_game/main/main.cpp index 1f8e7a36b..d1156881d 100644 --- a/source/glest_game/main/main.cpp +++ b/source/glest_game/main/main.cpp @@ -4173,6 +4173,7 @@ int glestMain(int argc, char** argv) { Shared::Graphics::Font::scaleFontValue = config.getFloat("FONT_SCALE_SIZE",floatToStr(Shared::Graphics::Font::scaleFontValue).c_str()); Shared::Graphics::Font::scaleFontValueCenterHFactor = config.getFloat("FONT_SCALE_CENTERH_FACTOR",floatToStr(Shared::Graphics::Font::scaleFontValueCenterHFactor).c_str()); Shared::Graphics::Font::langHeightText = config.getString("FONT_HEIGHT_TEXT",Shared::Graphics::Font::langHeightText.c_str()); + Shared::Graphics::Font::fontSupportMixedRightToLeft = config.getBool("FONT_RIGHTTOLEFT_MIXED_SUPPORT",intToStr(Shared::Graphics::Font::fontSupportMixedRightToLeft).c_str()); // Example values: // DEFAULT_CHARSET (English) = 1 diff --git a/source/shared_lib/include/graphics/font.h b/source/shared_lib/include/graphics/font.h index 463ff0cdb..dc11bf209 100644 --- a/source/shared_lib/include/graphics/font.h +++ b/source/shared_lib/include/graphics/font.h @@ -13,6 +13,7 @@ #define _SHARED_GRAPHICS_FONT_H_ #include +#include #include "font_text.h" #include "leak_dumper.h" @@ -69,6 +70,7 @@ public: static bool forceLegacyFonts; static bool forceFTGLFonts; static bool fontIsRightToLeft; + static bool fontSupportMixedRightToLeft; static float scaleFontValue; static float scaleFontValueCenterHFactor; static int baseSize; @@ -110,6 +112,7 @@ public: int getSize() const; void setSize(int size); + static std::vector > extract_mixed_LTR_RTL_map(string &str_); static void bidi_cvt(string &str_); static void resetToDefaults(); diff --git a/source/shared_lib/sources/graphics/font.cpp b/source/shared_lib/sources/graphics/font.cpp index e14ec1e6e..a4db7f7a2 100644 --- a/source/shared_lib/sources/graphics/font.cpp +++ b/source/shared_lib/sources/graphics/font.cpp @@ -55,6 +55,7 @@ bool Font::fontIsMultibyte = false; bool Font::forceLegacyFonts = false; bool Font::fontIsRightToLeft = false; bool Font::forceFTGLFonts = false; +bool Font::fontSupportMixedRightToLeft = false; // This value is used to scale the font text rendering // in 3D render mode @@ -76,7 +77,7 @@ void Font::resetToDefaults() { Font::fontIsMultibyte = false; //Font::forceLegacyFonts = false; Font::fontIsRightToLeft = false; - + Font::fontSupportMixedRightToLeft = false; // This value is used to scale the font text rendering // in 3D render mode Font::scaleFontValue = 0.80f; @@ -309,8 +310,225 @@ void Font::setSize(int size) { } } +bool is_non_ASCII(const int &c) { + return (c < 0) || (c >= 128); +} +bool is_ASCII(const int &c) { + return !is_non_ASCII(c); +} + +bool prev_word_is_ASCII(const string &str_,int end_index) { + bool result = false; + if(end_index < 0) { + //printf("Line: %d str [%s] end_index: %d\n",__LINE__,str_.substr(end_index).c_str(),end_index); + return result; + } + int start_index = end_index; + + //printf("Line: %d str [%s] end_index: %d word [%s]\n",__LINE__,str_.c_str(),end_index,str_.substr(end_index).c_str()); + + for (; start_index >= 0; --start_index) { + if(str_[start_index] == ' ') { + start_index++; + break; + } +// if(str_.substr(start_index,2) == "\\n") { +// start_index+=2; +// break; +// } + } + if(start_index < 0) { + start_index = 0; + } + //printf("Line: %d start_index: %d end_index: %d\n",__LINE__,start_index,end_index); + if(end_index >= 0) { + if(start_index == end_index) { + // another space + // !!! not sure what to do! + //printf("Line: %d [%s]\n",__LINE__,str_.substr(start_index).c_str()); + + if(str_[start_index] == ' ') { + return prev_word_is_ASCII(str_,start_index-1); + } + else { + return isalnum(str_[start_index]); + } + } + else { + int length = end_index-start_index+1; + string word = str_.substr(start_index,length); + //printf("Line: %d word [%s] length: %d\n",__LINE__,word.c_str(),length); + for(int index = 0; index < word.size(); ++index) { + //printf("%c = %d,",word[index],isalnum(word[index])); + if(isalnum(word[index])) { + //printf("Prev %c = %d [%d] [%s],",word[index],isalnum(word[index]),index,(index > 0 ? word.substr(index-1,2).c_str() : "n/a")); +// if(index > 0 && word.substr(index-1,2) == "\\n") { +// continue; +// } + + result = true; + break; + } + } + //printf("Line: %d result = %d\n",__LINE__,result); + } + } + return result; +} + +bool next_word_is_ASCII(const string &str_,int start_index) { + bool result = false; + if(start_index >= str_.size()) { + //printf("Line: %d str [%s] start_index: %d\n",__LINE__,str_.substr(start_index).c_str(),start_index); + return result; + } + + int end_index = start_index; + + //printf("Line: %d str [%s] start_index: %d\n",__LINE__,str_.c_str(),start_index); + + for (; end_index < str_.size(); ++end_index) { + if(str_[end_index] == ' ') { + end_index--; + break; + } +// if(str_.substr(end_index,2) == "\\n") { +// end_index-=2; +// break; +// } + + } + if(end_index >= str_.size()) { + end_index = str_.size()-1; + } + + //printf("Line: %d start_index: %d end_index: %d\n",__LINE__,start_index,end_index); + if(start_index >= 0) { + if(start_index == end_index) { + // another space + // !!! not sure what to do! + //printf("Line: %d [%s]\n",__LINE__,str_.substr(start_index).c_str()); + + if(str_[start_index] == ' ') { + return next_word_is_ASCII(str_,end_index+1); + } + else { + return isalnum(str_[start_index]); + } + + } + else { + int length = end_index-start_index+1; + string word = str_.substr(start_index,length); + //printf("Line: %d word [%s] length: %d\n",__LINE__,word.c_str(),length); + int alphaCount = 0; + for(int index = 0; index < word.size(); ++index) { + //printf("%c = %d,",word[index],isalnum(word[index])); + if(isalnum(word[index])) { + //printf("Next %c = %d [%d] [%s],",word[index],isalnum(word[index]),index,(index > 0 ? word.substr(index-1,2).c_str() : "n/a")); +// if(index > 0 && word.substr(index-1,2) == "\\n") { +// continue; +// } + result = true; + break; + } + } + //printf("Line: %d result = %d\n",__LINE__,result); + } + } + return result; +} + +vector > Font::extract_mixed_LTR_RTL_map(string &str_) { + vector > ascii_char_map; + +// replaceAll(str_, "\\n", " \\n "); + for (int index = 0; index < str_.size(); ++index) { + if(is_ASCII(str_[index]) == true) { + if(str_[index] == ' ') { + // Check both sides of the space to see what to do with it + if(prev_word_is_ASCII(str_,index-1) == false) { + //printf("#1 Prev Skip %d [%s]\n",index,str_.substr(index).c_str()); + + if(next_word_is_ASCII(str_,index+1) == false) { + //printf("#2 Prev Skip %d [%s]\n",index,str_.substr(index).c_str()); + //printf("#1 Keep %d [%s]\n",index,str_.substr(index).c_str()); + continue; + } + } +// if(next_word_is_ASCII(str_,index+1) == false) { +// //printf("Next Skip %d [%s]\n",index,str_.substr(index).c_str()); +// //printf("#2 Keep %d [%s]\n",index,str_.substr(index).c_str()); +// continue; +// } + } +// else if(str_.substr(index,2) == "\\n" || +// (index-1 >= 0 && str_.substr(index-1,2) == "\\n")) { +//// +//// //printf("Next Skip %d [%s]\n",index,str_.substr(index).c_str()); +//// //printf("#3 Keep %d [%s]\n",index,str_.substr(index).c_str()); +//// +// //printf("Newline Skip %d [%s]\n",index,str_.substr(index).c_str()); +// continue; +// } + // previous character is a space + else if(index-1 >= 0 && str_[index-1]== ' ') { + if(index+1 < str_.size() && str_[index+1] != ' ' && + next_word_is_ASCII(str_,index+1) == false) { + //printf("Next Skip %d [%s]\n",index,str_.substr(index).c_str()); + //printf("#3 Keep %d [%s]\n",index,str_.substr(index).c_str()); + continue; + } + } + // next character is a space + else if(index+1 < str_.size() && str_[index+1] == ' ') { + if(index-1 >= 0 && str_[index-1] != ' ' && + prev_word_is_ASCII(str_,index-1) == false) { + //printf("Next Skip %d [%s]\n",index,str_.substr(index).c_str()); + //printf("#4 Keep %d [%s]\n",index,str_.substr(index).c_str()); + continue; + } + } + else if(index-1 >= 0 && prev_word_is_ASCII(str_,index-1) == false) { +// //printf("Next Skip %d [%s]\n",index,str_.substr(index).c_str()); + //printf("#5 Keep %d [%s] alpha: %d\n",index,str_.substr(index).c_str(),isalnum(str_[index-1])); + if(index+1 < str_.size() && next_word_is_ASCII(str_,index+1) == false) { + continue; + } + else if(index+1 >= str_.size()) { + continue; + } + } + else if(index+1 < str_.size() && next_word_is_ASCII(str_,index+1) == false) { +// +// //printf("Next Skip %d [%s]\n",index,str_.substr(index).c_str()); + //printf("#6 Keep %d [%s] alpha: %d\n",index,str_.substr(index).c_str(),isalnum(str_[index+1])); + if(index-1 >= 0 && prev_word_is_ASCII(str_,index-1) == false) { + continue; + } + else if(index-1 < 0) { + continue; + } + } + } + else { + //printf("#5 Keep %d [%s]\n",index,str_.substr(index).c_str()); + continue; + } + //printf("Removal %d [%s]\n",index,str_.substr(index).c_str()); + ascii_char_map.push_back(make_pair(str_[index],index)); + } + + for (int index = ascii_char_map.size()-1; index >= 0; --index) { + str_.erase(ascii_char_map[index].second,1); + } + + return ascii_char_map; +} + void Font::bidi_cvt(string &str_) { +/* #ifdef HAVE_FRIBIDI char *c_str = const_cast(str_.c_str()); // fribidi forgot const... FriBidiStrIndex len = (int)str_.length(); @@ -351,7 +569,133 @@ void Font::bidi_cvt(string &str_) { delete[] bidi_visual; delete[] utf8str; #endif +*/ +#ifdef HAVE_FRIBIDI + + //printf("BEFORE: [%s]\n",str_.c_str()); + + string new_value = ""; + bool hasSoftNewLines = false; + bool hasHardNewLines = false; + vector lines; + + if(str_.find("\\n") != str_.npos) { + Tokenize(str_,lines,"\\n"); + hasSoftNewLines = true; + } + else if(str_.find("\n") != str_.npos) { + Tokenize(str_,lines,"\n"); + hasHardNewLines = true; + } + else { + lines.push_back(str_); + } + + for(int lineIndex = 0; lineIndex < lines.size(); ++lineIndex) { + if(new_value != "") { + if(hasSoftNewLines == true) { + new_value += "\\n"; + } + else if(hasHardNewLines == true) { + new_value += "\n"; + } + } + str_ = lines[lineIndex]; + //printf("Line: %d [%s]\n",lineIndex,str_.c_str()); + + vector > ascii_char_map; + if(Font::fontSupportMixedRightToLeft == true) { + ascii_char_map = extract_mixed_LTR_RTL_map(str_); + } + + //FriBidi C string holding the original text (that is probably with logical hebrew) + FriBidiChar *logical = NULL; + //FriBidi C string for the output text (that should be visual hebrew) + FriBidiChar *visual = NULL; + + FriBidiStrIndex *ltov = NULL; + FriBidiStrIndex *vtol = NULL; + + //C string holding the originall text (not nnecessarily as unicode) + char *ip = NULL; + //C string for the output text (not necessarily as unicode) + char *op = NULL; + + //Size to allocate for the char arrays + int size = str_.size() + 2; + + //Allocate memory: + //It's probably way too much, but at least it's not too little + logical = new FriBidiChar[size * 3]; + visual = new FriBidiChar[size * 3]; + ip = new char[size * 3]; + op = new char[size * 3]; + + ltov = new FriBidiStrIndex[size * 3]; + vtol = new FriBidiStrIndex[size * 3]; + + FriBidiCharType base; + size_t len, orig_len; + + //A bool type to see if conversion succeded + fribidi_boolean log2vis; + + //Holds information telling fribidi to use UTF-8 + FriBidiCharSet char_set_num; + char_set_num = fribidi_parse_charset ("UTF-8"); + + //Copy the given to string into the ip string + strcpy(ip, str_.c_str()); + + //Find length of originall text + orig_len = len = strlen( ip ); + + //Insert ip to logical as unicode (and find it's size now) + len = fribidi_charset_to_unicode (char_set_num, ip, len, logical); + + base = FRIBIDI_TYPE_ON; + + //printf("STRIPPED: [%s]\n",str_.c_str()); + + //Convert logical text to visual + log2vis = fribidi_log2vis (logical, len, &base, /* output: */ visual, ltov, vtol, NULL); + + //If convertion was successful + if(log2vis) + { + //Remove bidi marks (that we don't need) from the output text + len = fribidi_remove_bidi_marks (visual, len, ltov, vtol, NULL); + + //Convert unicode string back to the encoding the input string was in + fribidi_unicode_to_charset ( char_set_num, visual, len ,op); + + //Insert the output string into the result + str_ = op; + + //printf("LOG2VIS: [%s]\n",str_.c_str()); + + if(ascii_char_map.empty() == false) { + for (int index = 0; index < (int)ascii_char_map.size(); ++index) { + str_.insert(ascii_char_map[index].second,1,ascii_char_map[index].first); + } + } + //printf("AFTER: [%s]\n",str_.c_str()); + } + + //Free allocated memory + delete [] ltov; + delete [] visual; + delete [] logical; + delete [] ip; + delete [] op; + + new_value += str_; + } + str_ = new_value; + //printf("NEW: [%s]\n",str_.c_str()); + +#endif /* string out = "" ; diff --git a/source/shared_lib/sources/platform/common/platform_common.cpp b/source/shared_lib/sources/platform/common/platform_common.cpp index 4e0eb37d9..ca07284af 100644 --- a/source/shared_lib/sources/platform/common/platform_common.cpp +++ b/source/shared_lib/sources/platform/common/platform_common.cpp @@ -261,7 +261,7 @@ void Tokenize(const string& str,vector& tokens,const string& delimiters) break; } tokens.push_back( string( textLine.substr( pos, nextPos - pos ) ) ); - pos = nextPos + 1; + pos = nextPos + delimiters.size(); } } diff --git a/source/tests/CMakeLists.txt b/source/tests/CMakeLists.txt index 2512d52ff..dca63da2c 100644 --- a/source/tests/CMakeLists.txt +++ b/source/tests/CMakeLists.txt @@ -21,6 +21,17 @@ IF(BUILD_MEGAGLEST_TESTS) SET(EXTERNAL_LIBS ${EXTERNAL_LIBS} ${SDL_LIBRARY}) ENDIF() + find_package( FriBiDi ) + if(ENABLE_FRIBIDI AND FRIBIDI_LIBRARIES) + add_definitions(-DHAVE_FRIBIDI) + + include_directories( ${FRIBIDI_INCLUDE_DIR} ) + SET(EXTERNAL_LIBS ${EXTERNAL_LIBS} ${FRIBIDI_LIBRARIES}) + + elseif(ENABLE_FRIBIDI AND NOT FRIBIDI_LIBRARIES) + message("Could not find FriBiDi. Disabling FriBiDi support.") + endif() + ######################################################################################### # megaglest test code diff --git a/source/tests/shared_lib/graphics/font_test.cpp b/source/tests/shared_lib/graphics/font_test.cpp new file mode 100644 index 000000000..4614632c8 --- /dev/null +++ b/source/tests/shared_lib/graphics/font_test.cpp @@ -0,0 +1,138 @@ +// ============================================================== +// This file is part of MegaGlest Unit Tests (www.megaglest.org) +// +// Copyright (C) 2013 Mark Vejvoda +// +// You can redistribute this code and/or modify it under +// the terms of the GNU General Public License as published +// by the Free Software Foundation; either version 2 of the +// License, or (at your option) any later version +// ============================================================== + +#include +#include +#include "font.h" +#include +#include + +#ifdef WIN32 +#include +#else +#include +#endif + +using namespace Shared::Graphics; + +// +// Tests for font class +// +class FontTest : public CppUnit::TestFixture { + // Register the suite of tests for this fixture + CPPUNIT_TEST_SUITE( FontTest ); + + CPPUNIT_TEST( test_LTR_RTL_Mixed ); + + CPPUNIT_TEST_SUITE_END(); + // End of Fixture registration + +public: + + void test_LTR_RTL_Mixed() { + Font::fontSupportMixedRightToLeft = true; + + string IntroText1 = "מבוסס על \"award-winning classic Glest\""; + string expected = IntroText1; + CPPUNIT_ASSERT_EQUAL( 45,(int)IntroText1.size() ); + + std::vector > result = Font::extract_mixed_LTR_RTL_map(IntroText1); + //CPPUNIT_ASSERT_EQUAL( 30, (int)result.size() ); + +#ifdef HAVE_FRIBIDI + IntroText1 = expected; + Font::bidi_cvt(IntroText1); + + CPPUNIT_ASSERT_EQUAL( 45,(int)IntroText1.size() ); + CPPUNIT_ASSERT_EQUAL( string("לע ססובמ"),IntroText1.substr(0, 15) ); + CPPUNIT_ASSERT_EQUAL( string("\"award-winning classic Glest\""),IntroText1.substr(16) ); +#endif + + string LuaDisableSecuritySandbox = "בטל אבטחת ארגז חול של Lua"; + LuaDisableSecuritySandbox.insert(18,"\""); + LuaDisableSecuritySandbox.insert(34,"\""); + //printf("Result1: [%s]\n",LuaDisableSecuritySandbox.c_str()); + string expected2 = LuaDisableSecuritySandbox; + CPPUNIT_ASSERT_EQUAL( 44,(int)LuaDisableSecuritySandbox.size() ); + + result = Font::extract_mixed_LTR_RTL_map(LuaDisableSecuritySandbox); + //CPPUNIT_ASSERT_EQUAL( 7, (int)result.size() ); + + //printf("Result: [%s]\n",LuaDisableSecuritySandbox.c_str()); + +#ifdef HAVE_FRIBIDI + LuaDisableSecuritySandbox = expected2; + Font::bidi_cvt(LuaDisableSecuritySandbox); + + CPPUNIT_ASSERT_EQUAL( 44,(int)LuaDisableSecuritySandbox.size() ); + string expected_result = "לש לוח זגרא תחטבא לטב"; + expected_result.insert(5,"\""); + expected_result.insert(21,"\""); + + //printf("Test: expected [%s] actual [%s]\n",expected_result.c_str(),LuaDisableSecuritySandbox.substr(0,40).c_str()); + CPPUNIT_ASSERT_EQUAL( expected_result,LuaDisableSecuritySandbox.substr(0,40) ); + CPPUNIT_ASSERT_EQUAL( string("Lua"),LuaDisableSecuritySandbox.substr(41) ); +#endif + + string PrivacyPlease = "הפעל מצב פרטי\\n(הסתר מדינה, וכו)"; + PrivacyPlease.insert(52,"\""); + string expected3 = PrivacyPlease; + //printf("Result: [%s]\n",PrivacyPlease.c_str()); + CPPUNIT_ASSERT_EQUAL( 56,(int)PrivacyPlease.size() ); + +#ifdef HAVE_FRIBIDI + Font::bidi_cvt(PrivacyPlease); + + CPPUNIT_ASSERT_EQUAL( 56,(int)PrivacyPlease.size() ); + string expected_result3 = "יטרפ בצמ לעפה\\n(וכו ,הנידמ רתסה)"; + expected_result3.insert(29,"\""); + + //printf("Test: expected [%s] actual [%s]\n",expected_result3.c_str(),PrivacyPlease.c_str()); + CPPUNIT_ASSERT_EQUAL( expected_result3,PrivacyPlease ); +#endif + + + + string FactionName_indian = "Indian (אינדיאנים)"; + string expected4 = FactionName_indian; + //printf("Result: [%s]\n",PrivacyPlease.c_str()); + CPPUNIT_ASSERT_EQUAL( 27,(int)FactionName_indian.size() ); + +#ifdef HAVE_FRIBIDI + Font::bidi_cvt(FactionName_indian); + + CPPUNIT_ASSERT_EQUAL( 27,(int)FactionName_indian.size() ); + string expected_result4 = "Indian (םינאידניא)"; + + //printf("Test: expected [%s] actual [%s]\n",expected_result3.c_str(),PrivacyPlease.c_str()); + CPPUNIT_ASSERT_EQUAL( expected_result4,FactionName_indian ); +#endif + + // This test still failing: xx IP xx + string LanIP = "כתובות IP מקומי:192.168.0.150 ( 61357 / 61357 )"; + string expected5 = LanIP; + CPPUNIT_ASSERT_EQUAL( 59,(int)LanIP.size() ); + +#ifdef HAVE_FRIBIDI +// Font::bidi_cvt(LanIP); +// +// CPPUNIT_ASSERT_EQUAL( 59,(int)LanIP.size() ); +// string expected_result5 = "abc"; +// +// CPPUNIT_ASSERT_EQUAL( expected_result5,LanIP ); +#endif + } +}; + + +// Test Suite Registrations +CPPUNIT_TEST_SUITE_REGISTRATION( FontTest ); +//