Home » U++ Library support » U++ Libraries and TheIDE: i18n, Unicode and Internationalization » 16 bits wchar
|
|
|
|
|
|
|
|
| Re: 16 bits wchar [message #12302 is a reply to message #8036] |
Wed, 24 October 2007 11:58   |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
I've been sick and I didn't leave the house so I couldn't post. But here is my code:
int utf8codepointEE(const byte *s, const byte *z, int &lmod, int & dep)
{
if (s < z)
{
dword code = (byte)*s++;
int codePoint = 0;
if(code < 0x80)
{
dep = 1;
lmod = 1;
return code;
}
else if (code < 0xC2)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
else if (code < 0xE0)
{
if(s >= z)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
if (s[0] < 0x80 || s[0] >= 0xC0)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
codePoint = ((code - 0xC0) << 6) + *s - 0x80;
if(codePoint < 0x80 || codePoint > 0x07FF)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
else
{
dep = 2;
lmod = 2;
return codePoint;
}
}
else if (code < 0xF0)
{
if(s + 1 >= z)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
if(s[0] < 0x80 || s[0] >= 0xC0 || s[1] < 0x80 || s[1] >= 0xC0)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
codePoint = ((code - 0xE0) << 12) + ((s[0] - 0x80) << 6) + s[1] - 0x80;
if(codePoint < 0x0800 || codePoint > 0xFFFF)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
else
{
dep = 3;
lmod = 3;
return codePoint;
}
}
else if (code < 0xF5)
{
if(s + 2 >= z)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
if(s[0] < 0x80 || s[0] >= 0xc0 || s[1] < 0x80 || s[1] >= 0xc0 ||
s[2] < 0x80 || s[2] >= 0xc0)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
codePoint = ((code - 0xf0) << 18) + ((s[0] - 0x80) << 12) +
((s[1] - 0x80) << 6) + s[2] - 0x80;
if(codePoint < 0x010000 || codePoint > 0x10FFFF)
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
else
{
dep = 4;
lmod = 4;
return codePoint;
}
}
else
{
dep = 1;
lmod = 3;
return 0xEE00 + code;
}
}
else
return -1;
}
int utf8lenEE(const char *_s, int len)
{
const byte *s = (const byte *)_s;
const byte *lim = s + len;
int codePoint = 0;
int length = 0;
while(s < lim) {
int lmod, dep;
int codePoint = utf8codepointEE(s, lim, lmod, dep);
ASSERT(codePoint != -1);
length += lmod;
s += dep;
}
return length;
}
int utf8lenDeEE(const char *_s, int len)
{
const byte *s = (const byte *)_s;
const byte *lim = s + len;
int codePoint = 0;
int length = 0;
while(s < lim) {
int lmod, dep;
int codePoint = utf8codepointEE(s, lim, lmod, dep);
ASSERT(codePoint != -1);
if ((codePoint & 0xFFFFFF00) == 0xEE00)
{
length++;
s += dep;
}
else
{
length += lmod;
s += dep;
}
}
return length;
}
inline byte * putUtf8(byte *s, int codePoint)
{
if (codePoint < 0x80)
*s++ = codePoint;
else if (codePoint < 0x0800)
{
*s++ = 0xC0 | (codePoint >> 6);
*s++ = 0x80 | (codePoint & 0x3f);
}
else if (codePoint < 0xFFFF)
{
*s++ = 0xE0 | (codePoint >> 12);
*s++ = 0x80 | (codePoint >> 6) & 0x3F;
*s++ = 0x80 | (codePoint & 0x3F);
}
else
{
*s++ = 0xF0 | (codePoint >> 18);
*s++ = 0x80 | (codePoint >> 12) & 0x3F;
*s++ = 0x80 | (codePoint >> 6) & 0x3F;
*s++ = 0x80 | (codePoint & 0x3F);
}
return s;
}
String ToUtf8EE(const char *_s, int _len)
{
int tlen = utf8lenEE(_s, _len);
if (tlen == -1)
return "";
StringBuffer result(tlen);
byte *s = (byte *) _s;
const byte *lim = s + _len;
byte *z = (byte *) ~result;
int length = 0;
while(s < lim) {
int lmod, dep;
int codePoint = utf8codepointEE(s, lim, lmod, dep);
if (codePoint == -1)
return "";
length += lmod;
s += dep;
z = putUtf8(z, codePoint);
}
ASSERT(length == tlen);
return result;
}
String FromUtf8EE(const char *_s, int _len)
{
int tlen = utf8lenDeEE(_s, _len);
if (tlen == -1)
return "";
StringBuffer result(tlen);
byte *s = (byte *) _s;
const byte *lim = s + _len;
byte *z = (byte *) ~result;
int length = 0;
while(s < lim) {
int lmod, dep;
int codePoint = utf8codepointEE(s, lim, lmod, dep);
if (codePoint == -1)
return "";
if ((codePoint & 0xFFFFFF00) == 0xEE00)
{
codePoint -= 0xEE00;
*z++ = codePoint;
lmod = 1;
}
else
z = putUtf8(z, codePoint);
length += lmod;
s += dep;
}
ASSERT(length == tlen);
return result;
}
It is up to you to decide what exactly you want to do with Unicode. And if you let me know, I could help. So please decide, and if you want to leave it as it is, I will find something else to work on.
|
|
|
|
| Re: 16 bits wchar [message #12303 is a reply to message #12302] |
Wed, 24 October 2007 13:27 |
 |
mirek
Messages: 14291
Registered: November 2005
|
Ultimate Member |
|
|
I like the code. However, I still do not see too many practical uses.
Therefore: new package UnicodeEx is perhaps a right place where to save it, agreed?
As for future plans, yes, I think that going 32bits is the ultimate solution. Anyway, before that, I would like to see many other things resolved in U++. IMO, RTL support is now the priority in this area. Maybe, if you like to play with language issues, you can invest your spare time there... I do not expect the actual code, rather informations.
Mirek
|
|
|
|
|
|
| Re: 16 bits wchar [message #12333 is a reply to message #12308] |
Thu, 25 October 2007 14:47 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
I also wrote a conversion algorithm from Utf8 to Utf16, which is quite similar to my previous one.
Since I already done these, I would like to optimize them a little. I have a couple of questions though.
1. My code point extraction routine is a little to long and quite redundant. The same sequence that handles an incorrect value is called a lot of times. I would like to get rid of these repetitions. I could use a macro, but I don't like to expose a dangerous macro to the rest of the file, so I could undef it after the function. Or, this would be the perfect case to where the use of goto could be justified and almost needed. Can I use goto?
EDIT:
P.S.: The modifed version of utf8codepointEE optimized and using goto is 49 lines long, while the original was 115. And I still consider it pretty clear, maybe even more clear because I can it on one screen.
2. The algorithm is a little bit inefficient because it first calculates the length of the new buffer, and then it fills it. But by calculating the length, we already get enough info to populate it with the correct values, reducing the number of calculations by half. But if I do this, I would need to preallocate first a possibly bigger than necessary buffer, fill it directly, than copy it in the new string and free the buffer. This has one extra allocation, and I'm not sure how efficient allocations are in U++. AFAIK, you replaced the default allocator. If it has similar efficiency as the standard one, the price of the allocation is not that large, but maybe you are against this approach as it more STL like, with allocating a lot of extra data and doing copies. If the allocator is faster, or if you have a caching mechanism, than I think that it could be a lot faster this way.
[Updated on: Thu, 25 October 2007 15:57] Report message to a moderator |
|
|
|
|
|
| Re: 16 bits wchar [message #12388 is a reply to message #12333] |
Sat, 27 October 2007 11:11 |
|
mirek
Messages: 14291
Registered: November 2005
|
Ultimate Member |
|
|
| cbpporter wrote on Thu, 25 October 2007 08:47 |
I also wrote a conversion algorithm from Utf8 to Utf16, which is quite similar to my previous one.
Since I already done these, I would like to optimize them a little. I have a couple of questions though.
1. My code point extraction routine is a little to long and quite redundant. The same sequence that handles an incorrect value is called a lot of times. I would like to get rid of these repetitions. I could use a macro, but I don't like to expose a dangerous macro to the rest of the file, so I could undef it after the function. Or, this would be the perfect case to where the use of goto could be justified and almost needed. Can I use goto?
|
Of course. I have no problem with using anything in IMPLEMENTATION. I believe that the main task is to keep interfaces clear. If goto or macro are able to speedup or simplify things, go for it. No need to undefine macro either, as long as it is used in .cpp only.
(The only things I would ask in implementation: If you decide to use platform/machine/CPU specific things like "asm", be sure to provide crossplatform "default" implementation, or at least implement it for all supported platforms).
| Quote: |
2. The algorithm is a little bit inefficient because it first calculates the length of the new buffer, and then it fills it. But by calculating the length, we already get enough info to populate it with the correct values, reducing the number of calculations by half. But if I do this, I would need to preallocate first a possibly bigger than necessary buffer, fill it directly, than copy it in the new string and free the buffer. This has one extra allocation, and I'm not sure how efficient allocations are in U++.
|
I believe they are quite efficient. Most of time, about 20 CPU instructions have to be executed to allocate a memory block.
However, I am a little bit afraid that the "copy" will make it inefficient...
OTOH, IME, guessing never helps to resolve optimization issues. If you really want to play hard, benchmark 
Also consider putting StringBuffer to the mix as well.
| Quote: |
If the allocator is faster, or if you have a caching mechanism, than I think that it could be a lot faster this way.
|
Well, it is as fast as to make the STL idea of speed optimized allocators in container templates obsolete 
Mirek
|
|
|
|
| Re: 16 bits wchar [message #12504 is a reply to message #12388] |
Tue, 06 November 2007 13:31 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
I've been a little busy the past days, so I didn't have time to benchmark stuff or do more optimizations. I attached my to modified files, because I'm tired of copying and pasting so much code, making the thread hard to read.
I also looked over the RTL issue. I used the resources from unicode.org, and mainly the "bidirectional algorithm". It is not that hard, but you have to split the text in paragraphs, than lines, then compute the direction based on control chars, create a dummy string and display it. If you add cursor movement, I think the issue is not that simple.
The question is how far do you want to go with RTL. The simplest solution is to just add a right click option to editable texts or to check the first character of a string to make sure that it is not a RTL mark. Or you could implement the full algorithm. And also, these control characters must be exclude from searches and other string comparison operations.
-
Attachment: CharSet.zip
(Size: 19.67KB, Downloaded 625 times)
|
|
|
|
| Re: 16 bits wchar [message #12564 is a reply to message #12504] |
Fri, 09 November 2007 10:39 |
|
mirek
Messages: 14291
Registered: November 2005
|
Ultimate Member |
|
|
| cbpporter wrote on Tue, 06 November 2007 07:31 |
I've been a little busy the past days, so I didn't have time to benchmark stuff or do more optimizations. I attached my to modified files, because I'm tired of copying and pasting so much code, making the thread hard to read.
I also looked over the RTL issue. I used the resources from unicode.org, and mainly the "bidirectional algorithm". It is not that hard, but you have to split the text in paragraphs, than lines, then compute the direction based on control chars, create a dummy string and display it. If you add cursor movement, I think the issue is not that simple.
The question is how far do you want to go with RTL. The simplest solution is to just add a right click option to editable texts or to check the first character of a string to make sure that it is not a RTL mark. Or you could implement the full algorithm. And also, these control characters must be exclude from searches and other string comparison operations.
|
Hm, have not we agreed to produce UnicodeEx package?
Mirek
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| Re: 16 bits wchar [message #17208 is a reply to message #16970] |
Sat, 02 August 2008 13:27 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
I have finally made some progress on this!
But not under Linux. I just couldn't get characters outside BMP to print, because all the characters were interpreted as two. Anyway, it is surely possible since most applications do manage to print them, but since I never coded for X before, probably I'm doing something wrong.
There is also a funny little story with me installing everything my distro had regarding fonts in hope of improving the number of displayable characters. It turns out that everything was almost 1Gb of fonts and related stuff and now I do have some extra fonts visible, but with the price of any drawing operation being slowed down to a crawl. So we have here a classical less is more situation.
But under windows I'm having better luck and am now displaying almost the full range of the JIS standard characters! Surrogate pairs are enabled by default, but I needed to install some free fonts. It is strange that still this is not enough, and I had to add some fallback fonts to the registry to get the display working. I guess Windows does not search every possible font for the characters, and somehow filters them, excluding the font that are needed. U++ does do any extra searching in different fonts under Windows (and Linux), so maybe we need to take into account somehow these registry settings.
From U++'s point of view, in order to get everything working I still need to get GetTestSize/FontInfo::GetCM working with surrogate pairs.
Do you know of other key functions or classes that I need to look over to get basic output working? And could you explain in a few words how font compositioning works for U++. I found the code, but font compositioning is not used when I try to draw text. It will probably need to be modified to get it to work with surrogates also.
|
|
|
|
| Re: 16 bits wchar [message #17214 is a reply to message #17208] |
Sat, 02 August 2008 18:34 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
Great! I've gotten GetTextSize to work! I also investigated and fallback registry settings are not necessary for test drawing and size computation to work. For plane 0 characters, if font is available, character will be drawn (except in a case if will get back to later), and if not, little black rectangless will be drawn in correct position and size. Fallback font doesn't seem to be used at all. Maybe if I uninstall standard CJK fonts, Windows will start using fallback
For plane two, the situation is the same, except that fallback setting must be present in order for characters to be drawn. Without, even when font is present, placeholder rectangles will be drawn.
There is only one last problem. For some characters in plane 0, I can't get the character to show. All other Unicode enabled apps on my system render correctly, even without fallback setting, but in U++ these characters appear as little black boxes. A workaround is to specify a font name directly which supports the given characters. Using this workaround, I can get full JIS support with two extra free fonts and one registry setting. But specifying the font manually is not a long term solution, and I need to find out why U++ has problems with some characters in the following ranges: 0x3402-0x4d77 and 0xfa30-0xfa6a.
|
|
|
|
|
|
| Re: 16 bits wchar [message #17229 is a reply to message #17215] |
Sun, 03 August 2008 14:51 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
I've finished Label too and that's about it for the immediate support that I need for CJK. I fixed the previous problem with characters not being drawn by using a hardcoded font name for those problematic ranges. I guess Windows font support is not perfect either .A better solution would be to determine if the font can display the character, and if not, change the font by probably using a list determined at application startup. But I'm afraid that it isn't that simple to do with current font rendering methods and we should get back to it at the next text output engine refactoring (maybe when we do it for Linux, where it is more needed).
I only had to update a couple of U++ functions, and I wrote different encoding conversion functions which I explicitly call instead of the standard ones to limit my changes to specific parts of code and let the rest use the defaults.
I will probably need an edit control updated also, but for now I'm pretty happy with my over 13000 unique characters displayed, so full JIS support.
BTW, the Core2000 font available on the Internet has a number of broken codepoints, drawing the wrong characters in several cases. It is pretty hard to notice unless you know what to look for, so if anybody is using it, try out "HAN NOM A" instead, which hasn't shown any error up to now.
The question is what now. Since I'm happy with my fixes and nobody else seems to have needs regarding CJK support, I could just rename the couple of functions I modified and override Paint in a control that inherits from Label and thus keep my changes local and become U++ version agnostic. Of course, I will release a package in Bazaar for those who for some particular reason need more than Unicode 1.1 support, but a fair warning is due: my changes are strongly biased towards Japanese characters, so Chinese or Korean specific issues might still exist.
Or I could merge my changes with my installed version of U++, use it for a while to see if there are other problems (Qtf and edit controls are sure to not enjoy surrogate pairs) and continue researching how to best migrate U++ entirely to the new scheme. I'm only going to do this if you want these changes and if you want them relatively soon, i.e. in 1-2 devs. If not, I'll go with variant one because I still need to implement EUC-JP encoding support, for which I need huge conversion tables .
|
|
|
|
|
|
|
|
| Re: 16 bits wchar [message #17245 is a reply to message #17242] |
Mon, 04 August 2008 15:53 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
| luzr wrote on Mon, 04 August 2008 16:07 |
Well, is my understanding correct that your method leads to non-BMP support in UTF-8/String and non-BMP support in WString via surrogate pairs?
First is fine and very good achievement (except that we still have font issue in X11).
|
Yes, full Unicode code range for conversions and experimental support for size based calculation, fonts and output is what I'm trying to achieve.
Under Linux we will either go with some determined at start font for some ranges of Unicode, or we need to do full font pooling on display operation, and somehow cache the results. I don't know how slow the operation of font enquery is, but with my 1GB of fonts it is pretty slow with full ppoling (i.e. Opera or Character Map).
| Quote: |
WString should be the means of manipulating unicode texts on per-character basis (e.g. in editor).
|
Using multiple code units per character doesn't disable the use of a text editor or any means of manipulating Unicode texts. It just needs a little bit smarter methods for some operations. I know that using only one word is convenient, but Unicode says that there are up to two words per codepoint and there is no other work around than using 32 bits, which is not a lot better, because not even with UTF32 there isn't a 1:1 relationship between character and display operation of that character. Nonwhitespaces, separators, control characters, combining characters and others must be filtered out, and the end result is the same as if you would use 16 bit chars (where the same operations must be done and I don't think they are done right now). Have you ever tried using combining characters in Upp? And even worse, using combining characters with zero width placeholders, where
I think than one by one all methods that take a string must and traverse it must be reviewed and altered to use a new style of traversing. This only applies to codepoint based addressing, like in GetTextSize. This could be done in an unified way, with iterators, or even "fake index" iterators (which will be a little bit slower than iterators, who should have the same performance as index based traversing).
Anyway performance shouldn't be a problem, because I've been experimenting with a faster method of conversion, which uses local caches short strings up to a static length, bypassing the general algorithm of traverse data, compute code points, determine if escaping is necessary, return length and then recompute data and using a faster method in which the second computation is done only for long strings. It should be faster, but I'm not done benchmarking yet because Linux console apps refuse to print anything since Today (and to connect to mysql, but that is unrelated).
This way there is no need for WString actually, except the fact that it helps as an optimization because Win32 uses it. In the end, we will probably need a full text layout engine, breaking text in multiple segments, and drawing them one by one to support composition, multichar composition, RTL.
|
|
|
|
| Re: 16 bits wchar [message #17248 is a reply to message #17245] |
Mon, 04 August 2008 17:14 |
|
mirek
Messages: 14291
Registered: November 2005
|
Ultimate Member |
|
|
| cbpporter wrote on Mon, 04 August 2008 09:53 |
Using multiple code units per character doesn't disable the use of a text editor or any means of manipulating Unicode texts. It just needs a little bit smarter methods for some operations. I know that using only one word is convenient, but Unicode says that there are up to two words per codepoint and there is no other work around than using 32 bits, which is not a lot better, because not even with UTF32 there isn't a 1:1 relationship between character and display operation of that character. Nonwhitespaces, separators, control characters, combining characters and others must be filtered out, and the end result is the same as if you would use 16 bit chars (where the same operations must be done and I don't think they are done right now).
|
Well, this rather sound like we should kick out WString altogether and keep just UTF-8:)
| Quote: |
This way there is no need for WString actually, except the fact that it helps as an optimization because Win32 uses it. In the end, we will probably need a full text layout engine, breaking text in multiple segments, and drawing them one by one to support composition, multichar composition, RTL.
|
Ah, right
OTOH, on logical level, I still see characters on the screen. And those characters should be edited on per-character basis.
Maybe we just need smarter encoding than UNICODE?
Makes me think - realistically, there is a lot of "reserved" positions in BMP. Could we just use them for this?
Mirek
|
|
|
|
| Re: 16 bits wchar [message #17253 is a reply to message #17248] |
Mon, 04 August 2008 22:47 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
| luzr wrote on Mon, 04 August 2008 18:14 |
Well, this rather sound like we should kick out WString altogether and keep just UTF-8:)
|
I think that we should keep both, and even add LString eventually just for the sake of completeness. In other package if your worried about exe size.
| Quote: |
Maybe we just need smarter encoding than UNICODE?
Makes me think - realistically, there is a lot of "reserved" positions in BMP. Could we just use them for this?
|
Well there is nothing better than Unicode AFAIK. It may seem sometimes like there is too much fuss with it, but if you are in my place and have to deal with other legacy encodings, you would have to deal with EUC, EUC-JP, ShiftJIS, JIS and a couple of ISOs, where a lot of these encoding don't guarantee round-trip conversion, and you'll see that Unicode is a true blessing. Great that I have iConv to ease the burden a little.
And BTW, Unicode forbids the use of the reserved or unassigned code points for any use .
Anyway I ran my benchmarks on my Windows machine where console output still works. I did the tests with some experimental methods which are not complete, so the results could be a little inaccurate, but they are still interesting enough too post.
I used 3 methods to convert from a two UTF8 sets to UTF16. The first method is the standard U++ FromUtf8. The second is my FromUtf8SR, which takes into account 4 byte characters, and the third is the highly experimental FromUtf8SR2. The first data set consists of 200 latin characters, representing 200 code points (the letter c 200 times). The second one consists of 100 kanji, 3 characters each, totaling 300 bytes. On second thought, I should have used same sized data sets. All conversions are run 1000000 times.
In Debug mode:
Latin
3125
3203
2078
Kanji
3891
3906
2406
Nothing too impressive here. First method, the standard one is a little faster than mine, and the experimental one is considerably faster.
In Release mode:
Latin
484
485
390
Kanji
4718
3157
812
Here, for kanji, my method really is a lot faster. But in release mode, FromUtf8 for an all kanji input is slower than in Debug mode. Can someone verify this? Maybe I messed something up.
As I said my experimental method is really experimental and not complete yet (I hope it is thread safe also). I hope I'm not chasing after wild geese (is that an expression?) and I didn't miss something that should render my experimental method useless or wrong, because the numbers are great!
|
|
|
|
| Re: 16 bits wchar [message #17257 is a reply to message #17253] |
Tue, 05 August 2008 00:03 |
|
mirek
Messages: 14291
Registered: November 2005
|
Ultimate Member |
|
|
[quote title=cbpporter wrote on Mon, 04 August 2008 16:47]
Well there is nothing better than Unicode AFAIK. It may seem sometimes like there is too much fuss with it, but if you are in my place and have to deal with other legacy encodings, you would have to deal with EUC, EUC-JP, ShiftJIS, JIS and a couple of ISOs, where a lot of these encoding don't guarantee round-trip conversion, and you'll see that Unicode is a true blessing. Great that I have iConv to ease the burden a little.
And BTW, Unicode forbids the use of the reserved or unassigned code points for any use .
[/code]
I obviously do not understand the depth of the problem, anyway:
One code-point corresponds, at the end of process, to one font glyph. Is that correct?
Meanwhile, it can be made of several unicode words/dword. Correct?
If yes, how much codepoints we need in *existing fonts*?
If we can fit all possible font glyphs into 64KB codepoints, problem is solved. Of course, we would need some more conversion routines between our "UnicodeEx" and the "real Unicode"...
Mirek
[Updated on: Tue, 05 August 2008 00:04] Report message to a moderator |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| Re: 16 bits wchar [message #17268 is a reply to message #17266] |
Tue, 05 August 2008 12:03 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
| luzr wrote on Tue, 05 August 2008 01:51 |
Anyway, might I ask you to think about / comment codepoint == glyph and distinct(codepoint) < 64K claims?
|
I really can't imagine how that would be possible.
First of all, how do you expect to squish almost 100K characters in 64K? Some kind of dynamic character set loading would be needed, and still a string could not contain every possible character.
And second, in Unicode codepoint != glyph. All the 90k+ codepoints can be combined theoretically to produce and endless number of glyphs. Think of Unicode as a comparably more feature poor Qtf. Codepoints are commands. 99% of commands are "print glyph X", but the rest allow you to manipulate the layout and appearance of glyph. It is not a visual manipulation, like with font, rather manipulation that alters the abstract concept of a glyph, like adding diacritics.
The reason why this is not that obvious is that Win API handles this for you automatically. Most users and even developers are not familiar with this process, and if somehow their input data contains such characters, Win controls will display them correctly. All common diacritics are handled pretty well, but uncommon ones which are often incorrectly handled. This could be one of U++ strong points in the future. When all font issues are resolved (probably not before 2009.1  ), if we would offer full combining characters support algorithmically where fonts fail, we would certainly be in a relatively unique position.
but since we don't use native controls, we are more exposed to them. Under Windows, when you use such text in non editable controls in U++, you get correct result, but if you use an EditString for example, you have to press cursor keys multiple times to step through a character which visually is made out of only one glyph, but uses several code points as representation.
This problem can be relatively easily addressed, by updating a couple of functions and making sure than Windows API always gets full chunks of text.
Under Linux, such support is a lot poorer. Since we send to X text one codepoint at a time, no composition can take place. And I don't even know if the methods from X that are in use can handle such texts. All my experiments in U++ gave the same result: diacritics are removed and the rest of characters are displayed as whitespace. KDE editors seemed quite happy with such codes, while gedit displayed the characters correctly, but without composing them in the same place., so basically it did not do any better than U++ if we would have font pooling.
As always, I come to the same conclusion: nobody really cares for proper internationalization and Unicode (except Qt or KDE, who seems to have best support out of all, comparable and maybe better than Windows, but seemingly poorer because of available fonts).
| Quote: |
Hm, I was thinking about our problem a lot....
I believe that we should do one important thing first - scan all available fonts and count/list all codepoints there...
|
Yes, that would help under windows and is must under Linux. We could even use some "heuristics", i.e. if a font has 2 Arabic characters, there is a high probability that it handles all Arabic characters from that given Unicode range. Maybe we can get away by splinting all codepoints into ranges on a per script basis, and only test some key characters, but I can't be sure without testing.
|
|
|
|
| Re: 16 bits wchar [message #17277 is a reply to message #17268] |
Tue, 05 August 2008 15:12 |
|
mirek
Messages: 14291
Registered: November 2005
|
Ultimate Member |
|
|
| cbpporter wrote on Tue, 05 August 2008 06:03 |
| luzr wrote on Tue, 05 August 2008 01:51 |
Anyway, might I ask you to think about / comment codepoint == glyph and distinct(codepoint) < 64K claims?
|
I really can't imagine how that would be possible.
First of all, how do you expect to squish almost 100K characters in 64K? Some kind of dynamic character set loading would be needed, and still a string could not contain every possible character.
|
Yes, meanwhile I have studied it a little bit more, you are right.
| Quote: |
And second, in Unicode codepoint != glyph. All the 90k+ codepoints can be combined theoretically to produce and endless number of glyphs. Think of Unicode as a comparably more feature poor Qtf. Codepoints are commands. 99% of commands are "print glyph X", but the rest allow you to manipulate the layout and appearance of glyph. It is not a visual manipulation, like with font, rather manipulation that alters the abstract concept of a glyph, like adding diacritics.
|
Well, I was studying this as well and came to conclusion that combining is of little concern.
First, AFAIK, basic Unicode "compliance" does not requite it.
Second, all important ("real") combining codepoints have characters in Unicode.
IMO, I would regard combining as sort of formating info, similar to '\n' or '\t' - something that we need to be aware about (and, in fact, we already are, sort of, see UnicodeCombine...) but do not need to actively support in editors etc...
BTW, that UnicodeCombine is exactly the sort of support that makes sense.
| Quote: |
All common diacritics are handled pretty well, but uncommon ones which are often incorrectly handled.
|
This is because there is no general way how to create combined glyph....
| Quote: |
Under Windows, when you use such text in non editable controls in U++, you get correct result, but if you use an EditString for example, you have to press cursor keys multiple times to step through a character which visually is made out of only one glyph, but uses several code points as representation.
|
Does not make sense to me...
| Quote: |
This problem can be relatively easily addressed, by updating a couple of functions and making sure than Windows API always gets full chunks of text.
|
IMO, this would be pretty hard to address in fact. Or result in confusing user interface.
| Quote: |
Under Linux, such support is a lot poorer. Since we send to X text one codepoint at a time, no composition can take place.
|
Actually, we do not. Interface accepts strings. But I doubt it manages combining.
| Quote: |
And I don't even know if the methods from X that are in use can handle such texts. All my experiments in U++ gave the same result: diacritics are removed and the rest of characters are displayed as whitespace. KDE editors seemed quite happy with such codes, while gedit displayed the characters correctly, but without composing them in the same place., so basically it did not do any better than U++ if we would have font pooling.
|
We will, I promise (Well, I would rather describe it as "font substitution"...).
| Quote: |
As always, I come to the same conclusion: nobody really cares for proper internationalization and Unicode
|
The question is how combining really helps... IMO, it is not worth the enormous trouble it brings...
| Quote: |
Yes, that would help under windows and is must under Linux. We could even use some "heuristics", i.e. if a font has 2 Arabic characters, there is a high probability that it handles all Arabic characters from that given Unicode range. Maybe we can get away by splinting all codepoints into ranges on a per script basis, and only test some key characters, but I can't be sure without testing.
|
Oh, for the beginning, I was rather thinking about "offline experimental scan" to find out what is really going on
Maybe we should then match "standard substitution fonts" for all basic fonts.
Also interesting point is what then happens to my heurestic "glyph fixing" for characters 256-512 (U++ synthetises missing glyphs there by combining characters 0-256). So it is sort of alternative approach to font substitution. But I would keep it as it results in better looking texts.
Mirek
|
|
|
|
|
|
| Re: 16 bits wchar [message #17282 is a reply to message #17277] |
Tue, 05 August 2008 15:57 |
cbpporter
Messages: 1428
Registered: September 2007
|
Ultimate Contributor |
|
|
| luzr wrote on Tue, 05 August 2008 16:12 |
Well, I was studying this as well and came to conclusion that combining is of little concern.
First, AFAIK, basic Unicode "compliance" does not requite it.
Second, all important ("real") combining codepoints have characters in Unicode.
|
Sure, it does not require it, but is relatively easy to implement. I have a pretty clear idea on how to do it. But you are right, it's not a priority right now.
| Quote: |
This is because there is no general way how to create combined glyph....
|
Compute size of base character, retrieve align of character that is combined with, align in a rect that has the size as a maximum of both and draw. Basically in pseudocode:
draw(curx, cury, basechar);
draw(curx + deltax, cury + deltay, combinedchar);
Finding out delta is not that easy, but doable. This is pretty much what Qt (empirically determined) does and is near perfect.
| Quote: |
Does not make sense to me...
|
Was doesn't make sense. Basically what I said is that you cant feed composed characters to an editable control. And expect align and keyboard/mouse navigation to work.
| Quote: |
IMO, this would be pretty hard to address in fact. Or result in confusing user interface.
|
I don't understand why this affects user interface. Everything looks the same from the point of view of the user.
| Quote: |
Actually, we do not. Interface accepts strings. But I doubt it manages combining.
|
You are doing:
for(int i = 0; i < n; i++) {
wchar h = text[i];
XftDrawString16(...,(FcChar16 *)&h, 1);
That's drawing characters one at a time (if angle is not zero). If a composition system is available, in this case it will not trigger because drawing characters that are used for composition one at a time doesn't make any sense.
But if angle is zero, you are right. I guess that it does not do composition.
The offline scan seems like a good idea.
|
|
|
|
Goto Forum:
Current Time: Fri Oct 02 20:58:16 GMT+2 2026
Total time taken to generate the page: 0.03794 seconds
|
U++ framework
Search
Help
Members
Register
Login
Home
