diff --git a/include/lgi/common/Charset.h b/include/lgi/common/Charset.h --- a/include/lgi/common/Charset.h +++ b/include/lgi/common/Charset.h @@ -1,81 +1,81 @@ #pragma once #include /// Charset definitions enum LCharSetType { CpNone, CpMapped, CpUtf8, CpUtf16, CpUtf32, CpIconv, CpWindowsDb }; /// Charset information class class LgiClass LCharset { public: /// Standard charset name const char *Charset; /// Human description const char *Description; /// 128 shorts that map the 0x80-0xff range to unicode short *UnicodeMap; /// Charset's name for calling iconv const char *IconvName; /// Comma separated list of alternate names used for this charset const char *AlternateNames; /// General type of the charset LCharSetType Type; /// Constructor LCharset(const char *cp = 0, const char *des = 0, short *map = 0, const char *alt = 0); /// Returns true if the charset is a unicode variant bool IsUnicode(); /// Gets the iconv name const char *GetIconvName(); /// Returns whether Lgi can convert to/from this charset at the moment. bool IsAvailable(); }; /// Charset table manager class class LgiClass LCharsetSystem { struct LCharsetSystemPriv *d; public: static LCharsetSystem *Inst(); LCharsetSystem(); ~LCharsetSystem(); // Hooks std::function DetectCharset; // Get the charset info LCharset *GetCsInfo(const char *Cp); LCharset *GetCsList(); }; /// Returns information about a charset. LgiFunc LCharset *LGetCsInfo(const char *Cs); /// Returns the start of an array of supported charsets, terminated by /// one with a NULL 'Charset' member. LgiFunc LCharset *LGetCsList(); /// Returns the charset that best fits the input data LgiFunc const char *LUnicodeToCharset ( /// The input text const char *Utf8, /// The byte length of the input text ssize_t Len = -1, /// An optional list of preferred charsets to look through first - List *Prefs = 0 + LString::Array *Prefs = NULL ); diff --git a/include/lgi/common/TextConvert.h b/include/lgi/common/TextConvert.h --- a/include/lgi/common/TextConvert.h +++ b/include/lgi/common/TextConvert.h @@ -1,15 +1,15 @@ #pragma once bool Is8Bit(const char *Text); [[deprecated]] char *DecodeBase64Str(char *Str, ssize_t Len = -1); LString LDecodeBase64Str(LString Str); [[deprecated]] char *DecodeQuotedPrintableStr(char *Str, ssize_t Len = -1); LString LDecodeQuotedPrintableStr(LString Str); [[deprecated]] char *DecodeRfc2047(char *Str); LString LDecodeRfc2047(LString Str); -[[deprecated]] char *EncodeRfc2047(char *Str, const char *Charset, List *CharsetPrefs, ssize_t LineLength = 0); -LString LEncodeRfc2047(LString Str, const char *Charset, List *CharsetPrefs, ssize_t LineLength = 0); +[[deprecated]] char *EncodeRfc2047(char *Input, const char *InCharset, LString::Array *OutCharsets = NULL, ssize_t LineLength = 0); +LString LEncodeRfc2047(LString Input, const char *InCharset, LString::Array *OutCharsets = NULL, ssize_t LineLength = 0); diff --git a/src/common/Gdc2/Font/Charset.cpp b/src/common/Gdc2/Font/Charset.cpp --- a/src/common/Gdc2/Font/Charset.cpp +++ b/src/common/Gdc2/Font/Charset.cpp @@ -1,1711 +1,1711 @@ #include #include #include #include "lgi/common/Lgi.h" #include "lgi/common/Font.h" #include "lgi/common/Charset.h" struct UnicodeMappings { int Unicode; char Ascii; } MissingMaps[] = { {0x2019, '\''}, {0x201C, '\"'}, {0x201D, '\"'}, {0, 0} }; typedef uint32_t iso2022jp_block[16]; iso2022jp_block *iso2022jp_map[128]; iso2022jp_block iso2022jp_blocks[] = { {0,0,0x10000000,0,0,0x53118c,0x800000,0x800000,0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0,0,0,0,0,0xfffe0000,0xfffe03fb,0x3fb,0}, {0xffff0002,0xffffffff,0x2ffff,0,0,0,0,0,0,0,0,0,0,0,0,0}, {0x33510000,0x80d0063,0,0,0,0,0,0,0x8,0x800,0,0,0xf0000,0,0x140000,0}, {0x6404098d,0x20301f81,0x40000,0xcc3,0xcc,0x20,0,0,0x40000,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0,0x3999900f,0x99999939,0x804,0,0,0x300c0003,0xc8c0,0x8000}, {0x60,0,0x5,0xa400,0,0,0,0,0,0,0,0,0,0,0,0}, {0x103fffef,0,0xfffffffe,0xffffffff,0x780fffff,0xfffffffe,0xffffffff,0x787fffff,0,0,0,0,0,0,0,0}, {0x43f36f8b,0x9b462442,0xe3e0e82c,0x400a0004,0xdb365f65,0x4497977,0xe3f0ecd7,0x8c56038,0x3403e602,0x35518000,0x7eabe0c8,0x98698200,0x2942a948,0x8060e803,0xad93441c,0x4568c03a}, {0x8656aa60,0x2403f7a,0x14618388,0x21741020,0x7022021,0x40bc3000,0x4462a624,0xa2060a8,0x85740217,0x9c840402,0x14157bfb,0x11e27f24,0x2efb665,0x20ff1f75,0x38403a70,0x676326c3}, {0x20924dd9,0xfc946b0,0x4850bc98,0xa03f8638,0x88162388,0x52323e09,0xe3a422aa,0xc72c00dd,0x26e1a166,0x8f0a840b,0x559e27eb,0x89bbc241,0x85400014,0x8496361,0x8ad07f0c,0x5cfff3e}, {0xa803ff1a,0x7b407a41,0x80024745,0x38eb0500,0x5d851,0x710c9934,0x1000397,0x24046366,0x5180d0,0x430ac000,0x30c89071,0x58000008,0xf7000e99,0x415f80,0x941000b0,0x62800018}, {0x9d00240,0x1568200,0x8015004,0x5101d10,0x1084c1,0x10504025,0x4d8a410f,0xa60d4009,0x914cab19,0x98121c0,0x3c485,0x80000652,0x80b04,0x9041d,0x905c4849,0x16900009}, {0x22200c65,0x24338412,0x47960c03,0x42250a04,0x90880028,0x4f084900,0xd3aa14a2,0x3e87d830,0x1f618604,0x41867ea4,0x5b3c390,0x211857a5,0x2a48241e,0x4a041128,0x161b0a40,0x88400d60}, {0x9502020a,0x10608221,0x4000243,0x80001444,0xc040000,0x70000000,0xc11a06,0xc00024a,0x401a00,0x40451404,0xbdb30029,0x52b0a78,0xbfa0bba9,0x8379407c,0xe81d12fc,0xc5694bf6}, {0x44aeff6,0xff022115,0x402bed63,0x242d033,0x131000,0x59ca1b02,0x20000a0,0x2c41a703,0x8ff24880,0x204,0x10055800,0x489200,0x20011894,0x34805004,0x684c3200,0x68be49ea}, {0x2e42184c,0x21c9a820,0x80b050b9,0xff7c001e,0x14e0849a,0x1e028c1,0xac49870e,0xdddb130f,0x89fbbe1a,0x51a2a2e0,0x32ca5502,0x928b3e46,0x438f1dbf,0x32186703,0x33c03028,0xa9230811}, {0x3a65c000,0x4028fe3,0x86252c4e,0xa1bf3d,0x8cd43a1a,0x317c06c9,0x950a00e0,0xedb018b,0x8c20e34b,0xf0101182,0xa7287d94,0x40fbc9ac,0x6534484,0x44445a90,0x13fc8,0xf5d40048}, {0xec577701,0x891dc442,0x49286b83,0xd2424109,0x59fe061d,0x3a221800,0x3b9fb7e4,0xc0eaf003,0x82021386,0xe4008980,0x10a1b200,0xcc44b80,0x8944d309,0x48341faf,0xc458259,0x450420a}, {0x10c8a040,0x44503140,0x1004004,0x5408280,0x442c0108,0x1a056a30,0x51420a6,0x645690cf,0x31000021,0xcbf09c18,0x63e2a120,0x1b5104c,0x9a83538c,0x3281b8b2,0xa84987a,0xc0233e7}, {0x9018d4cc,0x9070a1a1,0xe0048a1e,0x451c3d4,0x21c2439a,0x53104844,0x36400292,0xf3bd0241,0xe8f0ab09,0xa5d27dc0,0xd24bc242,0xd0afa43f,0x34a11aa0,0x3d88247,0x651bc452,0xc83ad294}, {0x40c8001c,0x33140e06,0xb21b614f,0xc0d00088,0xa898a02a,0x166ba1c5,0x85b42e50,0x604c08b,0x1e04f933,0xa251056e,0x76380400,0x73b8ec07,0x18324406,0xc8164081,0x63097c8a,0xaa042980}, {0xca9c1c24,0x27604e0e,0x83000990,0x81040046,0x10816011,0x908540d,0xcc0a000e,0xc000500,0xa0440430,0x6784008b,0x8a195288,0x8b18865e,0x41602e59,0x9cbe8c10,0x891c6861,0x89800}, {0x89a8100,0x41900018,0xe4a14007,0x640d0505,0xe4d310e,0xff0a4806,0x2aa81632,0xb852e,0xca841800,0x696c0e20,0x16000032,0x3905658,0x1a285120,0x11248000,0x432618e1,0xeaa5d52}, {0xae280fa0,0x4500fa7b,0x89406408,0xc044c880,0xb1419005,0x24c48424,0x603a1a34,0xc1949000,0x3a8246,0xc106180d,0x99100022,0x1511e050,0x824057,0x20a041a,0x8930004f,0x444ad813}, {0xed228a02,0x400510c0,0x1021000,0x31018808,0x2044600,0x708f000,0xa2008900,0x22020000,0x16100200,0x10400042,0x2605200,0x200052f4,0x82308510,0x42021100,0x80b54308,0x9a2070e1}, {0x8012040,0xfc653500,0xab0419c1,0x62140286,0x440087,0x2449085,0xa85405c,0x33803207,0xb8c00400,0xc0d0ce20,0x80c030,0xd250508,0x400a90,0x80c0200,0x40006505,0x41026421}, {0x268,0x847c0024,0xde200002,0x40498619,0x40000808,0x20010084,0x10108400,0x1c742cd,0xd52a7038,0x1d8f1968,0x3e12be50,0x81d92ef5,0x2412cec4,0x732e0828,0x4b3424ac,0xd41d020c}, {0x80002a02,0x8110097,0x114411c4,0x7d451786,0x64949d9,0x87914000,0xd8c4254c,0x491444ba,0xc8001b92,0x15800271,0xc000081,0xc200096a,0x40024800,0xba493021,0x1c802080,0x1008e2ac}, {0x341004,0x841400e1,0x20000020,0x10149800,0x4aa70c2,0x54208688,0x4130c62,0x20109180,0x2064082,0x54001c40,0xe4e90383,0x84802125,0x2000e433,0xe60944c0,0x81260a03,0x80112da}, {0x97906901,0xf8864001,0x81e24d,0xa6510a0e,0x81ec011a,0x8441c600,0xb62cadb8,0x8741a46f,0x4b028d54,0x2681161,0x2057bb60,0x43350a0,0xb7b4a8c0,0x1122402,0x20009ad3,0xc82271}, {0x809e2081,0xe1800c8a,0x8151b009,0x40281031,0x89a52a0e,0x620e69b6,0xd1444425,0x4d548085,0x1fb12c75,0x862dd807,0x4841d87c,0x226e414e,0x9e088200,0xed37f80c,0x75268c80,0x8149313}, {0xc8040e32,0x6ea6484e,0x66702c4a,0xba0126c0,0x185dd30c,0,0,0,0,0x5400000,0x81337020,0x3a54f81,0x641055ec,0x2344c318,0x341462,0x1a090a43}, {0x13a5187b,0xa8480102,0xc5440440,0xe2dd8106,0x2d481af0,0x416b626,0x6e405058,0x31128032,0xc0007e4,0x420a8208,0x803b4840,0x87134860,0x3428850d,0xe5290319,0x870a2345,0x5c1825a9}, {0xd9c577a6,0x3e85e00,0xa7000081,0x41c6cd54,0xa2042800,0x2b0ab860,0xda9e0020,0xe1a08ea,0x11c0427c,0x3768908,0x1058621,0x18a80000,0xc44846a0,0x20220d05,0x91485422,0x28978a01}, {0x87898,0x31221605,0x8804240,0x6a2fa4e,0x92110814,0x9b042002,0x6432e52,0x90105000,0x85ba0041,0x20203042,0x5a04f0b,0x40802708,0x1a930591,0x600df50,0x3021a202,0x4e800630}, {0x4c80cc4,0x8001a004,0xd4316000,0xa020880,0x281c00,0x418e18,0xca106ad0,0x4b00f210,0x1506274d,0x88900220,0x82a85a00,0x81504549,0x80002004,0x2c088804,0x508d1,0x4ac48001}, {0x62e020,0xa42008e,0x6a8c3055,0xe0a5090e,0x42c42906,0x80b34814,0xb330803e,0x731c0102,0x600d1494,0x9400c20,0xc040301a,0xc094a451,0x5c88dca,0xa40c96c2,0x34040001,0x11000c8}, {0xa9c9550d,0x1c5a2428,0x48370142,0x100f7a4d,0x452a32b4,0x9205317b,0x5c44b894,0x458a68d7,0x2ed15097,0x42081943,0x9d40d202,0x20979840,0x64d5409,0,0,0}, {0,0x84800000,0x4215542,0x17001c06,0x61107624,0xb9ddff87,0x5c0a659f,0x3c00245d,0x59adb0,0,0,0x9b28d0,0x2000422,0x44080108,0xac409804,0x90288d0a}, {0xe0018700,0x310400,0x82211794,0x10540019,0x21a2cb2,0x40039c02,0x88043d60,0x7900080c,0xba3c1628,0xcb088640,0x90807274,0x1e,0xd8000000,0x9c87e188,0x4124034,0x2791ae64}, {0xe6fbe86b,0x5366408f,0x537feea6,0xb5e4e32b,0x2869f,0x1228548,0x8004402,0x20a02116,0x2040004,0x52000,0x1547e00,0x1ac162c,0x10852a84,0x5308c14,0xb943fbc3,0x906000ca}, {0x40326000,0x80901200,0x4c810b30,0x40020054,0x1d6a0029,0x2802000,0x48000,0x150c2610,0x7018040,0xc24d94d,0x18502810,0x50205001,0x4d01000,0x2017080,0x21c30108,0x132}, {0x7190088,0x5600802,0x4c0e0012,0xf0a10405,0x2,0,0,0,0,0,0,0x800000,0x35a8e8d,0x5a0421bd,0x11703488,0x26}, {0x10000000,0x8804c502,0xf801b815,0x25ed147c,0x1bb0ed60,0x1bd70589,0x1a627af3,0xac50d0c,0x524ae5d1,0x63050490,0x52440354,0x16122b57,0x1101a872,0x182949,0x10080948,0x886c6000}, {0x58f916e,0x39903012,0x4930f840,0x1b8880,0,0x428500,0x98000058,0x7014ea04,0x611d1628,0x60005113,0xa71a24,0,0x3c00000,0x10187120,0xa9270172,0x89066004}, {0x20cc022,0x40810900,0x8ca0202d,0xe34,0,0x11012100,0xc11a8011,0x892ec4c,0x85000040,0x1806c7ac,0x512e03e,0x108000,0x80ce4008,0x2106d01,0x8568641,0x27011e}, {0x83d3750,0x4e05e032,0x48401c0,0x1400081,0,0,0,0x591aa0,0x882443c8,0xc8001d48,0x72030152,0x4049013,0x4008280,0xd148a10,0x2088056,0x2704a040}, {0x4c000000,0,0,0xa3200000,0xa0ae1902,0xdf002660,0x7b15f010,0x3ad08121,0x284180,0x48001003,0x8014cc00,0xc414cf,0x30202000,0x1,0,0}, {0,0,0,0,0,0,0,0,0xffffdf7a,0xefffffff,0x3fffffff,0,0,0,0,0x2} }; class LgiIso2022Jp { public: LgiIso2022Jp() { int n, o = 0, i = 0; for (n=0; n<3; n++) iso2022jp_map[o++] = &iso2022jp_blocks[i++]; o += 13; for (n=0; n<4; n++) iso2022jp_map[o++] = &iso2022jp_blocks[i++];; o += 4; iso2022jp_map[o++] = &iso2022jp_blocks[i++];; o += 14; for (n=0; n<41; n++) iso2022jp_map[o++] = &iso2022jp_blocks[i++];; o += 47; iso2022jp_map[o++] = &iso2022jp_blocks[i++];; LAssert(o == 128); } bool CanEncode(char16 *s, ssize_t l) { if (s) { if (l < 0) l = StrlenW(s); for (int i=0; i> 9]; if (!*block) { return false; } u &= 0x1ff; if ( ( *block[u >> 5] & (1 << (u & 0x1f)) ) == 0 ) { return false; } } return true; } return false; } } Iso2022Jp; ///////////////////////////////////////////////////////////////////////////////////// bool LIsUtf8(const char *s, ssize_t len) { #define LenCheck(Need) \ if (len >= 0 && (len - (s - Start)) < Need) \ goto Utf8Error; #define TrailCheck() \ if (!IsUtf8_Trail(*s)) \ goto Utf8Error; \ s++; if (!s || *s == 0) return true; const char *Start = s; while ( ( len < 0 || ((s - Start) < len) ) && *s ) { if (IsUtf8_1Byte(*s)) { s++; } else if (IsUtf8_2Byte(*s)) { s++; LenCheck(1); TrailCheck(); } else if (IsUtf8_3Byte(*s)) { s++; LenCheck(2); TrailCheck(); TrailCheck(); } else if (IsUtf8_4Byte(*s)) { s++; LenCheck(3); TrailCheck(); TrailCheck(); TrailCheck(); } else goto Utf8Error; } return true; Utf8Error: #if 1 LgiTrace("%s:%i - Invalid utf @ offset=%i, bytes=", _FL, (int) (s - Start)); auto end = len < 0 ? NULL : Start + len; for (auto i = 0; i < 16; i++) { if ( (end && s >= end) || *s == 0 ) break; LgiTrace("%02.2x,", (uint8_t)*s++); } LgiTrace("\n"); #endif return false; } ///////////////////////////////////////////////////////////////////////////////////// short _gdc_usascii_mapping[128] = { // 0x80 - 0x8f 0xc7, 0xfc, 0xe9, 0xe2, 0xe4, 0xe0, 0xe5, 0xe7, 0xea, 0xeb, 0xe8, 0xef, 0xee, 0xec, 0xc4, 0xc5, // 0x90 - 0x9f 0xc9, 0xe6, 0xc6, 0xf4, 0xf6, 0xf2, 0xfb, 0xf9, 0xff, 0xd6, 0xdc, 0xa2, 0xa3, 0xa5, 0x20a7, 0x192, // 0xa0 - 0xaf 0xe1, 0xed, 0xf3, 0xfa, 0xf1, 0xd1, 0xb2, 0xb0, 0xbf, 0x2310, 0xac, 0xbd, 0xbc, 0xa1, 0xab, 0xbb, // 0xb0 - 0xbf 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510, // 0xc0 - 0xcf 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567, // 0xd0 - 0xdf 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, // 0xe0 - 0xef 0x3b1, 0x3b2, 0x393, 0x3a0, 0x3a3, 0x3c3, 0x3bc, 0x3c4, 0x3a6, 0x398, 0x3a9, 0x3b4, 0x221e, 0xd8, 0x3b6, 0x2229, // 0xf0 - 0xff 0x2261, 0xb1, 0x2265, 0x2264, 0x2320, 0x2321, 0xf7, 0x2248, 0xb0, 0x2022, 0x2219, 0x221a, 0x207f, 178, 0x25a0, 0x25a1 }; // This mapping just NUL's out the characters between 0x80 and 0x9f which aren't defined // in the ISO spec. The rest of the characters map to themselves. short _gdc_ISO_8859_identity_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; short _gdc_ISO_8859_2_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xa0, 0x104, 0x2d8, 0x141, 0xa4, 0x13d, 0x15a, 0xa7, 0xa8, 0x160, 0x15e, 0x164, 0x179, 0xad, 0x17d, 0x17b, 0xb0, 0x105, 0x2db, 0x142, 0xb4, 0x13e, 0x15b, 0x2c7, 0xb8, 0x161, 0x15f, 0x165, 0x17a, 0x2dd, 0x17e, 0x17c, 0x154, 0xc1, 0xc2, 0x102, 0xc4, 0x139, 0x106, 0xc7, 0x10c, 0xc9, 0x118, 0xcb, 0x11a, 0xcd, 0xce, 0x10e, 0x110, 0x143, 0x147, 0xd3, 0xd4, 0x150, 0xd6, 0xd7, 0x158, 0x16e, 0xda, 0x170, 0xdc, 0xdd, 0x162, 0xdf, 0x155, 0xe1, 0xe2, 0x103, 0xe4, 0x13a, 0x107, 0xe7, 0x10d, 0xe9, 0x119, 0xeb, 0x11b, 0xed, 0xee, 0x10f, 0x111, 0x144, 0x148, 0xf3, 0xf4, 0x151, 0xf6, 0xf7, 0x159, 0x16f, 0xfa, 0x171, 0xfc, 0xfd, 0x163, 0x2d9 }; short _gdc_ISO_8859_3_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xa0, 0x126, 0x2d8, 0xa3, 0xa4, 0, 0x124, 0xa7, 0xa8, 0x130, 0x15e, 0x11e, 0x134, 0xad, 0, 0x17b, 0xb0, 0x127, 0xb2, 0xb3, 0xb4, 0xb5, 0x125, 0xb7, 0xb8, 0x131, 0x15f, 0x11f, 0x135, 0xbd, 0, 0x17c, 0xc0, 0xc1, 0xc2, 0, 0xc4, 0x10a, 0x108, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0, 0xd1, 0xd2, 0xd3, 0xd4, 0x120, 0xd6, 0xd7, 0x11c, 0xd9, 0xda, 0xdb, 0xdc, 0x16c, 0x15c, 0xdf, 0xe0, 0xe1, 0xe2, 0, 0xe4, 0x10b, 0x109, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0, 0xf1, 0xf2, 0xf3, 0xf4, 0x121, 0xf6, 0xf7, 0x11d, 0xf9, 0xfa, 0xfb, 0xfc, 0x16d, 0x15d, 0x2d9 }; short _gdc_ISO_8859_4_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x0A0, 0x104, 0x138, 0x156, 0x0A4, 0x128, 0x13B, 0x0A7, 0x0A8, 0x160, 0x112, 0x122, 0x166, 0x0AD, 0x17D, 0x0AF, 0x0B0, 0x105, 0x2DB, 0x157, 0x0B4, 0x129, 0x13C, 0x2C7, 0x0B8, 0x161, 0x113, 0x123, 0x167, 0x14A, 0x17E, 0x14B, 0x100, 0x0C1, 0x0C2, 0x0C3, 0x0C4, 0x0C5, 0x0C6, 0x12E, 0x10C, 0x0C9, 0x118, 0x0CB, 0x116, 0x0CD, 0x0CE, 0x12A, 0x110, 0x145, 0x14C, 0x136, 0x0D4, 0x0D5, 0x0D6, 0x0D7, 0x0D8, 0x172, 0x0DA, 0x0DB, 0x0DC, 0x168, 0x16A, 0x0DF, 0x101, 0x0E1, 0x0E2, 0x0E3, 0x0E4, 0x0E5, 0x0E6, 0x12F, 0x10D, 0x0E9, 0x119, 0x0EB, 0x117, 0x0ED, 0x0EE, 0x12B, 0x111, 0x146, 0x14D, 0x137, 0x0F4, 0x0F5, 0x0F6, 0x0F7, 0x0F8, 0x173, 0x0FA, 0x0FB, 0x0FC, 0x169, 0x16B, 0x2D9 }; short _gdc_ISO_8859_5_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x0A0, 0x401, 0x402, 0x403, 0x404, 0x405, 0x406, 0x407, 0x408, 0x409, 0x40A, 0x40B, 0x40C, 0x0AD, 0x40E, 0x40F, 0x410, 0x411, 0x412, 0x413, 0x414, 0x415, 0x416, 0x417, 0x418, 0x419, 0x41A, 0x41B, 0x41C, 0x41D, 0x41E, 0x41F, 0x420, 0x421, 0x422, 0x423, 0x424, 0x425, 0x426, 0x427, 0x428, 0x429, 0x42A, 0x42B, 0x42C, 0x42D, 0x42E, 0x42F, 0x430, 0x431, 0x432, 0x433, 0x434, 0x435, 0x436, 0x437, 0x438, 0x439, 0x43A, 0x43B, 0x43C, 0x43D, 0x43E, 0x43F, 0x440, 0x441, 0x442, 0x443, 0x444, 0x445, 0x446, 0x447, 0x448, 0x449, 0x44A, 0x44B, 0x44C, 0x44D, 0x44E, 0x44F, 0x2116, 0x451, 0x452, 0x453, 0x454, 0x455, 0x456, 0x457, 0x458, 0x459, 0x45A, 0x45B, 0x45C, 0x0A7, 0x45E, 0x45F }; short _gdc_ISO_8859_6_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xA0, 0, 0, 0, 0xA4, 0, 0, 0, 0, 0, 0, 0, 0x60C, 0xAD, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x61B, 0, 0, 0, 0x61F, 0, 0x621, 0x622, 0x623, 0x624, 0x625, 0x626, 0x627, 0x628, 0x629, 0x62A, 0x62B, 0x62C, 0x62D, 0x62E, 0x62F, 0x630, 0x631, 0x632, 0x633, 0x634, 0x635, 0x636, 0x637, 0x638, 0x639, 0x63A, 0, 0, 0, 0, 0, 0x640, 0x641, 0x642, 0x643, 0x644, 0x645, 0x646, 0x647, 0x648, 0x649, 0x64A, 0x64B, 0x64C, 0x64D, 0x64E, 0x64F, 0x650, 0x651, 0x652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; short _gdc_ISO_8859_7_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x0A0, 0x2BD, 0x2BC, 0x0A3, 0, 0, 0x0A6, 0x0A7, 0x0A8, 0x0A9, 0, 0x0AB, 0x0AC, 0x0AD, 0, 0x015, 0x0B0, 0x0B1, 0x0B2, 0x0B3, 0x384, 0x385, 0x386, 0x0B7, 0x388, 0x389, 0x38A, 0x0BB, 0x38C, 0x0BD, 0x38E, 0x38F, 0x390, 0x391, 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, 0, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x3AA, 0x3AB, 0x3AC, 0x3AD, 0x3AE, 0x3AF, 0x3B0, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x3CA, 0x3CB, 0x3CC, 0x3CD, 0x3CE, 0 }; short _gdc_ISO_8859_8_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x0A0, 0, 0x0A2, 0x0A3, 0x0A4, 0x0A5, 0x0A6, 0x0A7, 0x0A8, 0x0A9, 0x0D7, 0x0AB, 0x0AC, 0x0AD, 0x0AE, 0x203E, 0x0B0, 0x0B1, 0x0B2, 0x0B3, 0x0B4, 0x0B5, 0x0B6, 0x0B7, 0x0B8, 0x0B9, 0x0F7, 0x0BB, 0x0BC, 0x0BD, 0x0BE, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2017, 0x5D0, 0x5D1, 0x5D2, 0x5D3, 0x5D4, 0x5D5, 0x5D6, 0x5D7, 0x5D8, 0x5D9, 0x5DA, 0x5DB, 0x5DC, 0x5DD, 0x5DE, 0x5DF, 0x5E0, 0x5E1, 0x5E2, 0x5E3, 0x5E4, 0x5E5, 0x5E6, 0x5E7, 0x5E8, 0x5E9, 0x5EA, 0, 0, 0, 0, 0 }; short _gdc_ISO_8859_9_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0x11E, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0x130, 0x15E, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0x11F, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x131, 0x15F, 0xFF }; short _gdc_ISO_8859_13_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xA0, 0x201D, 0xA2, 0xA3, 0xA4, 0x201E, 0xA6, 0xA7, 0xD8, 0xA9, 0x156, 0xAB, 0xAC, 0xAD, 0xAE, 0xC6, 0xB0, 0xB1, 0xB2, 0xB3, 0x201C, 0xB5, 0xB6, 0xB7, 0xF8, 0xB9, 0x157, 0xBB, 0xBC, 0xBD, 0xBE, 0xE6, 0x104, 0x12E, 0x100, 0x106, 0xC4, 0xC5, 0x118, 0x112, 0x10C, 0xC9, 0x179, 0x116, 0x122, 0x136, 0x12A, 0x13B, 0x160, 0x143, 0x145, 0xD3, 0x14C, 0xD5, 0xD6, 0xD7, 0x172, 0x141, 0x15A, 0x16A, 0xDC, 0x17B, 0x17D, 0xDF, 0x105, 0x12F, 0x101, 0x107, 0xE4, 0xE5, 0x119, 0x113, 0x10D, 0xE9, 0x17A, 0x117, 0x123, 0x137, 0x12B, 0x13C, 0x161, 0x144, 0x146, 0xF3, 0x14D, 0xF5, 0xF6, 0xF7, 0x173, 0x142, 0x15B, 0x16B, 0xFC, 0x17C, 0x17E, 0x2019 }; short _gdc_ISO_8859_15_mapping[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xA0, 0xA1, 0xA2, 0xA3, 0x20AC, 0xA5, 0x160, 0xA7, 0x161, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0x17D, 0xB5, 0xB6, 0xB7, 0x17E, 0xB9, 0xBA, 0xBB, 0x152, 0x153, 0x178, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF }; short _gdc_win_874_mapping[128] = { 0x20AC, 0, 0, 0, 0, 0x2026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0, 0, 0, 0, 0, 0, 0, 0, 0xA0, 0xE01, 0xE02, 0xE03, 0xE04, 0xE05, 0xE06, 0xE07, 0xE08, 0xE09, 0xE0A, 0xE0B, 0xE0C, 0xE0D, 0xE0E, 0xE0F, 0xE10, 0xE11, 0xE12, 0xE13, 0xE14, 0xE15, 0xE16, 0xE17, 0xE18, 0xE19, 0xE1A, 0xE1B, 0xE1C, 0xE1D, 0xE1E, 0xE1F, 0xE20, 0xE21, 0xE22, 0xE23, 0xE24, 0xE25, 0xE26, 0xE27, 0xE28, 0xE29, 0xE2A, 0xE2B, 0xE2C, 0xE2D, 0xE2E, 0xE2F, 0xE30, 0xE31, 0xE32, 0xE33, 0xE34, 0xE35, 0xE36, 0xE37, 0xE38, 0xE39, 0xE3A, 0, 0, 0, 0, 0xE3F, 0xE40, 0xE41, 0xE42, 0xE43, 0xE44, 0xE45, 0xE46, 0xE47, 0xE48, 0xE49, 0xE4A, 0xE4B, 0xE4C, 0xE4D, 0xE4E, 0xE4F, 0xE50, 0xE51, 0xE52, 0xE53, 0xE54, 0xE55, 0xE56, 0xE57, 0xE58, 0xE59, 0xE5A, 0xE5B, 0, 0, 0, 0, }; short _gdc_win_1250_mapping[128] = { 0x20AC, 0, 0x201A, 0, 0x201E, 0x2026, 0x2020, 0x2021, 0, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, 0xA0, 0x02C7, 0x02D8, 0x0141, 0xA4, 0x0104, 0xA6, 0xA7, 0xA8, 0xA9, 0x015E, 0xAB, 0xAC, 0xAD, 0xAE, 0x017B, 0xB0, 0xB1, 0x02DB, 0x0142, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0x0105, 0x015F, 0xBB, 0x013D, 0x02DD, 0x013E, 0x017C, 0x0154, 0xC1, 0xC2, 0x0102, 0xC4, 0x0139, 0x0106, 0xC7, 0x010C, 0xC9, 0x0118, 0xCB, 0x011A, 0xCD, 0xCE, 0x010E, 0x0110, 0x0143, 0x0147, 0xD3, 0xD4, 0x0150, 0xD6, 0xD7, 0x0158, 0x016E, 0xDA, 0x0170, 0xDC, 0xDD, 0x0162, 0xDF, 0x0155, 0xE1, 0xE2, 0x0103, 0xE4, 0x013A, 0x0107, 0xE7, 0x010D, 0xE9, 0x0119, 0xEB, 0x011B, 0xED, 0xEE, 0x010F, 0x0111, 0x0144, 0x0148, 0xF3, 0xF4, 0x0151, 0xF6, 0xF7, 0x0159, 0x016F, 0xFA, 0x0171, 0xFC, 0xFD, 0x0163, 0x02D9 }; short _gdc_win_1251_mapping[128] = { 0x402, 0x403, 0x201A, 0x453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x409, 0x2039, 0x40A, 0x40C, 0x40B, 0x40F, 0x452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0, 0x2122, 0x459, 0x203A, 0x45A, 0x45C, 0x45B, 0x45F, 0x0A0, 0x40E, 0x45E, 0x408, 0x0A4, 0x490, 0x0A6, 0x0A7, 0x401, 0x0A9, 0x404, 0x0AB, 0x0AC, 0x0AD, 0x0AE, 0x407, 0x0B0, 0x0B1, 0x406, 0x456, 0x491, 0x0B5, 0x0B6, 0x0B7, 0x451, 0x2116, 0x454, 0x0BB, 0x458, 0x405, 0x455, 0x457, 0x410, 0x411, 0x412, 0x413, 0x414, 0x415, 0x416, 0x417, 0x418, 0x419, 0x41A, 0x41B, 0x41C, 0x41D, 0x41E, 0x41F, 0x420, 0x421, 0x422, 0x423, 0x424, 0x425, 0x426, 0x427, 0x428, 0x429, 0x42A, 0x42B, 0x42C, 0x42D, 0x42E, 0x42F, 0x430, 0x431, 0x432, 0x433, 0x434, 0x435, 0x436, 0x437, 0x438, 0x439, 0x43A, 0x43B, 0x43C, 0x43D, 0x43E, 0x43F, 0x440, 0x441, 0x442, 0x443, 0x444, 0x445, 0x446, 0x447, 0x448, 0x449, 0x44A, 0x44B, 0x44C, 0x44D, 0x44E, 0x44F }; short _gdc_win_1252_mapping[128] = { 0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017D, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF }; short _gdc_win_1253_mapping[128] = { 0x20AC, 0, 0x201A, 0x192, 0x201E, 0x2026, 0x2020, 0x2021, 0, 0x2030, 0, 0x2039, 0, 0, 0, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0, 0x2122, 0, 0x203A, 0, 0, 0, 0, 0xA0, 0x385, 0x386, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0, 0xAB, 0xAC, 0xAD, 0xAE, 0x2015, 0xB0, 0xB1, 0xB2, 0xB3, 0x384, 0xB5, 0xB6, 0xB7, 0x388, 0x389, 0x38A, 0xBB, 0x38C, 0xBD, 0x38E, 0x38F, 0x390, 0x391, 0x392, 0x393, 0x394, 0x395, 0x396, 0x397, 0x398, 0x399, 0x39A, 0x39B, 0x39C, 0x39D, 0x39E, 0x39F, 0x3A0, 0x3A1, 0, 0x3A3, 0x3A4, 0x3A5, 0x3A6, 0x3A7, 0x3A8, 0x3A9, 0x3AA, 0x3AB, 0x3AC, 0x3AD, 0x3AE, 0x3AF, 0x3B0, 0x3B1, 0x3B2, 0x3B3, 0x3B4, 0x3B5, 0x3B6, 0x3B7, 0x3B8, 0x3B9, 0x3BA, 0x3BB, 0x3BC, 0x3BD, 0x3BE, 0x3BF, 0x3C0, 0x3C1, 0x3C2, 0x3C3, 0x3C4, 0x3C5, 0x3C6, 0x3C7, 0x3C8, 0x3C9, 0x3CA, 0x3CB, 0x3CC, 0x3CD, 0x3CE, 0 }; short _gdc_win_1254_mapping[128] = { 0x20AC, 0, 0x201A, 0x192, 0x201E, 0x2026, 0x2020, 0x2021, 0x2C6, 0x2030, 0x160, 0x2039, 0x152, 0, 0, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x2DC, 0x2122, 0x161, 0x203A, 0x153, 0, 0, 0x178, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0x11E, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0x130, 0x15E, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0x11F, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x131, 0x15F, 0xFF, }; short _gdc_win_1255_mapping[128] = { 0x20AC, 0, 0x201A, 0x192, 0x201E, 0x2026, 0x2020, 0x2021, 0x2C6, 0x2030, 0, 0x2039, 0, 0, 0, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x2DC, 0x2122, 0, 0x203A, 0, 0, 0, 0, 0xA0, 0xA1, 0xA2, 0xA3, 0x20AA, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xD7, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xF7, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0x5B0, 0x5B1, 0x5B2, 0x5B3, 0x5B4, 0x5B5, 0x5B6, 0x5B7, 0x5B8, 0x5B9, 0, 0x5BB, 0x5BC, 0x5BD, 0x5BE, 0x5BF, 0x5C0, 0x5C1, 0x5C2, 0x5C3, 0x5F0, 0x5F1, 0x5F2, 0x5F3, 0x5F4, 0, 0, 0, 0, 0, 0, 0, 0x5D0, 0x5D1, 0x5D2, 0x5D3, 0x5D4, 0x5D5, 0x5D6, 0x5D7, 0x5D8, 0x5D9, 0x5DA, 0x5DB, 0x5DC, 0x5DD, 0x5DE, 0x5DF, 0x5E0, 0x5E1, 0x5E2, 0x5E3, 0x5E4, 0x5E5, 0x5E6, 0x5E7, 0x5E8, 0x5E9, 0x5EA, 0, 0, 0x200E, 0x200F, 0, }; short _gdc_win_1256_mapping[128] = { 0x20AC, 0x67E, 0x201A, 0x192, 0x201E, 0x2026, 0x2020, 0x2021, 0x2C6, 0x2030, 0x679, 0x2039, 0x152, 0x686, 0x698, 0x688, 0x6AF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x6A9, 0x2122, 0x691, 0x203A, 0x153, 0x200C, 0x200D, 0x6BA, 0xA0, 0x60C, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x6BE, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0x61B, 0xBB, 0xBC, 0xBD, 0xBE, 0x61F, 0x6C1, 0x621, 0x622, 0x623, 0x624, 0x625, 0x626, 0x627, 0x628, 0x629, 0x62A, 0x62B, 0x62C, 0x62D, 0x62E, 0x62F, 0x630, 0x631, 0x632, 0x633, 0x634, 0x635, 0x636, 0xD7, 0x637, 0x638, 0x639, 0x63A, 0x640, 0x641, 0x642, 0x643, 0xE0, 0x644, 0xE2, 0x645, 0x646, 0x647, 0x648, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0x649, 0x64A, 0xEE, 0xEF, 0x64B, 0x64C, 0x64D, 0x64E, 0xF4, 0x64F, 0x650, 0xF7, 0x651, 0xF9, 0x652, 0xFB, 0xFC, 0x200E, 0x200F, 0x6D2, }; short _gdc_win_1257_mapping[128] = { 0x20AC, 0, 0x201A, 0, 0x201E, 0x2026, 0x2020, 0x2021, 0, 0x2030, 0, 0x2039, 0, 0xA8, 0x2C7, 0xB8, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0, 0x2122, 0, 0x203A, 0, 0xAF, 0x2DB, 0, 0xA0, 0, 0xA2, 0xA3, 0xA4, 0, 0xA6, 0xA7, 0xD8, 0xA9, 0x156, 0xAB, 0xAC, 0xAD, 0xAE, 0xC6, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xF8, 0xB9, 0x157, 0xBB, 0xBC, 0xBD, 0xBE, 0xE6, 0x104, 0x12E, 0x100, 0x106, 0xC4, 0xC5, 0x118, 0x112, 0x10C, 0xC9, 0x179, 0x116, 0x122, 0x136, 0x12A, 0x13B, 0x160, 0x143, 0x145, 0xD3, 0x14C, 0xD5, 0xD6, 0xD7, 0x172, 0x141, 0x15A, 0x16A, 0xDC, 0x17B, 0x17D, 0xDF, 0x105, 0x12F, 0x101, 0x107, 0xE4, 0xE5, 0x119, 0x113, 0x10D, 0xE9, 0x17A, 0x117, 0x123, 0x137, 0x12B, 0x13C, 0x161, 0x144, 0x146, 0xF3, 0x14D, 0xF5, 0xF6, 0xF7, 0x173, 0x142, 0x15B, 0x16B, 0xFC, 0x17C, 0x17E, 0x2D9, }; short _gdc_win_1258_mapping[128] = { 0x20AC, 0, 0x201A, 0x192, 0x201E, 0x2026, 0x2020, 0x2021, 0x2C6, 0x2030, 0, 0x2039, 0x152, 0, 0, 0, 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x2DC, 0x2122, 0, 0x203A, 0x153, 0, 0, 0x178, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0x102, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0x300, 0xCD, 0xCE, 0xCF, 0x110, 0xD1, 0x309, 0xD3, 0xD4, 0x1A0, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0x1AF, 0x303, 0xDF, 0xE0, 0xE1, 0xE2, 0x103, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0x301, 0xED, 0xEE, 0xEF, 0x111, 0xF1, 0x323, 0xF3, 0xF4, 0x1A1, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x1B0, 0x20AB, 0xFF, }; short _gdc_koi8r_mapping[128] = { 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590, 0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2219, 0x221A, 0x2248, 0x2264, 0x2265, 0xA0, 0x2321, 0xB0, 0xB2, 0xB7, 0xF7, 0x2550, 0x2551, 0x2552, 0x451, 0x2553, 0x2554, 0x2555, 0x2556, 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x255C, 0x255D, 0x255E, 0x255F, 0x2560, 0x2561, 0x401, 0x2562, 0x2563, 0x2564, 0x2565, 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x256B, 0x256C, 0xA9, 0x44E, 0x430, 0x431, 0x446, 0x434, 0x435, 0x444, 0x433, 0x445, 0x438, 0x439, 0x43A, 0x43B, 0x43C, 0x43D, 0x43E, 0x43F, 0x44F, 0x440, 0x441, 0x442, 0x443, 0x436, 0x432, 0x44C, 0x44B, 0x437, 0x448, 0x44D, 0x449, 0x447, 0x44A, 0x42E, 0x410, 0x411, 0x426, 0x414, 0x415, 0x424, 0x413, 0x425, 0x418, 0x419, 0x41A, 0x41B, 0x41C, 0x41D, 0x41E, 0x41F, 0x42F, 0x420, 0x421, 0x422, 0x423, 0x416, 0x412, 0x42C, 0x42B, 0x417, 0x428, 0x42D, 0x429, 0x427, 0x42A }; short _gdc_koi8u_mapping[128] = { 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590, 0x2591, 0x2592, 0x2593, 0x2320, 0x25A0, 0x2019, 0x221A, 0x2248, 0x2264, 0x2265, 0xA0, 0x2321, 0xB0, 0xB2, 0xB7, 0xF7, 0x2550, 0x2551, 0x2552, 0x451, 0x454, 0x2554, 0x456, 0x457, 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x491, 0x255D, 0x255E, 0x255F, 0x2560, 0x2561, 0x401, 0x404, 0x2563, 0x406, 0x407, 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x490, 0x256C, 0xA9, 0x44E, 0x430, 0x431, 0x446, 0x434, 0x435, 0x444, 0x433, 0x445, 0x438, 0x439, 0x43A, 0x43B, 0x43C, 0x43D, 0x43E, 0x43F, 0x44F, 0x440, 0x441, 0x442, 0x443, 0x436, 0x432, 0x44C, 0x44B, 0x437, 0x448, 0x44D, 0x449, 0x447, 0x44A, 0x42E, 0x410, 0x411, 0x426, 0x414, 0x415, 0x424, 0x413, 0x425, 0x418, 0x419, 0x41A, 0x41B, 0x41C, 0x41D, 0x41E, 0x41F, 0x42F, 0x420, 0x421, 0x422, 0x423, 0x416, 0x412, 0x42C, 0x42B, 0x417, 0x428, 0x42D, 0x429, 0x427, 0x42A }; short _gdc_koi8ru_mapping[128] = { 0x2500, 0x2502, 0x250C, 0x2510, 0x2514, 0x2518, 0x251C, 0x2524, 0x252C, 0x2534, 0x253C, 0x2580, 0x2584, 0x2588, 0x258C, 0x2590, 0x2591, 0x2592, 0x2593, 0x201C, 0x25A0, 0x2219, 0x201D, 0x2014, 0x2116, 0x2122, 0xA0, 0xBB, 0xAE, 0xAB, 0xB7, 0xA4, 0x2550, 0x2551, 0x2552, 0x451, 0x454, 0x2554, 0x456, 0x457, 0x2557, 0x2558, 0x2559, 0x255A, 0x255B, 0x491, 0x45E, 0x255E, 0x255F, 0x2560, 0x2561, 0x401, 0x404, 0x2563, 0x406, 0x407, 0x2566, 0x2567, 0x2568, 0x2569, 0x256A, 0x490, 0x40E, 0xA9, 0x44E, 0x430, 0x431, 0x446, 0x434, 0x435, 0x444, 0x433, 0x445, 0x438, 0x439, 0x43A, 0x43B, 0x43C, 0x43D, 0x43E, 0x43F, 0x44F, 0x440, 0x441, 0x442, 0x443, 0x436, 0x432, 0x44C, 0x44B, 0x437, 0x448, 0x44D, 0x449, 0x447, 0x44A, 0x42E, 0x410, 0x411, 0x426, 0x414, 0x415, 0x424, 0x413, 0x425, 0x418, 0x419, 0x41A, 0x41B, 0x41C, 0x41D, 0x41E, 0x41F, 0x42F, 0x420, 0x421, 0x422, 0x423, 0x416, 0x412, 0x42C, 0x42B, 0x417, 0x428, 0x42D, 0x429, 0x427, 0x42A, }; #if defined WIN32 #define WinDef(d) d // Map the windows codepages to their real names. #define WINDOWS_1250 EASTEUROPE_CHARSET #define WINDOWS_1252 ANSI_CHARSET #define WINDOWS_1251 RUSSIAN_CHARSET #define WINDOWS_1253 GREEK_CHARSET #define WINDOWS_1254 TURKISH_CHARSET #define WINDOWS_1255 HEBREW_CHARSET #define WINDOWS_1256 ARABIC_CHARSET #define WINDOWS_1257 BALTIC_CHARSET #define WINDOWS_932 SHIFTJIS_CHARSET #define WINDOWS_936 GB2313_CHARSET #define WINDOWS_949 HANGEUL_CHARSET #define WINDOWS_950 CHINESEBIG5_CHARSET #else #define WinDef(d) 0 #endif LCharset::LCharset(const char *cp, const char *des, short *map, const char *alt) { Charset = cp; Description = des; UnicodeMap = map; IconvName = 0; AlternateNames = alt; Type = CpNone; if (cp) { if (stricmp(cp, "utf-8") == 0) { Type = CpUtf8; } else if (stricmp(cp, "utf-16") == 0) { Type = CpUtf16; } else if (stricmp(cp, "utf-32") == 0) { Type = CpUtf32; } else if (stricmp(cp, "ucs-2") == 0) { Type = CpUtf16; IconvName = "UCS-2-INTERNAL"; } else if (UnicodeMap) { Type = CpMapped; } #ifdef WIN32 else if (strnicmp(cp, "windows-", 8) == 0) { Type = CpWindowsDb; } #endif else { Type = CpIconv; } } } bool LCharset::IsUnicode() { return (Type == CpUtf8) || (Type == CpUtf16) || (Type == CpUtf32); } const char *LCharset::GetIconvName() { return IconvName ? IconvName : Charset; } bool LCharset::IsAvailable() { if (Type != CpIconv) return true; #ifndef LGI_STATIC LFontSystem *FontSys = LFontSystem::Inst(); if (FontSys) return FontSys->HasIconv(true); #endif // LGI_STATIC return false; } static LCharset LgiCharsets[] = { // Good 'ol ascii LCharset("us-ascii", "ASCII", _gdc_usascii_mapping, "ascii-us,iso-ir-6,ANSI_X3.4-1986,ISO_646.irv,ASCII,ISO646-US,us,IBM367,cp367,csASCII"), // Unicode (up here so they get found first) LCharset("utf-8", "Utf-8"), LCharset("utf-16", "Utf-16"), LCharset("utf-32", "Utf-32"), LCharset("ucs-2", "Ucs-2"), // ISO (get prefered over windows charsets by being first in this list) LCharset("iso-8859-1", "ISO 8859-1 (West Europe)", _gdc_ISO_8859_identity_mapping, "iso-ir-100,ISO_8859-1,latin1,l1,IBM819,CP819,csISOLatin1,iso8859-1"), LCharset("iso-8859-2", "ISO 8859-2 (East Europe)", _gdc_ISO_8859_2_mapping, "iso-ir-101,ISO_8859-2,latin2,l2,csISOLatin2"), LCharset("iso-8859-3", "ISO 8859-3 (Latin Script)", _gdc_ISO_8859_3_mapping, "iso-ir-109,ISO_8859-3,latin3,l3,csISOLatin3"), LCharset("iso-8859-4", "ISO 8859-4 (Baltic)", _gdc_ISO_8859_4_mapping, "iso-ir-110,ISO_8859-4,latin4,l4,csISOLatin4"), LCharset("iso-8859-5", "ISO 8859-5 (Russian)", _gdc_ISO_8859_5_mapping, "iso-ir-144,ISO_8859-5,cyrillic,csISOLatinCyrillic"), LCharset("iso-8859-6", "ISO 8859-6 (Arabic)", _gdc_ISO_8859_6_mapping, "iso-ir-127,ISO_8859-6,ECMA-114,ASMO-708,arabic,csISOLatinArabic"), LCharset("iso-8859-7", "ISO 8859-7 (Greek)", _gdc_ISO_8859_7_mapping, "iso-ir-126,ISO_8859-7,ELOT_928,ECMA-118,greek,greek8,csISOLatinGreek"), LCharset("iso-8859-8", "ISO 8859-8 (Hebrew)", _gdc_ISO_8859_8_mapping, "iso-ir-138,ISO_8859-8,hebrew,csISOLatinHebrew,iso-8859-8-i"), LCharset("iso-8859-9", "ISO 8859-9 (Turkish)", _gdc_ISO_8859_9_mapping, "iso-ir-148,ISO_8859-9,latin5,l5,csISOLatin5"), LCharset("iso-8859-13", "ISO 8859-13 (Baltik)", _gdc_ISO_8859_13_mapping, "ISO_8859-9"), LCharset("iso-8859-15", "ISO 8859-15 (Latic 9)", _gdc_ISO_8859_15_mapping, "ISO_8859-15"), // Windows LCharset("windows-874", "Windows 874 (Thai)", _gdc_win_874_mapping, "iso-8859-11,cp874"), LCharset("windows-932", "Windows 932 (Japanese)"), LCharset("windows-936", "Windows 936 (Chinese)"), LCharset("windows-949", "Windows 949 (Korean)"), LCharset("windows-950", "Windows 950 (Chinese)"), LCharset("windows-1250", "Windows 1250 (Latin 2)", _gdc_win_1250_mapping, "x-cp1250,cp1250"), LCharset("windows-1251", "Windows 1251 (Cyrillic)", _gdc_win_1251_mapping, "x-cp1251,cp1251"), LCharset("windows-1252", "Windows 1252 (Latin 1)", _gdc_win_1252_mapping, "x-cp1252,cp1252"), LCharset("windows-1253", "Windows 1253 (Greek)", _gdc_win_1253_mapping, "x-cp1253,cp1253"), LCharset("windows-1254", "Windows 1254 (Turkish)", _gdc_win_1254_mapping, "x-cp1254,cp1254"), LCharset("windows-1255", "Windows 1255 (Hebrew)", _gdc_win_1255_mapping, "x-cp1255,cp1255"), LCharset("windows-1256", "Windows 1256 (Arabic)", _gdc_win_1256_mapping, "x-cp1256,cp1256"), LCharset("windows-1257", "Windows 1257 (Baltic)", _gdc_win_1257_mapping, "x-cp1257,cp1257"), LCharset("windows-1258", "Windows 1258 (Veitnam)", _gdc_win_1258_mapping, "x-cp1258,cp1258"), // Russian LCharset("koi8-r", "KOI8-R", _gdc_koi8r_mapping, "csKOI8R"), LCharset("koi8-u", "KOI8-U", _gdc_koi8u_mapping, "csKOI8U"), LCharset("koi8-ru", "KOI8-RU", _gdc_koi8ru_mapping, "csKOI8RU"), LCharset("koi8-t", "KOI8-T (Tajik)"), // Codepages LCharset("cp850", "Cp850", 0, "IBM850,850,csPC850Multilingual"), LCharset("cp862", "Cp862", 0, "IBM862,862,csPC862LatinHebrew"), LCharset("cp866", "Cp866", 0, "IBM866,866,csIBM866"), LCharset("cp1133", "Cp1133 (Laotian)"), // Japanese LCharset("euc-jp", "EUC-JP", 0, "csEUCPkdFmtJapanese"), LCharset("shift_jis", "SHIFT_JIS", 0, "MS_Kanji,csShiftJIS"), LCharset("cp932", "cp932", 0, 0), LCharset("iso-2022-jp", "ISO-2022-JP", 0, "csISO2022JP"), LCharset("iso-2022-jp-1", "ISO-2022-JP-1"), LCharset("iso-2022-jp-2", "ISO-2022-JP-2", 0, "csISO2022JP2"), // Chinese LCharset("euc-cn", "EUC-CN (Chinese)"), LCharset("hz-gb-2312", "HZ (Chinese)", 0, "hz"), LCharset("gbk", "GBK (Chinese)", 0, "CP936,MS936,windows-936,x-gbk,gb2312,GB-2312,csGB2312,GB2312_CHARSET"), LCharset("gb18030", "GB18030 (Chinese)"), LCharset("euc-tw", "EUC-TW (Chinese)"), LCharset("big5", "BIG5 (Chinese)", 0, "csBig5"), LCharset("big5-hkscs", "BIG5-HKSCS (Chinese)"), // LCharset("gb2312", "GB-2312 (Chinese)", 0, "GB-2312,csGB2312"), LCharset("iso-2022-cn", "ISO-2022-CN (Chinese)"), LCharset("iso-2022-cn-eXT","ISO-2022-CN-EXT (Chinese)"), // Korean LCharset("euc-kr", "EUC-KR", 0, "csEUCKR"), LCharset("iso-2022-kr", "ISO-2022-KR", 0, "csISO2022KR"), LCharset("johab", "JOHAB"), LCharset("cp949", "CP949", 0, "ks_c_5601-1987,ks_c_5601"), // Armenian // LCharset("armscii-8", "ARMSCII-8 (Armenian)"), // Georgian LCharset("Georgian-Academy","Georgian-Academy"), LCharset("Georgian-PS", " Georgian-PS"), // Thai LCharset("tis-620", "TIS-620 (Thai)"), // Laotian LCharset("mulelao-1", "MuleLao-1"), // Vietnamese LCharset("viscii", "VISCII (Vietnamese)", 0, "csVISCII"), LCharset("tcvn", "TCVN (Vietnamese)"), // EOF marker LCharset() }; static LCharsetSystem CharsetSystem; LCharsetSystem *LCharsetSystem::Inst() { return &CharsetSystem; } LCharset *LGetCharsetInfo(const char *Cs) { return CharsetSystem.GetCsInfo(Cs); } ///////////////////////////////////////////////////////////////////////////// // Utf-16 conversion int LCpToAnsi(char *cp) { int Ansi = 0; if (cp && strnicmp(cp, "windows-", 8) == 0) { Ansi = atoi(cp+9); } return Ansi; } ssize_t LBufConvertCp(void *Out, const char *OutCp, ssize_t OutLen, const void *&In, const char *InCp, ssize_t &InLen) { int Status = 0; if (Out && OutCp && In && InCp) { LCharset *InInfo = LGetCharsetInfo(InCp); LCharset *OutInfo = LGetCharsetInfo(OutCp); if (InInfo && OutInfo) { char *In8 = (char*)In; uchar *Out8 = (uchar*)Out; if (InLen < 0) { switch (InInfo->Type) { case CpMapped: case CpUtf8: case CpIconv: InLen = (int)strlen((char*)In); break; case CpUtf16: case CpWindowsDb: InLen = StringLen((uint16*)In) << 1; break; case CpUtf32: InLen = StringLen((uint32_t*)In) << 2; break; default: LAssert(0); return 0; } } #ifdef WIN32 if (InInfo->Type == CpWindowsDb && OutInfo->Type == CpUtf16) { // mb -> unicode char Cp[32]; sprintf_s(Cp, sizeof(Cp), ".%s", InInfo->Charset + 8); setlocale(LC_ALL, Cp); void *Start = Out; while (OutLen >= sizeof(char16) && InLen > 0) { int s = mbtowc((char16*)Out, (char*)In, min(InLen, MB_CUR_MAX)); if (s > 0) { ((char*&)In) += s; InLen -= s; ((char16*&)Out)++; OutLen -= sizeof(char16); } else break; } return (NativeInt)Out-(NativeInt)Start; } else if (InInfo->Type == CpUtf16 && OutInfo->Type == CpWindowsDb) { // unicode -> mb char Cp[32]; sprintf_s(Cp, sizeof(Cp), ".%s", OutInfo->Charset + 8); setlocale(LC_ALL, Cp); void *Start = Out; while (OutLen >= MB_CUR_MAX && InLen > sizeof(char16) ) { #if 1 int s = 0; errno_t err = wctomb_s(&s, (char*)Out, OutLen, ((char16*)In)[0]); if (err || s == 0) break; #else int s = wctomb((char*)Out, ((char16*)In)[0] ); if (s > 0) #endif { ((char16*&)In)++; InLen -= sizeof(char16); ((char*&)Out) += s; OutLen -= s; } } return (NativeInt)Out-(NativeInt)Start; } else #endif if (InInfo->Type == CpIconv || OutInfo->Type == CpIconv) { #ifndef LGI_STATIC LFontSystem *Fs = LFontSystem::Inst(); if (Fs) return Fs->IconvConvert(OutInfo->GetIconvName(), (char*)Out, OutLen, InInfo->GetIconvName(), (const char*&)In, InLen); #else LAssert(!"No iconv in static build"); #endif } else { // Mapped or Utf conversion uint32_t Utf32 = 0; while (OutLen > 0 && InLen > 0) { char *RewindIn = In8; ptrdiff_t RewindInLen = InLen; // Convert input char to Utf-32 switch (InInfo->Type) { case CpMapped: { if (*In8) { uchar i = (uchar)*In8++; InLen--; if (i & 0x80) { Utf32 = InInfo->UnicodeMap[i - 0x80]; if (!Utf32) Utf32 = '?'; } else { Utf32 = i; } } else { Utf32 = 0; InLen = 0; } break; } case CpUtf8: { Utf32 = LgiUtf8To32((uint8_t *&)In8, InLen); break; } case CpUtf16: { Utf32 = LgiUtf16To32((const uint16_t *&)In8, InLen); if (Utf32 == 0xfeff || Utf32 == 0xfffe) continue; break; } case CpUtf32: { Utf32 = *((uint32_t*&)In8)++; InLen -= 4; break; } default: LAssert(0); break; } if (!Utf32) { break; } // Convert Utf-32 into output format switch (OutInfo->Type) { case CpMapped: { if (Utf32 & ~0x7f) { int n; for (n=0; n<128; n++) { if (OutInfo->UnicodeMap[n] == Utf32) { *Out8++ = 0x80 + n; break; } } if (n >= 128) { for (n=0; MissingMaps[n].Unicode; n++) { if (MissingMaps[n].Unicode == Utf32) { *Out8++ = MissingMaps[n].Ascii; break; } } if (!MissingMaps[n].Unicode) { *Out8++ = '?'; } } } else { *Out8++ = Utf32; } OutLen--; break; } case CpUtf8: { // uchar *PrevOut8 = Out8; if (!LgiUtf32To8(Utf32, (uint8_t*&) Out8, OutLen)) { // Not enough buffer to encode the character In8 = RewindIn; InLen = RewindInLen; OutLen = 0; } break; } case CpUtf16: { LgiUtf32To16(Utf32, (uint16_t*&)Out8, OutLen); break; } case CpUtf32: { *((uint32_t*&)Out8)++ = Utf32; OutLen -= 4; break; } default: { break; } } } In = (void*)In8; Status = (int) (Out8 - (uchar*)Out); } } else { // printf("%s:%i - LBufConvertCp failed '%s' -> '%s'.\n", __FILE__, __LINE__, InCp, OutCp); } } return Status; } template T *DupeString(T *s, ssize_t Len = -1) { if (!s) return NULL; if (Len < 0) { Len = 0; while (s[Len]) Len++; } T *ns = new T[Len+1]; if (!ns) return NULL; memcpy(ns, s, sizeof(T) * Len); ns[Len] = 0; return ns; } LString LStrConvertCp(const char *OutCp, const void *In, const char *InCp, ssize_t InLen) { if (!OutCp || !In || !InCp) return LString(); LCharset *InInfo = LGetCharsetInfo(InCp); LCharset *OutInfo = LGetCharsetInfo(OutCp); if (!InInfo || !OutInfo) return LString(); if (InLen < 0) { switch (InInfo->Type) { case CpMapped: case CpUtf8: case CpIconv: InLen = (int)strlen((char*)In); break; case CpUtf16: case CpWindowsDb: InLen = StringLen((uint16*)In) << 1; break; case CpUtf32: InLen = StringLen((uint32_t*)In) << 2; break; default: return LString(); } } switch (OutInfo->Type) { case CpMapped: case CpUtf8: case CpIconv: break; case CpUtf16: case CpWindowsDb: case CpUtf32: default: LAssert(!"LString doesn't >8bit char (yet)."); return LString(); } if (!stricmp(InCp, OutCp)) return LString((char*)In, InLen); LStringPipe b; if (InInfo->Type == CpIconv || OutInfo->Type == CpIconv) { #ifndef LGI_STATIC LFontSystem *Fs = LFontSystem::Inst(); if (Fs) { auto InCs = InInfo->GetIconvName(); auto OutCs = OutInfo->GetIconvName(); if (Fs->IconvConvert(OutCs, &b, InCs, (const char*&)In, InLen)) return b.NewLStr(); InCp = "iso-8859-1"; } #else LAssert(!"No inconv in static build"); #endif } char Buf[2 << 10]; while (InLen > 0) { ssize_t Bytes = LBufConvertCp(Buf, OutCp, sizeof(Buf), In, InCp, InLen); if (Bytes > 0) b.Write((uchar*)Buf, (int)Bytes); else break; } return b.NewLStr(); } void *LNewConvertCp(const char *OutCharset, const void *In, const char *InCharset, ssize_t InLen) { if (!OutCharset || !In || !InCharset) return NULL; auto InInfo = LGetCharsetInfo(InCharset); auto OutInfo = LGetCharsetInfo(OutCharset); if (!InInfo || !OutInfo) return NULL; LMemQueue b; if (InLen < 0) { switch (InInfo->Type) { case CpMapped: case CpUtf8: case CpIconv: InLen = (int)strlen((char*)In); break; case CpUtf16: case CpWindowsDb: InLen = StringLen((uint16*)In) << 1; break; case CpUtf32: InLen = StringLen((uint32_t*)In) << 2; break; default: LAssert(0); return NULL; } } int NullSize; switch (OutInfo->Type) { case CpMapped: case CpUtf8: case CpIconv: NullSize = 1; break; case CpUtf16: case CpWindowsDb: NullSize = 2; break; case CpUtf32: NullSize = 4; break; default: LAssert(0); return NULL; } if (!stricmp(InCharset, OutCharset)) { if (InInfo->Type == CpUtf16) return DupeString((uint16*)In, InLen/sizeof(uint16)); else if (InInfo->Type == CpUtf32) return DupeString((uint32_t*)In, InLen/sizeof(uint32_t)); else return NewStr((char*)In, InLen); } if (InInfo->Type == CpIconv || OutInfo->Type == CpIconv) { #ifndef LGI_STATIC LFontSystem *Fs = LFontSystem::Inst(); if (Fs) { const char *InCs = InInfo->GetIconvName(); const char *OutCs = OutInfo->GetIconvName(); if (!Fs->IconvConvert(OutCs, &b, InCs, (const char*&)In, InLen)) { InCharset = "iso-8859-1"; goto BufConvert; } } #else LAssert(!"No inconv in static build"); #endif } else { BufConvert: char Buf[2 << 10]; while (InLen > 0) { ssize_t Bytes = LBufConvertCp(Buf, OutCharset, sizeof(Buf), In, InCharset, InLen); if (Bytes > 0) { b.Write((uchar*)Buf, (int)Bytes); } else { break; } } } return b.GetSize() ? b.New(NullSize) : 0; } int LCharLen(const void *Str, const char *Cp, int Bytes) { if (Str && Cp) { LCharset *InInfo = LGetCharsetInfo(Cp); if (InInfo) { switch (InInfo->Type) { default: case CpMapped: { return (int)strlen((char*)Str); } case CpUtf8: { uchar *s = (uchar*)Str; int Len = 0; if (Bytes > 0) { uchar *e = s + Bytes; while (*s && s < e) { LgiNextUtf8((char*&)s); Len++; } } else { while (*s) { LgiNextUtf8((char*&)s); Len++; } } return Len; } case CpUtf16: { return StringLen((uint16*)Str); } case CpUtf32: { return StringLen((uint32_t*)Str); } } } } return 0; } bool LIsCpImplemented(char *Cp) { return LGetCharsetInfo(Cp) != 0; } const char *LAnsiToLgiCp(int AnsiCodePage) { if (AnsiCodePage < 0) { #ifdef WIN32 AnsiCodePage = GetACP(); #else return "utf-8"; #endif } #define WinCp(i) case i: return "windows-" #i; switch (AnsiCodePage) { WinCp(874) WinCp(932) WinCp(936) WinCp(949) WinCp(950) WinCp(1250) WinCp(1251) WinCp(1252) WinCp(1253) WinCp(1254) WinCp(1255) WinCp(1256) WinCp(1257) WinCp(1258) case 20127: return "us-ascii"; case 28591: return "iso-8859-1"; case 28592: return "iso-8859-2"; case 28593: return "iso-8859-3"; case 28594: return "iso-8859-4"; case 28595: return "iso-8859-5"; case 28596: return "iso-8859-6"; case 28597: return "iso-8859-7"; case 28598: return "iso-8859-8"; case 28599: return "iso-8859-9"; case 28600: return "ISO-8859-10"; case 28605: return "ISO-8859-15"; case 50220: case 50221: return "iso-2022-jp"; case 51932: return "euc-jp"; case 51949: return "euc-kr"; case 65001: return "utf-8"; } #undef WinCp return 0; } char *LSeekUtf8(const char *Ptr, ssize_t D, char *Start) { uchar *p = (uchar*)Ptr; if (p) { if (D >= 0) { for (int i=0; i(uchar*)Start; i++) { p--; while (p>(uchar*)Start && IsUtf8_Trail(*p)) p--; } } else { // You must pass a start point to move backwards in // the utf-8 string, otherwise you can run off the // beginning of the array. LAssert(0); } } return (char*)p; } bool LMatchCharset(short *Map, char16 *Utf, bool &Has8Bit) { if (Map && Utf) { // Test Charset because we have a map of all the chars in it... char16 *c; for (c = Utf; *c; c++) { if (*c > 0x7f) { // Check char Has8Bit = true; int i; for (i=0; i<128; i++) { if (Map[i] == *c) break; } if (i >= 128) { // Char not found return false; } } } if (Has8Bit) { if (!*c) { return true; } } } return false; } -const char *LUnicodeToCharset(const char *Utf8, ssize_t Len, List *Prefs) +const char *LUnicodeToCharset(const char *Utf8, ssize_t Len, LString::Array *Prefs) { const char *Status = "utf-8"; // The default.. LAutoWString Utf((char16*)LNewConvertCp(LGI_WideCharset, Utf8, "utf-8", Len)); if (Utf) { if (Prefs) { for (auto p: *Prefs) { LCharset *Cp = CharsetSystem.GetCsInfo(p); if (Cp && stricmp(Cp->Charset, "us-ascii") != 0 && Cp->UnicodeMap) { bool Has8Bit = false; if (LMatchCharset(Cp->UnicodeMap, Utf, Has8Bit)) { return Cp->Charset; } if (!Has8Bit) { return "us-ascii"; } } } } for (LCharset *Cp = LgiCharsets + 1; Cp->Charset; Cp++) { if (Cp->UnicodeMap) { bool Has8Bit = false; if (LMatchCharset(Cp->UnicodeMap, Utf, Has8Bit)) { return Cp->Charset; } if (!Has8Bit) { return "us-ascii"; } } } } return Status; } LString LToNativeCp(const char *In, ssize_t InLen) { const char *Cp = LAnsiToLgiCp(); LString s; #ifdef WIN32 LCharset *CpInfo = LGetCharsetInfo(Cp); if (!CpInfo || CpInfo->Type == CpWindowsDb) { if (In) { // Double byte charset conversion, don't rely on iconv // being around to do the conversion. setlocale(LC_ALL, ".ACP"); if (InLen < 0) InLen = strlen(In); LAutoWString Wide(Utf8ToWide(In, InLen)); if (Wide) { size_t Converted; auto Len = wcstombs_s(&Converted, NULL, 0, Wide, 0); if (s.Length(Len)) { wcstombs_s(&Converted, s.Get(), Len+1, Wide, Len+1); s.Get()[Len] = 0; } } } } #endif if (!s) s = LStrConvertCp(Cp, In, "utf-8", InLen); return s; } LString LFromNativeCp(const char *In, ssize_t InLen) { const char *Cp = LAnsiToLgiCp(); LString s; #ifdef WIN32 LCharset *CpInfo = LGetCharsetInfo(Cp); if (!CpInfo || CpInfo->Type == CpWindowsDb) { if (In) { // Double byte charset conversion, don't rely on iconv // being around to do the conversion. setlocale(LC_ALL, ".ACP"); if (InLen < 0) { #ifdef __GNUC__ // FIXME InLen = strlen(In); #else InLen = _mbstrlen(In); #endif } else { // Work out how many chars 'InLen' bytes is ssize_t Bytes = InLen; const char *i = In; int Chars = 0; while (*i && Bytes > 0) { int n = mblen(i, MB_CUR_MAX); if (n > 0) { Chars++; Bytes -= n; i += n; } else break; } InLen = Chars; } size_t Converted; size_t Len = mbstowcs_s(&Converted, NULL, 0, In, 0); if (Len) { LAutoWString Buf(new char16[Len+1]); if (Buf) { mbstowcs_s(&Converted, Buf, Len, In, Len); Buf[Len] = 0; s = Buf; } } } } #endif if (!s) s = LStrConvertCp("utf-8", In, Cp, InLen); return s; } /////////////////////////////////////////////////////////////////////////// struct LCharsetSystemPriv { LCharset *Utf8; LCharset *Utf16; LHashTbl, LCharset*> Charsets; LCharsetSystemPriv() : Charsets(512) { Utf8 = 0; Utf16 = 0; } }; LCharsetSystem::LCharsetSystem() { char l[256]; // Charset setup, store all the charset pointers // in a hash table for O(1) lookup. d = new LCharsetSystemPriv; LAssert(LgiCharsets->Charset != NULL); for (LCharset *Cs = LgiCharsets; Cs->Charset; Cs++) { strcpy_s(l, sizeof(l), Cs->Charset); #ifdef _MSC_VER _strlwr_s(l, sizeof(l)); #else strlwr(l); #endif if (!stricmp(l, "utf-8")) d->Utf8 = Cs; else if (!stricmp(l, "utf-16")) d->Utf16 = Cs; d->Charsets.Add(l, Cs); auto a = LString(Cs->AlternateNames).SplitDelimit(","); for (int n=0; nCharsets.Add(l, Cs); } } } LCharsetSystem::~LCharsetSystem() { DeleteObj(d); } LCharset *LCharsetSystem::GetCsInfo(const char *Cp) { if (Cp && d) { // Lookup the charset in the hash table char l[256]; strcpy_s(l, sizeof(l), Cp); #ifdef _MSC_VER _strlwr_s(l, sizeof(l)); #else strlwr(l); #endif if (!stricmp(l, "utf-8")) return d->Utf8; else if (!stricmp(l, "utf-16")) return d->Utf16; LCharset *Cs = (LCharset*) d->Charsets.Find(l); if (Cs) { return Cs; } else { // printf("%s:%i - No charset '%s' in font sub system.\n", __FILE__, __LINE__, l); // printf("Charsets=%i\n", Charsets->GetUsed()); } } return 0; } LCharset *LGetCsInfo(const char *Cs) { return CharsetSystem.GetCsInfo(Cs); } LCharset *LCharsetSystem::GetCsList() { return LgiCharsets; } LCharset *LGetCsList() { return LgiCharsets; } diff --git a/src/common/Text/TextConvert.cpp b/src/common/Text/TextConvert.cpp --- a/src/common/Text/TextConvert.cpp +++ b/src/common/Text/TextConvert.cpp @@ -1,474 +1,512 @@ #include "lgi/common/Lgi.h" #include "lgi/common/TextConvert.h" #include "lgi/common/Mime.h" #include "lgi/common/Base64.h" #include "lgi/common/Charset.h" // return true if there are any characters with the 0x80 bit set bool Is8Bit(const char *Text) { if (!Text) return false; while (*Text) { if (*Text & 0x80) return true; Text++; } return false; } char ConvHexToBin(char c) { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'a' && c <= 'f') return c + 10 - 'a'; if (c >= 'A' && c <= 'F') return c + 10 - 'A'; return 0; } char *DecodeBase64Str(char *Str, ssize_t Len) { if (Str) { ssize_t B64Len = (Len < 0) ? strlen(Str) : Len; ssize_t BinLen = BufferLen_64ToBin(B64Len); char *s = new char[BinLen+1]; if (s) { ssize_t Converted = ConvertBase64ToBinary((uchar*)s, BinLen, Str, B64Len); s[Converted] = 0; DeleteArray(Str); Str = s; } } return Str; } LString LDecodeBase64Str(LString Str) { LString r; ssize_t BinLen = BufferLen_64ToBin(Str.Length()); if (Str && r.Length(BinLen) > 0) { ssize_t Converted = ConvertBase64ToBinary((uchar*)r.Get(), r.Length(), Str.Get(), Str.Length()); if (Converted >= 0) r.Get()[Converted] = 0; else r.Empty(); } return r; } char *DecodeQuotedPrintableStr(char *Str, ssize_t Len) { if (Str) { if (Len < 0) Len = strlen(Str); uchar *s = new uchar[Len+1]; if (s) { char *Out = (char*) s; char *Text = Str; for (int i=0; i s) p.Write(s, e - s); break; } e++; } if (Decode) { // is there a word remaining bool Encoded = false; char *Start = e + 2; char *First = strchr(Start, '?'); char *Second = First ? strchr(First + 1, '?') : NULL; char *End = Second ? strstr(Second + 1, "?=") : NULL; if (End) { LString Cp(Start, First - Start); int Type = CONTENT_NONE; bool StripUnderscores = false; if (ToUpper(First[1]) == 'B') { // Base64 encoding Type = CONTENT_BASE64; } else if (ToUpper(First[1]) == 'Q') { // Quoted printable Type = CONTENT_QUOTED_PRINTABLE; StripUnderscores = true; } if (Type != CONTENT_NONE) { Second++; LString Block(Second, End - Second); if (Block) { switch (Type) { case CONTENT_BASE64: Block = LDecodeBase64Str(Block); break; case CONTENT_QUOTED_PRINTABLE: Block = LDecodeQuotedPrintableStr(Block); break; } if (StripUnderscores) { for (char *i=Block; *i; i++) { if (*i == '_') *i = ' '; } } if (Cp && !_stricmp(Cp, "utf-8")) { p.Write(Block); } else { auto Inst = LCharsetSystem::Inst(); LString Detect = Inst && Inst->DetectCharset ? Inst->DetectCharset(Block) : LString(); LAutoString Utf8((char*)LNewConvertCp("utf-8", Block, Detect ? Detect : Cp, Block.Length())); if (Utf8) { if (LIsUtf8(Utf8)) p.Write((uchar*)Utf8.Get(), strlen(Utf8)); } else { p.Write(Block); } } } s = End + 2; if (*s == '\n') { s++; while (*s && strchr(WhiteSpace, *s)) s++; } Encoded = true; } } if (!Encoded) { // Encoding error, just emit the raw string and exit. size_t Len = strlen(s); p.Write((uchar*) s, Len); break; } } else if (Descape) { // Un-escape the string... e++; if (*e) p.Write(e, 1); else break; s = e + 1; } else { // Last segment of string... LAssert(*e == 0); if (e > s) p.Write(s, e - s); break; } } DeleteArray(Str); return p.NewStr(); } #define MIME_MAX_LINE 76 -static void EncodeRfc2047_Impl( char *Str, size_t Length, - const char *Charset, - List *CharsetPrefs, +static void EncodeRfc2047_Impl( char *Input, size_t Length, + const char *InCharset, + LString::Array *OutCharsets, ssize_t LineLength, std::function Process) { - if (!Str) + if (!Input) return; - if (!Charset) - Charset = "utf-8"; + if (!InCharset) + InCharset = "utf-8"; LStringPipe p(256); - if (Is8Bit(Str)) + if (Is8Bit(Input)) { // pick an encoding bool Base64 = false; - const char *DestCp = "utf-8"; - if (Stricmp(Charset, "utf-8") == 0) - DestCp = LUnicodeToCharset(Str, Length, CharsetPrefs); + const char *DestCs = "utf-8"; + char *Buf = NULL; + if (!OutCharsets || + (OutCharsets && OutCharsets->Length() == 0) || + Stristr(InCharset, "utf") != NULL) + { + if (!Stricmp(InCharset, "utf-8")) + { + auto DetectedCs = LUnicodeToCharset(Input, Length, OutCharsets); + if (DetectedCs) + Buf = (char*)LNewConvertCp(DestCs = DetectedCs, Input, InCharset, Length); + } + else + { + // Not utf-8 so convert to that first... + LAutoString utf8((char*)LNewConvertCp("utf-8", Input, InCharset)); + if (utf8) + { + auto DetectCs = LUnicodeToCharset(utf8, Strlen(utf8.Get()), OutCharsets); + if (DetectCs) + Buf = (char*)LNewConvertCp(DestCs = DetectCs, Input, InCharset, Length); + } + } + } + else + { + for (auto Cs: *OutCharsets) + { + if (Buf = (char*)LNewConvertCp(DestCs = Cs, Input, InCharset, Length)) + break; + } + if (!Buf || !DestCs) + { + // Fall back to utf-8 + Buf = (char*)LNewConvertCp(DestCs = "utf-8", Input, InCharset, Length); + } + } + + if (!Buf) + { + // Fall back to copy... + Buf = NewStr(Input, Length); + if (InCharset) + DestCs = InCharset; + } int Chars = 0; - for (unsigned i=0; i 0 && - ((double)Chars/Length) > 0.4 - ) + Length > 0 && + ((double)Chars/Length) > 0.4 ) { Base64 = true; } - char *Buf = (char*)LNewConvertCp(DestCp, Str, Charset, Length); if (Buf) { // encode the word char Prefix[64]; - int Ch = sprintf_s(Prefix, sizeof(Prefix), "=?%s?%c?", DestCp, Base64 ? 'B' : 'Q'); + int Ch = sprintf_s(Prefix, sizeof(Prefix), "=?%s?%c?", DestCs, Base64 ? 'B' : 'Q'); p.Write(Prefix, Ch); LineLength += Ch; if (Base64) { // Base64 size_t InLen = strlen(Buf); // int EstBytes = BufferLen_BinTo64(InLen); char Temp[512]; ssize_t Bytes = ConvertBinaryToBase64(Temp, sizeof(Temp), (uchar*)Buf, InLen); p.Push(Temp, Bytes); } else { // Quoted printable for (char *w = Buf; *w; w++) { if (*w == ' ') { if (LineLength > MIME_MAX_LINE - 3) { p.Print("?=\r\n\t%s", Prefix); LineLength = 1 + strlen(Prefix); } p.Write((char*)"_", 1); LineLength++; } else if (*w & 0x80 || *w == '_' || *w == '?' || *w == '=') { if (LineLength > MIME_MAX_LINE - 5) { p.Print("?=\r\n\t%s", Prefix); LineLength = 1 + strlen(Prefix); } char Temp[16]; Ch = sprintf_s(Temp, sizeof(Temp), "=%2.2X", (uchar)*w); p.Write(Temp, Ch); LineLength += Ch; } else { if (LineLength > MIME_MAX_LINE - 3) { p.Print("?=\r\n\t%s", Prefix); LineLength = 1 + strlen(Prefix); } p.Write(w, 1); LineLength++; } } } p.Push("?="); DeleteArray(Buf); } - Process(Str, p); + Process(Input, p); } else { bool RecodeNewLines = false; - for (char *s = Str; *s; s++) + for (char *s = Input; *s; s++) { - if (*s == '\n' && (s == Str || s[-1] != '\r')) + if (*s == '\n' && (s == Input || s[-1] != '\r')) { RecodeNewLines = true; break; } } if (RecodeNewLines) { - for (char *s = Str; *s; s++) + for (char *s = Input; *s; s++) { if (*s == '\r') ; else if (*s == '\n') p.Write("\r\n", 2); else p.Write(s, 1); } - Process(Str, p); + Process(Input, p); } } // It's not an error to not call 'OutputStr', in that case // the input is passed through to the output unchanged. } // Old heap string encode method (will eventually remove this...) -char *EncodeRfc2047(char *Str, const char *Charset, List *CharsetPrefs, ssize_t LineLength) +char *EncodeRfc2047(char *Str, const char *Charset, LString::Array *CharsetPrefs, ssize_t LineLength) { char *Out = Str; EncodeRfc2047_Impl( Str, Strlen(Str), Charset, CharsetPrefs, LineLength, [&Out](auto s, auto &pipe) { DeleteArray(s); Out = pipe.NewStr(); }); return Out; } // New LString encode method -LString LEncodeRfc2047(LString Str, const char *Charset, List *CharsetPrefs, ssize_t LineLength) +LString LEncodeRfc2047(LString Str, const char *Charset, LString::Array *CharsetPrefs, ssize_t LineLength) { EncodeRfc2047_Impl( Str.Get(), Str.Length(), Charset, CharsetPrefs, LineLength, [&Str](auto s, auto &pipe) { Str = pipe.NewLStr(); }); return Str; } diff --git a/test/UnitTests/UnitTests.vcxproj b/test/UnitTests/UnitTests.vcxproj --- a/test/UnitTests/UnitTests.vcxproj +++ b/test/UnitTests/UnitTests.vcxproj @@ -1,277 +1,278 @@  Debug Win32 Debug x64 Release Win32 Release x64 {18BF30E1-D77B-496E-8761-99A426DD3B41} 10.0 Application v142 false MultiByte Application v142 false MultiByte Application v142 false MultiByte Application v142 false MultiByte <_ProjectFileVersion>12.0.30501.0 $(Platform)$(Configuration)14\ $(Platform)$(Configuration)14\ true true $(Platform)$(Configuration)14\ $(Platform)$(Configuration)14\ $(Platform)$(Configuration)14\ $(Platform)$(Configuration)14\ false false $(Platform)$(Configuration)14\ $(Platform)$(Configuration)14\ .\Debug/UnitTests.tlb Disabled ..\..\include;..\..\include\lgi\win;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_CONSOLE;WINDOWS;LGI_UNIT_TESTS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL true $(IntDir)/UnitTests.pch $(IntDir) $(IntDir) $(IntDir)vc$(PlatformToolsetVersion).pdb true ProgramDatabase _DEBUG;%(PreprocessorDefinitions) 0x0c09 $(OutDir)$(TargetName)$(TargetExt) true true $(IntDir)/UnitTests.pdb Console false MachineX86 true .\Debug/UnitTests.bsc .\Debug/UnitTests.tlb Disabled ..\..\include;..\..\include\lgi\win;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_CONSOLE;WINDOWS;LGI_UNIT_TESTS;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebugDLL true $(IntDir)/UnitTests.pch $(IntDir) $(IntDir) $(IntDir)vc$(PlatformToolsetVersion).pdb true ProgramDatabase Level3 _DEBUG;%(PreprocessorDefinitions) 0x0c09 true true $(IntDir)/UnitTests.pdb Console false true .\Debug/UnitTests.bsc .\Release/UnitTests.tlb MinSpace OnlyExplicitInline ..\..\include;..\..\include\lgi\win;%(AdditionalIncludeDirectories) WIN32;NDEBUG;_CONSOLE;WINDOWS;LGI_UNIT_TESTS;%(PreprocessorDefinitions) true MultiThreadedDLL true true $(IntDir)/UnitTests.pch $(IntDir) $(IntDir) $(IntDir)vc$(PlatformToolsetVersion).pdb true NDEBUG;%(PreprocessorDefinitions) 0x0c09 $(OutDir)$(TargetName)$(TargetExt) true $(IntDir)/UnitTests.pdb Console false MachineX86 true .\Release/UnitTests.bsc .\Release/UnitTests.tlb MinSpace OnlyExplicitInline ..\..\include;..\..\include\lgi\win;%(AdditionalIncludeDirectories) WIN32;NDEBUG;_CONSOLE;WINDOWS;LGI_UNIT_TESTS;%(PreprocessorDefinitions) true MultiThreadedDLL true true $(IntDir)/UnitTests.pch $(IntDir) $(IntDir) $(IntDir)vc$(PlatformToolsetVersion).pdb true NDEBUG;%(PreprocessorDefinitions) 0x0c09 $(OutDir)$(TargetName)$(TargetExt) true $(IntDir)/UnitTests.pdb Console false true .\Release/UnitTests.bsc + {95df9ca4-6d37-4a85-a648-80c2712e0da1} \ No newline at end of file diff --git a/test/UnitTests/UnitTests.vcxproj.filters b/test/UnitTests/UnitTests.vcxproj.filters --- a/test/UnitTests/UnitTests.vcxproj.filters +++ b/test/UnitTests/UnitTests.vcxproj.filters @@ -1,74 +1,77 @@  {1b5d1f69-ef66-4844-baa5-25ec23d2151c} cpp;c;cxx;rc;def;r;odl;idl;hpj;bat {a734023f-ceb2-4882-9efc-5719c6bf4c8d} h;hpp;hxx;hm;inl {134d3911-e459-4919-8af8-82198f95a179} {8b34f4e6-17d2-42af-8c12-1386ba1acae0} Source Files Source Files Source Files Source Files Source Files Source Files Source Files Source Files Lgi Source Files\Testing Source Files Source Files Lgi Header Files Header Files Header Files Source Files\Testing + + Lgi + \ No newline at end of file diff --git a/test/UnitTests/src/StringTests.cpp b/test/UnitTests/src/StringTests.cpp --- a/test/UnitTests/src/StringTests.cpp +++ b/test/UnitTests/src/StringTests.cpp @@ -1,65 +1,76 @@ #include "lgi/common/Lgi.h" #include "lgi/common/TextConvert.h" #include "UnitTests.h" class PrivLStringTests { public: }; -LStringTests::LStringTests() : UnitTest("LStringPipeTest") +LStringTests::LStringTests() : UnitTest("LStringTests") { d = new PrivLStringTests; } LStringTests::~LStringTests() { DeleteObj(d); } bool LStringTests::Run() { /* Things to test: bool Is8Bit(const char *Text); [[deprecated]] char *DecodeBase64Str(char *Str, ssize_t Len = -1); LString LDecodeBase64Str(LString Str); [[deprecated]] char *DecodeQuotedPrintableStr(char *Str, ssize_t Len = -1); LString LDecodeQuotedPrintableStr(LString Str); [[deprecated]] char *DecodeRfc2047(char *Str); LString LDecodeRfc2047(LString Str); LString LEncodeRfc2047(LString Str, const char *Charset, List *CharsetPrefs, ssize_t LineLength = 0); */ - const char *EncodeResult1 = "=?utf-8?B?QmV5dHVsbGFoIEdlbsOn?="; + const char *EncodeResult1 = "=?iso-8859-1?Q?Beytullah_Gen=E7?="; const char *EncodeResult2 = "=?iso-8859-9?Q?Beytullah_Gen=E7?="; - List CharsetPrefs; + LString::Array CharsetPrefs; const char *Rfc2047Input = "Beytullah Genç"; + const char *UtfInput = "Beytullah Genç"; const char *Charset = "windows-1252"; - #if 0 - LAutoString result1( EncodeRfc2047(NewStr(Rfc2047Input), Charset, &CharsetPrefs)); + // No prefered charset testing: + LAutoString result1( EncodeRfc2047(NewStr(Rfc2047Input), Charset) ); if (Stricmp(result1.Get(), EncodeResult1)) return FAIL(_FL, "EncodeRfc2047"); - LString result2 = LEncodeRfc2047(Rfc2047Input, Charset, &CharsetPrefs); + LAutoString decode1( DecodeRfc2047(NewStr(result1)) ); + if (Strcmp(UtfInput, decode1.Get())) + return FAIL(_FL, "DecodeRfc2047"); + + LString result2 = LEncodeRfc2047(Rfc2047Input, Charset); if (Stricmp(result2.Get(), EncodeResult1)) return FAIL(_FL, "LEncodeRfc2047"); - #endif + LAutoString decode2( DecodeRfc2047(NewStr(result2)) ); + if (Strcmp(UtfInput, decode2.Get())) + return FAIL(_FL, "DecodeRfc2047"); + // Redo tests with a charset preference set: CharsetPrefs.Add("iso-8859-9"); - LAutoString result3( EncodeRfc2047(NewStr(Rfc2047Input), Charset, &CharsetPrefs)); - LString result4 = LEncodeRfc2047(Rfc2047Input, Charset, &CharsetPrefs); + if (Stricmp(result3.Get(), EncodeResult2)) + return FAIL(_FL, "EncodeRfc2047"); + LString result4 = LEncodeRfc2047(Rfc2047Input, Charset, &CharsetPrefs); + if (Stricmp(result4.Get(), EncodeResult2)) + return FAIL(_FL, "EncodeRfc2047"); return true; }