HPCC-Platform/system/jlib/junicode.cpp

/*##############################################################################

    Copyright (C) 2011 HPCC Systems.

    All rights reserved. This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
############################################################################## */


#include "platform.h"
#include "jliball.hpp"
#include "jerror.hpp"
#include "junicode.hpp"

/* Based on code extracted from the following source...  Changed quite signficantly */

/*
 * Copyright 2001 Unicode, Inc.
 * 
 * Disclaimer
 * 
 * This source code is provided as is by Unicode, Inc. No claims are
 * made as to fitness for any particular purpose. No warranties of any
 * kind are expressed or implied. The recipient agrees to determine
 * applicability of information provided. If this file has been
 * purchased on magnetic or optical media from Unicode, Inc., the
 * sole remedy for any claim will be exchange of defective media
 * within 90 days of receipt.
 * 
 * Limitations on Rights to Redistribute This Code
 * 
 * Unicode, Inc. hereby grants the right to freely use the information
 * supplied in this file in the creation of products supporting the
 * Unicode Standard, and to make copies of this file in any form
 * for internal or external distribution as long as this notice
 * remains attached.
 */

//----------------------------------------------------------------------------
static const int halfShift  = 10; /* used for shifting by 10 bits */

static const UTF32 halfBase = 0x0010000UL;
static const UTF32 halfMask = 0x3FFUL;

#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD
#define UNI_MAX_BMP (UTF32)0x0000FFFF
#define UNI_MAX_UTF16 (UTF32)0x0010FFFF
#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF

#define UNI_SUR_HIGH_START  (UTF32)0xD800
#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
#define UNI_SUR_LOW_START   (UTF32)0xDC00
#define UNI_SUR_LOW_END     (UTF32)0xDFFF


UTF32 UtfReader::next()
{
    switch (type)
    {
    case Utf8:    return next8();
    case Utf16le:  return next16le();
    case Utf16be:  return next16be();
    case Utf32le:  return next32le();
    case Utf32be:  return next32be();
    }
    UNIMPLEMENTED;
}

size32_t UtfReader::getLegalLength()
{
    const byte * saved = cur;
    while (next() < errorLowerLimit)
    {
    }
    size32_t ret = (size32_t)(cur-saved);
    cur = saved;
    return ret;
}

//---------------------------------------------------------------------------

UTF32 UtfReader::next32le()
{
    if (end - cur < 4) return sourceExhausted;
    UTF32 ch = *(UTF32 *)cur;
    if (strictConversion && ((ch >= UNI_SUR_HIGH_START) && (ch <= UNI_SUR_LOW_END)))
        return sourceIllegal;
    cur += sizeof(UTF32);
    return ch;
}

//---------------------------------------------------------------------------

UTF32 UtfReader::next32be()
{
    if (end - cur < 4) return sourceExhausted;
    UTF32 ch;
    _cpyrev4(&ch, cur);
    if (strictConversion && ((ch >= UNI_SUR_HIGH_START) && (ch <= UNI_SUR_LOW_END)))
        return sourceIllegal;
    cur += sizeof(UTF32);
    return ch;
}

//---------------------------------------------------------------------------

UTF32 UtfReader::next16le()
{
    if (end - cur < 2) return sourceExhausted;

    const byte * source = cur;
    UTF32 ch = source[0] | (source[1] << 8);
    source += 2;

    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END)
    {
        if (end - cur < 2)
            return sourceExhausted;

        UTF32 ch2 = source[0] | (source[1] << 8);
        source += 2;

        if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) 
        {
            ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
                + (ch2 - UNI_SUR_LOW_START) + halfBase;
        } 
        else if (strictConversion) /* it's an unpaired high surrogate */
            return sourceIllegal;
    } 
    else if ((strictConversion) && (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END))
        return sourceIllegal;

    cur = (const byte *)source;
    return ch;
}

//---------------------------------------------------------------------------

UTF32 UtfReader::next16be()
{
    if (end - cur < 2) return sourceExhausted;

    const byte * source = cur;
    UTF32 ch = (source[0] << 8) | source[1];
    source += 2;

    if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) 
    {
        if (end - cur < 2)
            return sourceExhausted;

        UTF32 ch2 = (source[0] << 8) | source[1];
        source += 2;

        if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) 
        {
            ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
                + (ch2 - UNI_SUR_LOW_START) + halfBase;
        } 
        else if (strictConversion) /* it's an unpaired high surrogate */
            return sourceIllegal;
    } 
    else if ((strictConversion) && (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END))
        return sourceIllegal;
    cur = source;
    return ch;
}

//---------------------------------------------------------------------------

//This is probably faster than a table lookup on modern processors since it would avoid a cache hit.
//Especially because first branch is the most common.
inline unsigned getTrailingBytesForUTF8(byte value)
{
    if (value < 0xc0)
        return 0;
    if (value < 0xe0)
        return 1;
    if (value < 0xf0)
        return 2;
    if (value < 0xf8)
        return 3;
    if (value < 0xfc)
        return 4;
    return 5;
}

/*
 * Magic values subtracted from a buffer value during UTF8 conversion.
 * This table contains as many values as there might be trailing bytes
 * in a UTF-8 sequence.
 */
static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 
                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };

/*
 * Utility routine to tell whether a sequence of bytes is legal UTF-8.
 * This must be called with the length pre-determined by the first byte.
 * If not calling this from ConvertUTF8to*, then the length can be set by:
 *  length = trailingBytesForUTF8[*source]+1;
 * and the sequence is illegal right away if there aren't that many bytes
 * available.
 * If presented with a length > 4, this returns false.  The Unicode
 * definition of UTF-8 goes up to 4-byte sequences.
 */
unsigned readUtf8Size(const void * _data)
{
    const byte * ptr = (const byte *)_data;
    return getTrailingBytesForUTF8(*ptr)+1;
}


UTF32 readUtf8Char(const void * _data)
{
    const byte * ptr = (const byte *)_data;
    unsigned short extraBytesToRead = getTrailingBytesForUTF8(*ptr);
    UTF32 ch = 0;
    switch (extraBytesToRead) {
        case 3: ch += *ptr++; ch <<= 6;
        case 2: ch += *ptr++; ch <<= 6;
        case 1: ch += *ptr++; ch <<= 6;
        case 0: ch += *ptr++;
    }
    return ch - offsetsFromUTF8[extraBytesToRead];
}

inline bool isLegalUTF8(const UTF8 *source, unsigned length) 
{
    UTF8 a;
    const UTF8 *srcptr = source+length;
    switch (length) 
    {
    default: return false;
        /* Everything else falls through when "true"... */
    case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
    case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
        switch (*source) 
        {
            /* no fall-through in this inner switch */
            case 0xE0: if (a < 0xA0) return false; break;
            case 0xF0: if (a < 0x90) return false; break;
            case 0xF4: if (a > 0x8F) return false; break;
            default:  if (a < 0x80) return false;
        }
        case 1: if (*source >= 0x80 && *source < 0xC2) return false;
        if (*source > 0xF4) return false;
    }
    return true;
}

/* --------------------------------------------------------------------- */

UTF32 UtfReader::next8()
{
    const UTF8* source = (const UTF8*)cur;
    if (source >= end) return sourceExhausted;

    unsigned short extraBytesToRead = getTrailingBytesForUTF8(*source);
    if (source + extraBytesToRead >= end)
        return sourceExhausted;

    /* Do this check whether lenient or strict */
    if (! isLegalUTF8(source, extraBytesToRead+1))
        return sourceIllegal;

    /*
     * The cases all fall through. See "Note A" below.
     */
    UTF32 ch = 0;
    switch (extraBytesToRead) {
        case 3: ch += *source++; ch <<= 6;
        case 2: ch += *source++; ch <<= 6;
        case 1: ch += *source++; ch <<= 6;
        case 0: ch += *source++;
    }
    cur = (const byte *)source;
    return ch - offsetsFromUTF8[extraBytesToRead];
}

//---------------------------------------------------------------------------

UTF32 readUtf8Character(unsigned len, const byte * & cur)
{
    const UTF8* source = (const UTF8*)cur;

    if (len == 0) return sourceExhausted;
    unsigned short extraBytesToRead = getTrailingBytesForUTF8(*source);
    if (extraBytesToRead >= len)
        return sourceExhausted;

    /* Do this check whether lenient or strict */
    if (! isLegalUTF8(source, extraBytesToRead+1))
        return sourceIllegal;

    /*
     * The cases all fall through. See "Note A" below.
     */
    UTF32 ch = 0;
    switch (extraBytesToRead) {
        case 3: ch += *source++; ch <<= 6;
        case 2: ch += *source++; ch <<= 6;
        case 1: ch += *source++; ch <<= 6;
        case 0: ch += *source++;
    }
    cur = (const byte *)source;
    return ch - offsetsFromUTF8[extraBytesToRead];
}

/*
 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
 * into the first byte, depending on how many bytes follow.  There are
 * as many entries in this table as there are UTF-8 sequence types.
 * (I.e., one byte sequence, two byte... six byte sequence.)
 */
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
static const UTF32 byteMask = 0xBF;
static const UTF32 byteMark = 0x80; 

unsigned writeUtf8(void * vtarget, unsigned maxLength, UTF32 ch)
{
    unsigned short bytesToWrite;
    /* Figure out how many bytes the result will require */
    if (ch < (UTF32)0x80) 
        bytesToWrite = 1;
    else if (ch < (UTF32)0x800) 
        bytesToWrite = 2;
    else if (ch < (UTF32)0x10000) 
        bytesToWrite = 3;
    else if (ch < (UTF32)0x200000)
        bytesToWrite = 4;
    else {              
        bytesToWrite = 2;
        ch = UNI_REPLACEMENT_CHAR;
    }

    if (bytesToWrite > maxLength)
        return 0;

    UTF8 * target = (UTF8 *)vtarget + bytesToWrite;
    switch (bytesToWrite) { /* note: everything falls through. */
        case 4: *--target = (ch | byteMark) & byteMask; ch >>= 6;
        case 3: *--target = (ch | byteMark) & byteMask; ch >>= 6;
        case 2: *--target = (ch | byteMark) & byteMask; ch >>= 6;
        case 1: *--target =  ch | firstByteMark[bytesToWrite];
    }
    return bytesToWrite;
}


unsigned writeUtf16le(void * vtarget, unsigned maxLength, UTF32 ch)
{
    if (maxLength < 2)
        return 0;

    UTF16 * target = (UTF16 *)vtarget;
    if (ch <= UNI_MAX_BMP) 
    { 
        /* Target is a character <= 0xFFFF */
        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
            ch = UNI_REPLACEMENT_CHAR;
        *target = ch;   /* normal case */
        return 2;
    } 
        
    if (ch > UNI_MAX_UTF16) 
    {
        *target = UNI_REPLACEMENT_CHAR;
        return 2;
    } 

    /* target is a character in range 0xFFFF - 0x10FFFF. */
    if (maxLength < 4)
        return 0;
    ch -= halfBase;
    target[0] = (ch >> halfShift) + UNI_SUR_HIGH_START;
    target[1] = (ch & halfMask) + UNI_SUR_LOW_START;
    return 4;
}

unsigned writeUtf16be(void * vtarget, unsigned maxLength, UTF32 ch)
{
    if (maxLength < 2)
        return 0;

    UTF16 * target = (UTF16 *)vtarget;
    UTF16 temp;
    if (ch <= UNI_MAX_BMP) 
    { 
        /* Target is a character <= 0xFFFF */
        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END)
            ch = UNI_REPLACEMENT_CHAR;
        temp = ch;
        _cpyrev2(target, &temp);    /* normal case */
        return 2;
    } 
        
    if (ch > UNI_MAX_UTF16) 
    {
        temp = UNI_REPLACEMENT_CHAR;
        _cpyrev2(target, &temp);
        return 2;
    } 

    /* target is a character in range 0xFFFF - 0x10FFFF. */
    if (maxLength < 4)
        return 0;
    ch -= halfBase;
    temp = (ch >> halfShift) + UNI_SUR_HIGH_START;
    _cpyrev2(target, &temp);
    temp = (ch & halfMask) + UNI_SUR_LOW_START;
    _cpyrev2(target+1, &temp);
    return 4;
}

unsigned writeUtf32le(void * vtarget, unsigned maxLength, UTF32 ch)
{
    if (maxLength < 4) return 0;
    *(UTF32 *)vtarget = ch;
    return 4;
}

unsigned writeUtf32be(void * vtarget, unsigned maxLength, UTF32 ch)
{
    if (maxLength < 4) return 0;
    _cpyrev4(vtarget, &ch);
    return 4;
}

//---------------------------------------------------------------------------

MemoryBuffer & appendUtf8(MemoryBuffer & out, UTF32 value)
{
    char temp[4];
    return out.append(writeUtf8(temp, sizeof(temp), value), temp);
}

MemoryBuffer & appendUtf16le(MemoryBuffer & out, UTF32 value)
{
    char temp[4];
    return out.append(writeUtf16le(temp, sizeof(temp), value), temp);
}

MemoryBuffer & appendUtf16be(MemoryBuffer & out, UTF32 value)
{
    char temp[4];
    return out.append(writeUtf16be(temp, sizeof(temp), value), temp);
}

MemoryBuffer & appendUtf32le(MemoryBuffer & out, UTF32 value)
{
    char temp[4];
    return out.append(writeUtf32le(temp, sizeof(temp), value), temp);
}

MemoryBuffer & appendUtf32be(MemoryBuffer & out, UTF32 value)
{
    char temp[4];
    return out.append(writeUtf32be(temp, sizeof(temp), value), temp);
}

MemoryBuffer & appendUtf(MemoryBuffer & out, UtfReader::UtfFormat targetType, UTF32 value)
{
    switch (targetType)
    {
    case UtfReader::Utf8:    appendUtf8(out, value); break;
    case UtfReader::Utf16le: appendUtf16le(out, value); break;
    case UtfReader::Utf16be: appendUtf16be(out, value); break;
    case UtfReader::Utf32le: appendUtf32le(out, value); break;
    case UtfReader::Utf32be: appendUtf32be(out, value); break;
    }
    return out;
}

/* ---------------------------------------------------------------------

    Note A.
    The fall-through switches in UTF-8 reading code save a
    temp variable, some decrements & conditionals.  The switches
    are equivalent to the following loop:
        {
            int tmpBytesToRead = extraBytesToRead+1;
            do {
                ch += *source++;
                --tmpBytesToRead;
                if (tmpBytesToRead) ch <<= 6;
            } while (tmpBytesToRead > 0);
        }
    In UTF-8 writing code, the switches on "bytesToWrite" are
    similarly unrolled loops.

   --------------------------------------------------------------------- */


bool convertUtf(MemoryBuffer & target, UtfReader::UtfFormat targetType, unsigned sourceLength, const void * source, UtfReader::UtfFormat sourceType)
{
    UtfReader input(sourceType, false);
    input.set(sourceLength, source);
    unsigned originalLength = target.length();
    loop
    {
        UTF32 next = input.next();
        if (next == sourceExhausted)
            return true;
        if (next == sourceIllegal)
        {
            target.setLength(originalLength);
            return false;
        }
        appendUtf(target, targetType, next);
    }
}

bool convertToUtf8(MemoryBuffer & target, unsigned sourceLength, const void * source)
{
    if (sourceLength < 2)
        return false;

    const byte * text = (const byte *)source;
    //check for leading BOM of 0xfeff in the appropriate encoding
    if ((text[0] == 0xfe) && (text[1] == 0xff))
        return convertUtf(target, UtfReader::Utf8, sourceLength-2, text+2, UtfReader::Utf16be);

    if ((text[0] == 0xff) && (text[1] == 0xfe))
    {
        if (sourceLength >= 4 && (text[2] == 0) && (text[3] == 0))
            return convertUtf(target, UtfReader::Utf8, sourceLength-4, text+4, UtfReader::Utf32le);
        return convertUtf(target, UtfReader::Utf8, sourceLength-2, text+2, UtfReader::Utf16le);
    }
    if (sourceLength > 4 && (text[0] == 0) && (text[1] == 0) && (text[2] == 0xfe) && (text[3] == 0xff))
        return convertUtf(target, UtfReader::Utf8, sourceLength-4, text+4, UtfReader::Utf32be);

    //Try and guess the format
    if (text[0] && !text[1])
    {
        if (text[2])
        {
            if (convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf16le))
                return true;
        }
        else
        {
            if (convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf32le))
                return true;
        }
    }
    else if (!text[0])
    {
        if (text[1])
        {
            if (convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf16be))
                return true;
        }
        else
        {
            if (convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf32be))
                return true;
        }
    }

    //No idea first one that matches wins!
    return
        convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf16le) ||
        convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf16be) ||
        convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf32le) ||
        convertUtf(target, UtfReader::Utf8, sourceLength, source, UtfReader::Utf32be);
}

//---------------------------------------------------------------------------

void addUtfActionList(StringMatcher & matcher, const char * text, unsigned action, unsigned * maxElementLength, UtfReader::UtfFormat utfFormat)
{
    if (!text)
        return;

    unsigned idx=0;
    while (*text)
    {
        StringBuffer str;
        while (*text)
        {
            char next = *text++;
            if (next == ',')
                break;
            if (next == '\\' && *text)
            {
                next = *text++;
                switch (next)
                {
                case 'r': next = '\r'; break;
                case 'n': next = '\n'; break;
                case 't': next = '\t'; break;
                case 'x':
                    //hex constant - at least we can define spaces then...
                    if (text[0] && text[1])
                    {
                        next = (hex2num(*text) << 4) | hex2num(text[1]);
                        text+=2;
                    }
                    break;
                default:
                    break; //otherwise \ just quotes the character e.g. \,
                }
            }
            str.append(next);
        }
        if (str.length())
        {
            MemoryBuffer converted;
            if (!convertUtf(converted, utfFormat, str.length(), str.str(), UtfReader::Utf8))
               throwError(JLIBERR_BadUtf8InArguments);
            matcher.queryAddEntry(converted.length(), converted.toByteArray(), action+(idx++<<8));
            if (maxElementLength && (converted.length() > *maxElementLength))
                *maxElementLength = converted.length();
        }
    }
}

extern jlib_decl bool replaceUtf(utfReplacementFunc func, MemoryBuffer & target, UtfReader::UtfFormat type, unsigned sourceLength, const void * source)
{
    UtfReader input(type, false);
    input.set(sourceLength, source);
    unsigned originalLength = target.length();
    loop
    {
        const byte * cur = input.cur;
        UTF32 next = input.next();
        if (next == sourceExhausted)
            return true;
        if (next == sourceIllegal)
        {
            target.setLength(originalLength);
            return false;
        }
        func(target, next, type, cur, input.cur-cur, cur==source);
    }
}

struct utf32ValidXmlCharRange
{
    UTF32 min;
    UTF32 max;
    bool start;
};

utf32ValidXmlCharRange utf32ValidXmlCharRanges[] = {
    {'0', '9', false},
    {'A', 'Z', true},
    {'a', 'z', true},
    {0xC0, 0xD6, true},
    {0xD8, 0xF6, true},
    {0xF8, 0x2FF, true},
    {0x300, 0x36F, false},
    {0x370, 0x37D, true},
    {0x37F, 0x1FFF, true},
    {0x200C, 0x200D, true},
    {0x203F, 0x2040, false},
    {0x2070, 0x218F, true},
    {0x2C00, 0x2FEF, true},
    {0x3001, 0xD7FF, true},
    {0xF900, 0xFDCF, true},
    {0xFDF0, 0xFFFD, true},
    {0x10000, 0xEFFFF, true},
    {0, 0, false}
};

inline bool replaceBelowRange(UTF32 match, UTF32 replace, int id, MemoryBuffer & target, UtfReader::UtfFormat type, const void * source, int len, bool start)
{
    utf32ValidXmlCharRange &r = utf32ValidXmlCharRanges[id];
    if (r.min==0)
        return true;
    if (match>r.max)
        return false;
    if (match<r.min)
    {
        appendUtf(target, type, replace);
        return true;
    }
    if (!r.start && start)
        appendUtf(target, type, replace);
    else
        target.append(len, source); //src and target are same, no need to reconvert
    return true;
}

MemoryBuffer & utfXmlNameReplacementFunc(MemoryBuffer & target, UTF32 match, UtfReader::UtfFormat type, const void * source, int len, bool start)
{
    if (match==':' || match=='_' || (!start && (match=='-' || match=='.' || match==0xB7)))
        return target.append(len, source);

    for (int i=0; !replaceBelowRange(match, '_', i, target, type, source, len, start); i++);

    return target;
}

extern jlib_decl bool appendUtfXmlName(MemoryBuffer & target, UtfReader::UtfFormat type, unsigned sourceLength, const void * source)
{
    return replaceUtf(utfXmlNameReplacementFunc, target, type, sourceLength, source);
}