LCSOGthb / Tools

"use strict";

var Buffer = require("safer-buffer").Buffer;

// Multibyte codec. In this scheme, a character is represented by 1 or more bytes.

// Our codec supports UTF-16 surrogates, extensions for GB18030 and unicode sequences.

// To save memory and loading time, we read table files only when requested.

exports._dbcs = DBCSCodec;

var UNASSIGNED = -1,

    GB18030_CODE = -2,

    SEQ_START  = -10,

    NODE_START = -1000,

    UNASSIGNED_NODE = new Array(0x100),

    DEF_CHAR = -1;

for (var i = 0; i < 0x100; i++)

    UNASSIGNED_NODE[i] = UNASSIGNED;

// Class DBCSCodec reads and initializes mapping tables.

function DBCSCodec(codecOptions, iconv) {

    this.encodingName = codecOptions.encodingName;

    if (!codecOptions)

        throw new Error("DBCS codec is called without the data.")

    if (!codecOptions.table)

        throw new Error("Encoding '" + this.encodingName + "' has no data.");

    // Load tables.

    var mappingTable = codecOptions.table();

    // Decode tables: MBCS -> Unicode.

    // decodeTables is a trie, encoded as an array of arrays of integers. Internal arrays are trie nodes and all have len = 256.

    // Trie root is decodeTables[0].

    // Values: >=  0 -> unicode character code. can be > 0xFFFF

    //         == UNASSIGNED -> unknown/unassigned sequence.

    //         == GB18030_CODE -> this is the end of a GB18030 4-byte sequence.

    //         <= NODE_START -> index of the next node in our trie to process next byte.

    //         <= SEQ_START  -> index of the start of a character code sequence, in decodeTableSeq.

    this.decodeTables = [];

    this.decodeTables[0] = UNASSIGNED_NODE.slice(0); // Create root node.

    // Sometimes a MBCS char corresponds to a sequence of unicode chars. We store them as arrays of integers here. 

    this.decodeTableSeq = [];

    // Actual mapping tables consist of chunks. Use them to fill up decode tables.

    for (var i = 0; i < mappingTable.length; i++)

        this._addDecodeChunk(mappingTable[i]);

    this.defaultCharUnicode = iconv.defaultCharUnicode;

    // Encode tables: Unicode -> DBCS.

    // `encodeTable` is array mapping from unicode char to encoded char. All its values are integers for performance.

    // Because it can be sparse, it is represented as array of buckets by 256 chars each. Bucket can be null.

    // Values: >=  0 -> it is a normal char. Write the value (if <=256 then 1 byte, if <=65536 then 2 bytes, etc.).

    //         == UNASSIGNED -> no conversion found. Output a default char.

    //         <= SEQ_START  -> it's an index in encodeTableSeq, see below. The character starts a sequence.

    this.encodeTable = [];

    // `encodeTableSeq` is used when a sequence of unicode characters is encoded as a single code. We use a tree of

    // objects where keys correspond to characters in sequence and leafs are the encoded dbcs values. A special DEF_CHAR key

    // means end of sequence (needed when one sequence is a strict subsequence of another).

    // Objects are kept separately from encodeTable to increase performance.

    this.encodeTableSeq = [];

    // Some chars can be decoded, but need not be encoded.

    var skipEncodeChars = {};

    if (codecOptions.encodeSkipVals)

        for (var i = 0; i < codecOptions.encodeSkipVals.length; i++) {

            var val = codecOptions.encodeSkipVals[i];

            if (typeof val === 'number')

                skipEncodeChars[val] = true;

            else

                for (var j = val.from; j <= val.to; j++)

                    skipEncodeChars[j] = true;

        }

    // Use decode trie to recursively fill out encode tables.

    this._fillEncodeTable(0, 0, skipEncodeChars);

    // Add more encoding pairs when needed.

    if (codecOptions.encodeAdd) {

        for (var uChar in codecOptions.encodeAdd)

            if (Object.prototype.hasOwnProperty.call(codecOptions.encodeAdd, uChar))

                this._setEncodeChar(uChar.charCodeAt(0), codecOptions.encodeAdd[uChar]);

    }

    this.defCharSB  = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)];

    if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?'];

    if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);

    // Load & create GB18030 tables when needed.

    if (typeof codecOptions.gb18030 === 'function') {

        this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.

        // Add GB18030 decode tables.

        var thirdByteNodeIdx = this.decodeTables.length;

        var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);

        var fourthByteNodeIdx = this.decodeTables.length;

        var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0);

        for (var i = 0x81; i <= 0xFE; i++) {

            var secondByteNodeIdx = NODE_START - this.decodeTables[0][i];

            var secondByteNode = this.decodeTables[secondByteNodeIdx];

            for (var j = 0x30; j <= 0x39; j++)

                secondByteNode[j] = NODE_START - thirdByteNodeIdx;

        }

        for (var i = 0x81; i <= 0xFE; i++)

            thirdByteNode[i] = NODE_START - fourthByteNodeIdx;

        for (var i = 0x30; i <= 0x39; i++)

            fourthByteNode[i] = GB18030_CODE

    }        

}

DBCSCodec.prototype.encoder = DBCSEncoder;

DBCSCodec.prototype.decoder = DBCSDecoder;

// Decoder helpers

DBCSCodec.prototype._getDecodeTrieNode = function(addr) {

    var bytes = [];

    for (; addr > 0; addr >>= 8)

        bytes.push(addr & 0xFF);

    if (bytes.length == 0)

        bytes.push(0);

    var node = this.decodeTables[0];

    for (var i = bytes.length-1; i > 0; i--) { // Traverse nodes deeper into the trie.

        var val = node[bytes[i]];

        if (val == UNASSIGNED) { // Create new node.

            node[bytes[i]] = NODE_START - this.decodeTables.length;

            this.decodeTables.push(node = UNASSIGNED_NODE.slice(0));

        }

        else if (val <= NODE_START) { // Existing node.

            node = this.decodeTables[NODE_START - val];

        }

        else

            throw new Error("Overwrite byte in " + this.encodingName + ", addr: " + addr.toString(16));

    }

    return node;

}

DBCSCodec.prototype._addDecodeChunk = function(chunk) {

    // First element of chunk is the hex mbcs code where we start.

    var curAddr = parseInt(chunk[0], 16);

    // Choose the decoding node where we'll write our chars.

    var writeTable = this._getDecodeTrieNode(curAddr);

    curAddr = curAddr & 0xFF;

    // Write all other elements of the chunk to the table.

    for (var k = 1; k < chunk.length; k++) {

        var part = chunk[k];

        if (typeof part === "string") { // String, write as-is.

            for (var l = 0; l < part.length;) {

                var code = part.charCodeAt(l++);

                if (0xD800 <= code && code < 0xDC00) { // Decode surrogate

                    var codeTrail = part.charCodeAt(l++);

                    if (0xDC00 <= codeTrail && codeTrail < 0xE000)

                        writeTable[curAddr++] = 0x10000 + (code - 0xD800) * 0x400 + (codeTrail - 0xDC00);

                    else

                        throw new Error("Incorrect surrogate pair in "  + this.encodingName + " at chunk " + chunk[0]);

                }

                else if (0x0FF0 < code && code <= 0x0FFF) { // Character sequence (our own encoding used)

                    var len = 0xFFF - code + 2;

                    var seq = [];

                    for (var m = 0; m < len; m++)

                        seq.push(part.charCodeAt(l++)); // Simple variation: don't support surrogates or subsequences in seq.

                    writeTable[curAddr++] = SEQ_START - this.decodeTableSeq.length;

                    this.decodeTableSeq.push(seq);

                }

                else

                    writeTable[curAddr++] = code; // Basic char

            }

        } 

        else if (typeof part === "number") { // Integer, meaning increasing sequence starting with prev character.

            var charCode = writeTable[curAddr - 1] + 1;

            for (var l = 0; l < part; l++)

                writeTable[curAddr++] = charCode++;

        }

        else

            throw new Error("Incorrect type '" + typeof part + "' given in "  + this.encodingName + " at chunk " + chunk[0]);

    }

    if (curAddr > 0xFF)

        throw new Error("Incorrect chunk in "  + this.encodingName + " at addr " + chunk[0] + ": too long" + curAddr);

}

// Encoder helpers

DBCSCodec.prototype._getEncodeBucket = function(uCode) {

    var high = uCode >> 8; // This could be > 0xFF because of astral characters.

    if (this.encodeTable[high] === undefined)

        this.encodeTable[high] = UNASSIGNED_NODE.slice(0); // Create bucket on demand.

    return this.encodeTable[high];

}

DBCSCodec.prototype._setEncodeChar = function(uCode, dbcsCode) {

    var bucket = this._getEncodeBucket(uCode);

    var low = uCode & 0xFF;

    if (bucket[low] <= SEQ_START)

        this.encodeTableSeq[SEQ_START-bucket[low]][DEF_CHAR] = dbcsCode; // There's already a sequence, set a single-char subsequence of it.

    else if (bucket[low] == UNASSIGNED)

        bucket[low] = dbcsCode;

}

DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) {

    // Get the root of character tree according to first character of the sequence.

    var uCode = seq[0];

    var bucket = this._getEncodeBucket(uCode);

    var low = uCode & 0xFF;

    var node;

    if (bucket[low] <= SEQ_START) {

        // There's already a sequence with  - use it.

        node = this.encodeTableSeq[SEQ_START-bucket[low]];

    }

    else {

        // There was no sequence object - allocate a new one.

        node = {};

        if (bucket[low] !== UNASSIGNED) node[DEF_CHAR] = bucket[low]; // If a char was set before - make it a single-char subsequence.

        bucket[low] = SEQ_START - this.encodeTableSeq.length;

        this.encodeTableSeq.push(node);

    }

    // Traverse the character tree, allocating new nodes as needed.

    for (var j = 1; j < seq.length-1; j++) {

        var oldVal = node[uCode];

        if (typeof oldVal === 'object')

            node = oldVal;

        else {

            node = node[uCode] = {}

            if (oldVal !== undefined)

                node[DEF_CHAR] = oldVal

        }

    }

    // Set the leaf to given dbcsCode.

    uCode = seq[seq.length-1];

    node[uCode] = dbcsCode;

}

DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) {

    var node = this.decodeTables[nodeIdx];

    for (var i = 0; i < 0x100; i++) {

        var uCode = node[i];

        var mbCode = prefix + i;

        if (skipEncodeChars[mbCode])

            continue;

        if (uCode >= 0)

            this._setEncodeChar(uCode, mbCode);

        else if (uCode <= NODE_START)

            this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars);

        else if (uCode <= SEQ_START)

            this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode);

    }

}

// == Encoder ==================================================================

function DBCSEncoder(options, codec) {

    // Encoder state

    this.leadSurrogate = -1;

    this.seqObj = undefined;

    // Static data

    this.encodeTable = codec.encodeTable;

    this.encodeTableSeq = codec.encodeTableSeq;

    this.defaultCharSingleByte = codec.defCharSB;

    this.gb18030 = codec.gb18030;

}

DBCSEncoder.prototype.write = function(str) {

    var newBuf = Buffer.alloc(str.length * (this.gb18030 ? 4 : 3)),

        leadSurrogate = this.leadSurrogate,

        seqObj = this.seqObj, nextChar = -1,

        i = 0, j = 0;

    while (true) {

        // 0. Get next character.

        if (nextChar === -1) {

            if (i == str.length) break;

            var uCode = str.charCodeAt(i++);

        }

        else {

            var uCode = nextChar;

            nextChar = -1;    

        }

        // 1. Handle surrogates.

        if (0xD800 <= uCode && uCode < 0xE000) { // Char is one of surrogates.

            if (uCode < 0xDC00) { // We've got lead surrogate.

                if (leadSurrogate === -1) {

                    leadSurrogate = uCode;

                    continue;

                } else {

                    leadSurrogate = uCode;

                    // Double lead surrogate found.

                    uCode = UNASSIGNED;

                }

            } else { // We've got trail surrogate.

                if (leadSurrogate !== -1) {

                    uCode = 0x10000 + (leadSurrogate - 0xD800) * 0x400 + (uCode - 0xDC00);

                    leadSurrogate = -1;

                } else {

                    // Incomplete surrogate pair - only trail surrogate found.

                    uCode = UNASSIGNED;

                }

            }

        }

        else if (leadSurrogate !== -1) {

            // Incomplete surrogate pair - only lead surrogate found.

            nextChar = uCode; uCode = UNASSIGNED; // Write an error, then current char.

            leadSurrogate = -1;

        }

        // 2. Convert uCode character.

        var dbcsCode = UNASSIGNED;

        if (seqObj !== undefined && uCode != UNASSIGNED) { // We are in the middle of the sequence

            var resCode = seqObj[uCode];

            if (typeof resCode === 'object') { // Sequence continues.

                seqObj = resCode;

                continue;

            } else if (typeof resCode == 'number') { // Sequence finished. Write it.

                dbcsCode = resCode;

            } else if (resCode == undefined) { // Current character is not part of the sequence.

                // Try default character for this sequence

                resCode = seqObj[DEF_CHAR];

                if (resCode !== undefined) {

                    dbcsCode = resCode; // Found. Write it.

                    nextChar = uCode; // Current character will be written too in the next iteration.

                } else {

                    // TODO: What if we have no default? (resCode == undefined)

                    // Then, we should write first char of the sequence as-is and try the rest recursively.

                    // Didn't do it for now because no encoding has this situation yet.

                    // Currently, just skip the sequence and write current char.

                }

            }

            seqObj = undefined;

        }

        else if (uCode >= 0) {  // Regular character

            var subtable = this.encodeTable[uCode >> 8];

            if (subtable !== undefined)

                dbcsCode = subtable[uCode & 0xFF];

            if (dbcsCode <= SEQ_START) { // Sequence start

                seqObj = this.encodeTableSeq[SEQ_START-dbcsCode];

                continue;

            }

            if (dbcsCode == UNASSIGNED && this.gb18030) {

                // Use GB18030 algorithm to find character(s) to write.

                var idx = findIdx(this.gb18030.uChars, uCode);

                if (idx != -1) {

                    var dbcsCode = this.gb18030.gbChars[idx] + (uCode - this.gb18030.uChars[idx]);

                    newBuf[j++] = 0x81 + Math.floor(dbcsCode / 12600); dbcsCode = dbcsCode % 12600;

                    newBuf[j++] = 0x30 + Math.floor(dbcsCode / 1260); dbcsCode = dbcsCode % 1260;

                    newBuf[j++] = 0x81 + Math.floor(dbcsCode / 10); dbcsCode = dbcsCode % 10;

                    newBuf[j++] = 0x30 + dbcsCode;

                    continue;

                }

            }

        }

        // 3. Write dbcsCode character.

        if (dbcsCode === UNASSIGNED)

            dbcsCode = this.defaultCharSingleByte;

        if (dbcsCode < 0x100) {

            newBuf[j++] = dbcsCode;

        }

        else if (dbcsCode < 0x10000) {

            newBuf[j++] = dbcsCode >> 8;   // high byte

            newBuf[j++] = dbcsCode & 0xFF; // low byte

        }

        else {

            newBuf[j++] = dbcsCode >> 16;

            newBuf[j++] = (dbcsCode >> 8) & 0xFF;

            newBuf[j++] = dbcsCode & 0xFF;

        }

    }

    this.seqObj = seqObj;

    this.leadSurrogate = leadSurrogate;

    return newBuf.slice(0, j);

}

DBCSEncoder.prototype.end = function() {

    if (this.leadSurrogate === -1 && this.seqObj === undefined)

        return; // All clean. Most often case.

    var newBuf = Buffer.alloc(10), j = 0;

    if (this.seqObj) { // We're in the sequence.

        var dbcsCode = this.seqObj[DEF_CHAR];

        if (dbcsCode !== undefined) { // Write beginning of the sequence.

            if (dbcsCode < 0x100) {

                newBuf[j++] = dbcsCode;

            }

            else {

                newBuf[j++] = dbcsCode >> 8;   // high byte

                newBuf[j++] = dbcsCode & 0xFF; // low byte

            }

        } else {

            // See todo above.

        }

        this.seqObj = undefined;

    }

    if (this.leadSurrogate !== -1) {

        // Incomplete surrogate pair - only lead surrogate found.

        newBuf[j++] = this.defaultCharSingleByte;

        this.leadSurrogate = -1;

    }

    return newBuf.slice(0, j);

}

// Export for testing

DBCSEncoder.prototype.findIdx = findIdx;

// == Decoder ==================================================================

function DBCSDecoder(options, codec) {

    // Decoder state

    this.nodeIdx = 0;

    this.prevBuf = Buffer.alloc(0);

    // Static data

    this.decodeTables = codec.decodeTables;

    this.decodeTableSeq = codec.decodeTableSeq;

    this.defaultCharUnicode = codec.defaultCharUnicode;

    this.gb18030 = codec.gb18030;

}

DBCSDecoder.prototype.write = function(buf) {

    var newBuf = Buffer.alloc(buf.length*2),

        nodeIdx = this.nodeIdx, 

        prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length,

        seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence.

        uCode;

    if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later.

        prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]);

    for (var i = 0, j = 0; i < buf.length; i++) {

        var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset];

        // Lookup in current trie node.

        var uCode = this.decodeTables[nodeIdx][curByte];

        if (uCode >= 0) { 

            // Normal character, just use it.

        }

        else if (uCode === UNASSIGNED) { // Unknown char.

            // TODO: Callback with seq.

            //var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);

            i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle).

            uCode = this.defaultCharUnicode.charCodeAt(0);

        }

        else if (uCode === GB18030_CODE) {

            var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);

            var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30);

            var idx = findIdx(this.gb18030.gbChars, ptr);

            uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx];

        }

        else if (uCode <= NODE_START) { // Go to next trie node.

            nodeIdx = NODE_START - uCode;

            continue;

        }

        else if (uCode <= SEQ_START) { // Output a sequence of chars.

            var seq = this.decodeTableSeq[SEQ_START - uCode];

            for (var k = 0; k < seq.length - 1; k++) {

                uCode = seq[k];

                newBuf[j++] = uCode & 0xFF;

                newBuf[j++] = uCode >> 8;

            }

            uCode = seq[seq.length-1];

        }

        else

            throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte);

        // Write the character to buffer, handling higher planes using surrogate pair.

        if (uCode > 0xFFFF) { 

            uCode -= 0x10000;

            var uCodeLead = 0xD800 + Math.floor(uCode / 0x400);

            newBuf[j++] = uCodeLead & 0xFF;

            newBuf[j++] = uCodeLead >> 8;

            uCode = 0xDC00 + uCode % 0x400;

        }

        newBuf[j++] = uCode & 0xFF;

        newBuf[j++] = uCode >> 8;

        // Reset trie node.

        nodeIdx = 0; seqStart = i+1;

    }

    this.nodeIdx = nodeIdx;

    this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset);

    return newBuf.slice(0, j).toString('ucs2');

}

DBCSDecoder.prototype.end = function() {

    var ret = '';

    // Try to parse all remaining chars.

    while (this.prevBuf.length > 0) {

        // Skip 1 character in the buffer.

        ret += this.defaultCharUnicode;

        var buf = this.prevBuf.slice(1);

        // Parse remaining as usual.

        this.prevBuf = Buffer.alloc(0);

        this.nodeIdx = 0;

        if (buf.length > 0)

            ret += this.write(buf);

    }

    this.nodeIdx = 0;

    return ret;

}

// Binary search for GB18030. Returns largest i such that table[i] <= val.

function findIdx(table, val) {

    if (table[0] > val)

        return -1;

    var l = 0, r = table.length;

    while (l < r-1) { // always table[l] <= val < table[r]

        var mid = l + Math.floor((r-l+1)/2);

        if (table[mid] <= val)

            l = mid;

        else

            r = mid;

    }

    return l;

}