| // SYS-REQ-014, SYS-REQ-060, SYS-REQ-061, SYS-REQ-062, SYS-REQ-063: string escape and Unicode handling |
| package jsonparser |
| |
| import ( |
| "bytes" |
| "unicode/utf8" |
| ) |
| |
| // JSON Unicode stuff: see https://tools.ietf.org/html/rfc7159#section-7 |
| |
| const supplementalPlanesOffset = 0x10000 |
| const highSurrogateOffset = 0xD800 |
| const lowSurrogateOffset = 0xDC00 |
| |
| const basicMultilingualPlaneReservedOffset = 0xDFFF |
| const basicMultilingualPlaneOffset = 0xFFFF |
| |
| func combineUTF16Surrogates(high, low rune) rune { |
| return supplementalPlanesOffset + (high-highSurrogateOffset)<<10 + (low - lowSurrogateOffset) |
| } |
| |
| const badHex = -1 |
| |
| func h2I(c byte) int { |
| switch { |
| case c >= '0' && c <= '9': |
| return int(c - '0') |
| case c >= 'A' && c <= 'F': |
| return int(c - 'A' + 10) |
| case c >= 'a' && c <= 'f': |
| return int(c - 'a' + 10) |
| } |
| return badHex |
| } |
| |
| // decodeSingleUnicodeEscape decodes a single \uXXXX escape sequence. The prefix \u is assumed to be present and |
| // is not checked. |
| // In JSON, these escapes can either come alone or as part of "UTF16 surrogate pairs" that must be handled together. |
| // This function only handles one; decodeUnicodeEscape handles this more complex case. |
| func decodeSingleUnicodeEscape(in []byte) (rune, bool) { |
| // We need at least 6 characters total |
| if len(in) < 6 { |
| return utf8.RuneError, false |
| } |
| |
| // Convert hex to decimal |
| h1, h2, h3, h4 := h2I(in[2]), h2I(in[3]), h2I(in[4]), h2I(in[5]) |
| if h1 == badHex || h2 == badHex || h3 == badHex || h4 == badHex { |
| return utf8.RuneError, false |
| } |
| |
| // Compose the hex digits |
| return rune(h1<<12 + h2<<8 + h3<<4 + h4), true |
| } |
| |
| // isUTF16EncodedRune checks if a rune is in the range for non-BMP characters, |
| // which is used to describe UTF16 chars. |
| // Source: https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane |
| func isUTF16EncodedRune(r rune) bool { |
| return highSurrogateOffset <= r && r <= basicMultilingualPlaneReservedOffset |
| } |
| |
| func decodeUnicodeEscape(in []byte) (rune, int) { |
| if r, ok := decodeSingleUnicodeEscape(in); !ok { |
| // Invalid Unicode escape |
| return utf8.RuneError, -1 |
| } else if !isUTF16EncodedRune(r) { |
| // Valid Unicode escape in Basic Multilingual Plane. |
| // Note: a single \uXXXX escape produces r in [0, 0xFFFF], so r is always |
| // within the BMP. The former r <= basicMultilingualPlaneOffset guard was |
| // tautological and has been removed — the real discriminator is whether r |
| // falls in the UTF-16 surrogate range. |
| return r, 6 |
| } else if r2, ok := decodeSingleUnicodeEscape(in[6:]); !ok { // Note: previous decodeSingleUnicodeEscape success guarantees at least 6 bytes remain |
| // UTF16 "high surrogate" without manditory valid following Unicode escape for the "low surrogate" |
| return utf8.RuneError, -1 |
| } else if r2 < lowSurrogateOffset { |
| // Invalid UTF16 "low surrogate" |
| return utf8.RuneError, -1 |
| } else { |
| // Valid UTF16 surrogate pair |
| return combineUTF16Surrogates(r, r2), 12 |
| } |
| } |
| |
| // backslashCharEscapeTable: when '\X' is found for some byte X, it is to be replaced with backslashCharEscapeTable[X] |
| var backslashCharEscapeTable = [...]byte{ |
| '"': '"', |
| '\\': '\\', |
| '/': '/', |
| 'b': '\b', |
| 'f': '\f', |
| 'n': '\n', |
| 'r': '\r', |
| 't': '\t', |
| } |
| |
| // unescapeToUTF8 unescapes the single escape sequence starting at 'in' into 'out' and returns |
| // how many characters were consumed from 'in' and emitted into 'out'. |
| // If a valid escape sequence does not appear as a prefix of 'in', (-1, -1) to signal the error. |
| func unescapeToUTF8(in, out []byte) (inLen int, outLen int) { |
| if len(in) < 2 || in[0] != '\\' { |
| // Invalid escape due to insufficient characters for any escape or no initial backslash |
| return -1, -1 |
| } |
| |
| // https://tools.ietf.org/html/rfc7159#section-7 |
| switch e := in[1]; e { |
| case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': |
| // Valid basic 2-character escapes (use lookup table) |
| out[0] = backslashCharEscapeTable[e] |
| return 2, 1 |
| case 'u': |
| // Unicode escape |
| if r, inLen := decodeUnicodeEscape(in); inLen == -1 { |
| // Invalid Unicode escape |
| return -1, -1 |
| } else { |
| // Valid Unicode escape; re-encode as UTF8 |
| outLen := utf8.EncodeRune(out, r) |
| return inLen, outLen |
| } |
| } |
| |
| return -1, -1 |
| } |
| |
| // unescape unescapes the string contained in 'in' and returns it as a slice. |
| // If 'in' contains no escaped characters: |
| // Returns 'in'. |
| // Else, if 'out' is of sufficient capacity (guaranteed if cap(out) >= len(in)): |
| // 'out' is used to build the unescaped string and is returned with no extra allocation |
| // Else: |
| // A new slice is allocated and returned. |
| func Unescape(in, out []byte) ([]byte, error) { |
| firstBackslash := bytes.IndexByte(in, '\\') |
| if firstBackslash == -1 { |
| return in, nil |
| } |
| |
| // Get a buffer of sufficient size (allocate if needed) |
| if cap(out) < len(in) { |
| out = make([]byte, len(in)) |
| } else { |
| out = out[0:len(in)] |
| } |
| |
| // Copy the first sequence of unescaped bytes to the output and obtain a buffer pointer (subslice) |
| copy(out, in[:firstBackslash]) |
| in = in[firstBackslash:] |
| buf := out[firstBackslash:] |
| |
| // The loop always exits via break: either on error (MalformedStringEscapeError) |
| // or after copying the final non-escaped tail. The former `for len(in) > 0` |
| // guard was structurally always true on re-entry since the else branch always |
| // leaves at least the backslash character in `in`. |
| for { |
| // Unescape the next escaped character |
| inLen, bufLen := unescapeToUTF8(in, buf) |
| if inLen == -1 { |
| return nil, MalformedStringEscapeError |
| } |
| |
| in = in[inLen:] |
| buf = buf[bufLen:] |
| |
| // Copy everything up until the next backslash |
| nextBackslash := bytes.IndexByte(in, '\\') |
| if nextBackslash == -1 { |
| copy(buf, in) |
| buf = buf[len(in):] |
| break |
| } else { |
| copy(buf, in[:nextBackslash]) |
| buf = buf[nextBackslash:] |
| in = in[nextBackslash:] |
| } |
| } |
| |
| // Trim the out buffer to the amount that was actually emitted |
| return out[:len(out)-len(buf)], nil |
| } |