| | 1 | /* |
| | 2 | ********************************************************************** |
| | 3 | * Copyright (C) 2005-2014, International Business Machines |
| | 4 | * Corporation and others. All Rights Reserved. |
| | 5 | ********************************************************************** |
| | 6 | */ |
| | 7 | |
| | 8 | #include "unicode/utypes.h" |
| | 9 | |
| | 10 | #if !UCONFIG_NO_CONVERSION |
| | 11 | |
| | 12 | #include "csrutf8.h" |
| | 13 | #include "csmatch.h" |
| | 14 | |
| | 15 | U_NAMESPACE_BEGIN |
| | 16 | |
| | 17 | CharsetRecog_UTF8::~CharsetRecog_UTF8() |
| | 18 | { |
| | 19 | // nothing to do |
| | 20 | } |
| | 21 | |
| | 22 | const char *CharsetRecog_UTF8::getName() const |
| | 23 | { |
| | 24 | return "UTF-8"; |
| | 25 | } |
| | 26 | |
| | 27 | UBool CharsetRecog_UTF8::match(InputText* input, CharsetMatch *results) const { |
| | 28 | bool hasBOM = FALSE; |
| | 29 | int32_t numValid = 0; |
| | 30 | int32_t numInvalid = 0; |
| | 31 | const uint8_t *inputBytes = input->fRawInput; |
| | 32 | int32_t i; |
| | 33 | int32_t trailBytes = 0; |
| | 34 | int32_t confidence; |
| | 35 | |
| | 36 | if (input->fRawLength >= 3 && |
| | 37 | inputBytes[0] == 0xEF && inputBytes[1] == 0xBB && inputBytes[2] == 0xBF) { |
| | 38 | hasBOM = TRUE; |
| | 39 | } |
| | 40 | |
| | 41 | // Scan for multi-byte sequences |
| | 42 | for (i=0; i < input->fRawLength; i += 1) { |
| | 43 | int32_t b = inputBytes[i]; |
| | 44 | |
| | 45 | if ((b & 0x80) == 0) { |
| | 46 | continue; // ASCII |
| | 47 | } |
| | 48 | |
| | 49 | // Hi bit on char found. Figure out how long the sequence should be |
| | 50 | if ((b & 0x0E0) == 0x0C0) { |
| | 51 | trailBytes = 1; |
| | 52 | } else if ((b & 0x0F0) == 0x0E0) { |
| | 53 | trailBytes = 2; |
| | 54 | } else if ((b & 0x0F8) == 0xF0) { |
| | 55 | trailBytes = 3; |
| | 56 | } else { |
| | 57 | numInvalid += 1; |
| | 58 | continue; |
| | 59 | } |
| | 60 | |
| | 61 | // Verify that we've got the right number of trail bytes in the sequence |
| | 62 | for (;;) { |
| | 63 | i += 1; |
| | 64 | |
| | 65 | if (i >= input->fRawLength) { |
| | 66 | break; |
| | 67 | } |
| | 68 | |
| | 69 | b = inputBytes[i]; |
| | 70 | |
| | 71 | if ((b & 0xC0) != 0x080) { |
| | 72 | numInvalid += 1; |
| | 73 | break; |
| | 74 | } |
| | 75 | |
| | 76 | if (--trailBytes == 0) { |
| | 77 | numValid += 1; |
| | 78 | break; |
| | 79 | } |
| | 80 | } |
| | 81 | |
| | 82 | } |
| | 83 | |
| | 84 | // Cook up some sort of confidence score, based on presence of a BOM |
| | 85 | // and the existence of valid and/or invalid multi-byte sequences. |
| | 86 | confidence = 0; |
| | 87 | if (hasBOM && numInvalid == 0) { |
| | 88 | confidence = 100; |
| | 89 | } else if (hasBOM && numValid > numInvalid*10) { |
| | 90 | confidence = 80; |
| | 91 | } else if (numValid > 3 && numInvalid == 0) { |
| | 92 | confidence = 100; |
| | 93 | } else if (numValid > 0 && numInvalid == 0) { |
| | 94 | confidence = 80; |
| | 95 | } else if (numValid == 0 && numInvalid == 0) { |
| | 96 | // Plain ASCII. Confidence must be > 10, it's more likely than UTF-16, which |
| | 97 | // accepts ASCII with confidence = 10. |
| | 98 | confidence = 15; |
| | 99 | } else if (numValid > numInvalid*10) { |
| | 100 | // Probably corruput utf-8 data. Valid sequences aren't likely by chance. |
| | 101 | confidence = 25; |
| | 102 | } |
| | 103 | |
| | 104 | results->set(input, this, confidence); |
| | 105 | return (confidence > 0); |
| | 106 | } |
| | 107 | |
| | 108 | U_NAMESPACE_END |
| | 109 | #endif |
projects / apple / icu.git / blame_incremental