icuSources/i18n/csrmbcs.h

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4  **********************************************************************
   5  *   Copyright (C) 2005-2012, International Business Machines
   6  *   Corporation and others.  All Rights Reserved.
   7  **********************************************************************
   8  */
   9
  10 #ifndef __CSRMBCS_H
  11 #define __CSRMBCS_H
  12
  13 #include "unicode/utypes.h"
  14
  15 #if !UCONFIG_NO_CONVERSION
  16
  17 #include "csrecog.h"
  18
  19 U_NAMESPACE_BEGIN
  20
  21 // "Character"  iterated character class.
  22 //    Recognizers for specific mbcs encodings make their "characters" available
  23 //    by providing a nextChar() function that fills in an instance of IteratedChar
  24 //    with the next char from the input.
  25 //    The returned characters are not converted to Unicode, but remain as the raw
  26 //    bytes (concatenated into an int) from the codepage data.
  27 //
  28 //  For Asian charsets, use the raw input rather than the input that has been
  29 //   stripped of markup.  Detection only considers multi-byte chars, effectively
  30 //   stripping markup anyway, and double byte chars do occur in markup too.
  31 //
  32 class IteratedChar : public UMemory
  33 {
  34 public:
  35     uint32_t charValue;             // 1-4 bytes from the raw input data
  36     int32_t  index;
  37     int32_t  nextIndex;
  38     UBool    error;
  39     UBool    done;
  40
  41 public:
  42     IteratedChar();
  43     //void reset();
  44     int32_t nextByte(InputText* det);
  45 };
  46
  47 #if U_PLATFORM_IS_DARWIN_BASED
  48 #define MAX_KEY_STRING_WITH_NULL 16
  49 #endif
  50
  51 class CharsetRecog_mbcs : public CharsetRecognizer {
  52
  53 protected:
  54     /**
  55      * Test the match of this charset with the input text data
  56      *      which is obtained via the CharsetDetector object.
  57      *
  58      * @param det  The CharsetDetector, which contains the input text
  59      *             to be checked for being in this charset.
  60      * @return     Two values packed into one int  (Damn java, anyhow)
  61      *             <br/>
  62      *             bits 0-7:  the match confidence, ranging from 0-100
  63      *             <br/>
  64      *             bits 8-15: The match reason, an enum-like value.
  65      */
  66 #if U_PLATFORM_IS_DARWIN_BASED
  67     int32_t match_mbcs(InputText* det, const uint16_t commonChars[], int32_t commonCharsLen, const uint8_t (*keyStrings)[MAX_KEY_STRING_WITH_NULL] ) const;
  68 #else
  69     int32_t match_mbcs(InputText* det, const uint16_t commonChars[], int32_t commonCharsLen) const;
  70 #endif
  71
  72 public:
  73
  74     virtual ~CharsetRecog_mbcs();
  75
  76     /**
  77      * Get the IANA name of this charset.
  78      * @return the charset name.
  79      */
  80
  81     const char *getName() const = 0;
  82     const char *getLanguage() const = 0;
  83     UBool match(InputText* input, CharsetMatch *results) const = 0;
  84
  85     /**
  86      * Get the next character (however many bytes it is) from the input data
  87      *    Subclasses for specific charset encodings must implement this function
  88      *    to get characters according to the rules of their encoding scheme.
  89      *
  90      *  This function is not a method of class IteratedChar only because
  91      *   that would require a lot of extra derived classes, which is awkward.
  92      * @param it  The IteratedChar "struct" into which the returned char is placed.
  93      * @param det The charset detector, which is needed to get at the input byte data
  94      *            being iterated over.
  95      * @return    True if a character was returned, false at end of input.
  96      */
  97     virtual UBool nextChar(IteratedChar *it, InputText *textIn) const = 0;
  98
  99 };
 100
 101
 102 /**
 103  *   Shift-JIS charset recognizer.
 104  *
 105  */
 106 class CharsetRecog_sjis : public CharsetRecog_mbcs {
 107 public:
 108     virtual ~CharsetRecog_sjis();
 109
 110     UBool nextChar(IteratedChar *it, InputText *det) const;
 111
 112     UBool match(InputText* input, CharsetMatch *results) const;
 113
 114     const char *getName() const;
 115     const char *getLanguage() const;
 116
 117 };
 118
 119
 120 /**
 121  *   EUC charset recognizers.  One abstract class that provides the common function
 122  *             for getting the next character according to the EUC encoding scheme,
 123  *             and nested derived classes for EUC_KR, EUC_JP, EUC_CN.
 124  *
 125  */
 126 class CharsetRecog_euc : public CharsetRecog_mbcs
 127 {
 128 public:
 129     virtual ~CharsetRecog_euc();
 130
 131     const char *getName() const = 0;
 132     const char *getLanguage() const = 0;
 133
 134     UBool match(InputText* input, CharsetMatch *results) const = 0;
 135     /*
 136      *  (non-Javadoc)
 137      *  Get the next character value for EUC based encodings.
 138      *  Character "value" is simply the raw bytes that make up the character
 139      *     packed into an int.
 140      */
 141     UBool nextChar(IteratedChar *it, InputText *det) const;
 142 };
 143
 144 /**
 145  * The charset recognize for EUC-JP.  A singleton instance of this class
 146  *    is created and kept by the public CharsetDetector class
 147  */
 148 class CharsetRecog_euc_jp : public CharsetRecog_euc
 149 {
 150 public:
 151     virtual ~CharsetRecog_euc_jp();
 152
 153     const char *getName() const;
 154     const char *getLanguage() const;
 155
 156     UBool match(InputText* input, CharsetMatch *results) const;
 157 };
 158
 159 /**
 160  * The charset recognize for EUC-KR.  A singleton instance of this class
 161  *    is created and kept by the public CharsetDetector class
 162  */
 163 class CharsetRecog_euc_kr : public CharsetRecog_euc
 164 {
 165 public:
 166     virtual ~CharsetRecog_euc_kr();
 167
 168     const char *getName() const;
 169     const char *getLanguage() const;
 170
 171     UBool match(InputText* input, CharsetMatch *results) const;
 172 };
 173
 174 /**
 175  *
 176  *   Big5 charset recognizer.
 177  *
 178  */
 179 class CharsetRecog_big5 : public CharsetRecog_mbcs
 180 {
 181 public:
 182     virtual ~CharsetRecog_big5();
 183
 184     UBool nextChar(IteratedChar* it, InputText* det) const;
 185
 186     const char *getName() const;
 187     const char *getLanguage() const;
 188
 189     UBool match(InputText* input, CharsetMatch *results) const;
 190 };
 191
 192
 193 /**
 194  *
 195  *   GB-18030 recognizer. Uses simplified Chinese statistics.
 196  *
 197  */
 198 class CharsetRecog_gb_18030 : public CharsetRecog_mbcs
 199 {
 200 public:
 201     virtual ~CharsetRecog_gb_18030();
 202
 203     UBool nextChar(IteratedChar* it, InputText* det) const;
 204
 205     const char *getName() const;
 206     const char *getLanguage() const;
 207
 208     UBool match(InputText* input, CharsetMatch *results) const;
 209 };
 210
 211 U_NAMESPACE_END
 212
 213 #endif
 214 #endif /* __CSRMBCS_H */