icuSources/i18n/unicode/ucsdet.h

   1 /*
   2  **********************************************************************
   3  *   Copyright (C) 2005-2007, International Business Machines
   4  *   Corporation and others.  All Rights Reserved.
   5  **********************************************************************
   6  *   file name:  ucsdet.h
   7  *   encoding:   US-ASCII
   8  *   indentation:4
   9  *
  10  *   created on: 2005Aug04
  11  *   created by: Andy Heninger
  12  *
  13  *   ICU Character Set Detection, API for C
  14  *
  15  *   Draft version 18 Oct 2005
  16  *
  17  */
  18
  19 #ifndef __UCSDET_H
  20 #define __UCSDET_H
  21
  22 #include "unicode/utypes.h"
  23
  24 #if !UCONFIG_NO_CONVERSION
  25 #include "unicode/uenum.h"
  26
  27 /**
  28  * \file
  29  * \brief C API: Charset Detection API
  30  *
  31  * This API provides a facility for detecting the
  32  * charset or encoding of character data in an unknown text format.
  33  * The input data can be from an array of bytes.
  34  * <p>
  35  * Character set detection is at best an imprecise operation.  The detection
  36  * process will attempt to identify the charset that best matches the characteristics
  37  * of the byte data, but the process is partly statistical in nature, and
  38  * the results can not be guaranteed to always be correct.
  39  * <p>
  40  * For best accuracy in charset detection, the input data should be primarily
  41  * in a single language, and a minimum of a few hundred bytes worth of plain text
  42  * in the language are needed.  The detection process will attempt to
  43  * ignore html or xml style markup that could otherwise obscure the content.
  44  */
  45
  46
  47 struct UCharsetDetector;
  48 /**
  49   * Structure representing a charset detector
  50   * @stable ICU 3.6
  51   */
  52 typedef struct UCharsetDetector UCharsetDetector;
  53
  54 struct UCharsetMatch;
  55 /**
  56   *  Opaque structure representing a match that was identified
  57   *  from a charset detection operation.
  58   *  @stable ICU 3.6
  59   */
  60 typedef struct UCharsetMatch UCharsetMatch;
  61
  62 /**
  63   *  Open a charset detector.
  64   *
  65   *  @param status Any error conditions occurring during the open
  66   *                operation are reported back in this variable.
  67   *  @return the newly opened charset detector.
  68   *  @stable ICU 3.6
  69   */
  70 U_STABLE UCharsetDetector * U_EXPORT2
  71 ucsdet_open(UErrorCode   *status);
  72
  73 /**
  74   * Close a charset detector.  All storage and any other resources
  75   *   owned by this charset detector will be released.  Failure to
  76   *   close a charset detector when finished with it can result in
  77   *   memory leaks in the application.
  78   *
  79   *  @param ucsd  The charset detector to be closed.
  80   *  @stable ICU 3.6
  81   */
  82 U_STABLE void U_EXPORT2
  83 ucsdet_close(UCharsetDetector *ucsd);
  84
  85 /**
  86   * Set the input byte data whose charset is to detected.
  87   *
  88   * Ownership of the input  text byte array remains with the caller.
  89   * The input string must not be altered or deleted until the charset
  90   * detector is either closed or reset to refer to different input text.
  91   *
  92   * @param ucsd   the charset detector to be used.
  93   * @param textIn the input text of unknown encoding.   .
  94   * @param len    the length of the input text, or -1 if the text
  95   *               is NUL terminated.
  96   * @param status any error conditions are reported back in this variable.
  97   *
  98   * @stable ICU 3.6
  99   */
 100 U_STABLE void U_EXPORT2
 101 ucsdet_setText(UCharsetDetector *ucsd, const char *textIn, int32_t len, UErrorCode *status);
 102
 103
 104 /** Set the declared encoding for charset detection.
 105  *  The declared encoding of an input text is an encoding obtained
 106  *  by the user from an http header or xml declaration or similar source that
 107  *  can be provided as an additional hint to the charset detector.
 108  *
 109  *  How and whether the declared encoding will be used during the
 110  *  detection process is TBD.
 111  *
 112  * @param ucsd      the charset detector to be used.
 113  * @param encoding  an encoding for the current data obtained from
 114  *                  a header or declaration or other source outside
 115  *                  of the byte data itself.
 116  * @param length    the length of the encoding name, or -1 if the name string
 117  *                  is NUL terminated.
 118  * @param status    any error conditions are reported back in this variable.
 119  *
 120  * @stable ICU 3.6
 121  */
 122 U_STABLE void U_EXPORT2
 123 ucsdet_setDeclaredEncoding(UCharsetDetector *ucsd, const char *encoding, int32_t length, UErrorCode *status);
 124
 125
 126 /**
 127  * Return the charset that best matches the supplied input data.
 128  *
 129  * Note though, that because the detection
 130  * only looks at the start of the input data,
 131  * there is a possibility that the returned charset will fail to handle
 132  * the full set of input data.
 133  * <p>
 134  * The returned UCharsetMatch object is owned by the UCharsetDetector.
 135  * It will remain valid until the detector input is reset, or until
 136  * the detector is closed.
 137  * <p>
 138  * The function will fail if
 139  *  <ul>
 140  *    <li>no charset appears to match the data.</li>
 141  *    <li>no input text has been provided</li>
 142  *  </ul>
 143  *
 144  * @param ucsd      the charset detector to be used.
 145  * @param status    any error conditions are reported back in this variable.
 146  * @return          a UCharsetMatch  representing the best matching charset,
 147  *                  or NULL if no charset matches the byte data.
 148  *
 149  * @stable ICU 3.6
 150  */
 151 U_STABLE const UCharsetMatch * U_EXPORT2
 152 ucsdet_detect(UCharsetDetector *ucsd, UErrorCode *status);
 153
 154
 155 /**
 156  *  Find all charset matches that appear to be consistent with the input,
 157  *  returning an array of results.  The results are ordered with the
 158  *  best quality match first.
 159  *
 160  *  Because the detection only looks at a limited amount of the
 161  *  input byte data, some of the returned charsets may fail to handle
 162  *  the all of input data.
 163  *  <p>
 164  *  The returned UCharsetMatch objects are owned by the UCharsetDetector.
 165  *  They will remain valid until the detector is closed or modified
 166  *
 167  * <p>
 168  * Return an error if
 169  *  <ul>
 170  *    <li>no charsets appear to match the input data.</li>
 171  *    <li>no input text has been provided</li>
 172  *  </ul>
 173  *
 174  * @param ucsd          the charset detector to be used.
 175  * @param matchesFound  pointer to a variable that will be set to the
 176  *                      number of charsets identified that are consistent with
 177  *                      the input data.  Output only.
 178  * @param status        any error conditions are reported back in this variable.
 179  * @return              A pointer to an array of pointers to UCharSetMatch objects.
 180  *                      This array, and the UCharSetMatch instances to which it refers,
 181  *                      are owned by the UCharsetDetector, and will remain valid until
 182  *                      the detector is closed or modified.
 183  * @stable ICU 3.6
 184  */
 185 U_STABLE const UCharsetMatch ** U_EXPORT2
 186 ucsdet_detectAll(UCharsetDetector *ucsd, int32_t *matchesFound, UErrorCode *status);
 187
 188
 189
 190 /**
 191  *  Get the name of the charset represented by a UCharsetMatch.
 192  *
 193  *  The storage for the returned name string is owned by the
 194  *  UCharsetMatch, and will remain valid while the UCharsetMatch
 195  *  is valid.
 196  *
 197  *  The name returned is suitable for use with the ICU conversion APIs.
 198  *
 199  *  @param ucsm    The charset match object.
 200  *  @param status  Any error conditions are reported back in this variable.
 201  *  @return        The name of the matching charset.
 202  *
 203  *  @stable ICU 3.6
 204  */
 205 U_STABLE const char * U_EXPORT2
 206 ucsdet_getName(const UCharsetMatch *ucsm, UErrorCode *status);
 207
 208 /**
 209  *  Get a confidence number for the quality of the match of the byte
 210  *  data with the charset.  Confidence numbers range from zero to 100,
 211  *  with 100 representing complete confidence and zero representing
 212  *  no confidence.
 213  *
 214  *  The confidence values are somewhat arbitrary.  They define an
 215  *  an ordering within the results for any single detection operation
 216  *  but are not generally comparable between the results for different input.
 217  *
 218  *  A confidence value of ten does have a general meaning - it is used
 219  *  for charsets that can represent the input data, but for which there
 220  *  is no other indication that suggests that the charset is the correct one.
 221  *  Pure 7 bit ASCII data, for example, is compatible with a
 222  *  great many charsets, most of which will appear as possible matches
 223  *  with a confidence of 10.
 224  *
 225  *  @param ucsm    The charset match object.
 226  *  @param status  Any error conditions are reported back in this variable.
 227  *  @return        A confidence number for the charset match.
 228  *
 229  *  @stable ICU 3.6
 230  */
 231 U_STABLE int32_t U_EXPORT2
 232 ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status);
 233
 234 /**
 235  *  Get the RFC 3066 code for the language of the input data.
 236  *
 237  *  The Charset Detection service is intended primarily for detecting
 238  *  charsets, not language.  For some, but not all, charsets, a language is
 239  *  identified as a byproduct of the detection process, and that is what
 240  *  is returned by this function.
 241  *
 242  *  CAUTION:
 243  *    1.  Language information is not available for input data encoded in
 244  *        all charsets. In particular, no language is identified
 245  *        for UTF-8 input data.
 246  *
 247  *    2.  Closely related languages may sometimes be confused.
 248  *
 249  *  If more accurate language detection is required, a linguistic
 250  *  analysis package should be used.
 251  *
 252  *  The storage for the returned name string is owned by the
 253  *  UCharsetMatch, and will remain valid while the UCharsetMatch
 254  *  is valid.
 255  *
 256  *  @param ucsm    The charset match object.
 257  *  @param status  Any error conditions are reported back in this variable.
 258  *  @return        The RFC 3066 code for the language of the input data, or
 259  *                 an empty string if the language could not be determined.
 260  *
 261  *  @stable ICU 3.6
 262  */
 263 U_STABLE const char * U_EXPORT2
 264 ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status);
 265
 266
 267 /**
 268   *  Get the entire input text as a UChar string, placing it into
 269   *  a caller-supplied buffer.  A terminating
 270   *  NUL character will be appended to the buffer if space is available.
 271   *
 272   *  The number of UChars in the output string, not including the terminating
 273   *  NUL, is returned.
 274   *
 275   *  If the supplied buffer is smaller than required to hold the output,
 276   *  the contents of the buffer are undefined.  The full output string length
 277   *  (in UChars) is returned as always, and can be used to allocate a buffer
 278   *  of the correct size.
 279   *
 280   *
 281   * @param ucsm    The charset match object.
 282   * @param buf     A UChar buffer to be filled with the converted text data.
 283   * @param cap     The capacity of the buffer in UChars.
 284   * @param status  Any error conditions are reported back in this variable.
 285   * @return        The number of UChars in the output string.
 286   *
 287   * @stable ICU 3.6
 288   */
 289 U_STABLE  int32_t U_EXPORT2
 290 ucsdet_getUChars(const UCharsetMatch *ucsm,
 291                  UChar *buf, int32_t cap, UErrorCode *status);
 292
 293
 294
 295 /**
 296   *  Get an iterator over the set of all detectable charsets -
 297   *  over the charsets that are known to the charset detection
 298   *  service.
 299   *
 300   *  The returned UEnumeration provides access to the names of
 301   *  the charsets.
 302   *
 303   *  The state of the Charset detector that is passed in does not
 304   *  affect the result of this function, but requiring a valid, open
 305   *  charset detector as a parameter insures that the charset detection
 306   *  service has been safely initialized and that the required detection
 307   *  data is available.
 308   *
 309   *  @param ucsd a Charset detector.
 310   *  @param status  Any error conditions are reported back in this variable.
 311   *  @return an iterator providing access to the detectable charset names.
 312   *  @stable ICU 3.6
 313   */
 314 U_STABLE  UEnumeration * U_EXPORT2
 315 ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd,  UErrorCode *status);
 316
 317
 318 /**
 319   *  Test whether input filtering is enabled for this charset detector.
 320   *  Input filtering removes text that appears to be HTML or xml
 321   *  markup from the input before applying the code page detection
 322   *  heuristics.
 323   *
 324   *  @param ucsd  The charset detector to check.
 325   *  @return TRUE if filtering is enabled.
 326   *  @stable ICU 3.6
 327   */
 328 U_STABLE  UBool U_EXPORT2
 329 ucsdet_isInputFilterEnabled(const UCharsetDetector *ucsd);
 330
 331
 332 /**
 333  * Enable filtering of input text. If filtering is enabled,
 334  * text within angle brackets ("<" and ">") will be removed
 335  * before detection, which will remove most HTML or xml markup.
 336  *
 337  * @param ucsd   the charset detector to be modified.
 338  * @param filter <code>true</code> to enable input text filtering.
 339  * @return The previous setting.
 340  *
 341  * @stable ICU 3.6
 342  */
 343 U_STABLE  UBool U_EXPORT2
 344 ucsdet_enableInputFilter(UCharsetDetector *ucsd, UBool filter);
 345
 346 #endif
 347 #endif   /* __UCSDET_H */
 348
 349