icu/unicode/utf.h

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 1999-2010, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  utf.h
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 1999sep09
  14 *   created by: Markus W. Scherer
  15 */
  16
  17 /**
  18  * \file
  19  * \brief C API: Code point macros
  20  *
  21  * This file defines macros for checking whether a code point is
  22  * a surrogate or a non-character etc.
  23  *
  24  * The UChar and UChar32 data types for Unicode code units and code points
  25  * are defined in umachines.h because they can be machine-dependent.
  26  *
  27  * utf.h is included by utypes.h and itself includes utf8.h and utf16.h after some
  28  * common definitions. Those files define macros for efficiently getting code points
  29  * in and out of UTF-8/16 strings.
  30  * utf16.h macros have "U16_" prefixes.
  31  * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling.
  32  *
  33  * ICU processes 16-bit Unicode strings.
  34  * Most of the time, such strings are well-formed UTF-16.
  35  * Single, unpaired surrogates must be handled as well, and are treated in ICU
  36  * like regular code points where possible.
  37  * (Pairs of surrogate code points are indistinguishable from supplementary
  38  * code points encoded as pairs of supplementary code units.)
  39  *
  40  * In fact, almost all Unicode code points in normal text (>99%)
  41  * are on the BMP (<=U+ffff) and even <=U+d7ff.
  42  * ICU functions handle supplementary code points (U+10000..U+10ffff)
  43  * but are optimized for the much more frequently occurring BMP code points.
  44  *
  45  * utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then
  46  * UChar is defined to be exactly wchar_t, otherwise uint16_t.
  47  *
  48  * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit
  49  * Unicode code point (Unicode scalar value, 0..0x10ffff).
  50  * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as
  51  * the definition of UChar. For details see the documentation for UChar32 itself.
  52  *
  53  * utf.h also defines a small number of C macros for single Unicode code points.
  54  * These are simple checks for surrogates and non-characters.
  55  * For actual Unicode character properties see uchar.h.
  56  *
  57  * By default, string operations must be done with error checking in case
  58  * a string is not well-formed UTF-16.
  59  * The macros will detect if a surrogate code unit is unpaired
  60  * (lead unit without trail unit or vice versa) and just return the unit itself
  61  * as the code point.
  62  * (It is an accidental property of Unicode and UTF-16 that all
  63  * malformed sequences can be expressed unambiguously with a distinct subrange
  64  * of Unicode code points.)
  65  *
  66  * The regular "safe" macros require that the initial, passed-in string index
  67  * is within bounds. They only check the index when they read more than one
  68  * code unit. This is usually done with code similar to the following loop:
  69  * <pre>while(i<length) {
  70  *   U16_NEXT(s, i, length, c);
  71  *   // use c
  72  * }</pre>
  73  *
  74  * When it is safe to assume that text is well-formed UTF-16
  75  * (does not contain single, unpaired surrogates), then one can use
  76  * U16_..._UNSAFE macros.
  77  * These do not check for proper code unit sequences or truncated text and may
  78  * yield wrong results or even cause a crash if they are used with "malformed"
  79  * text.
  80  * In practice, U16_..._UNSAFE macros will produce slightly less code but
  81  * should not be faster because the processing is only different when a
  82  * surrogate code unit is detected, which will be rare.
  83  *
  84  * Similarly for UTF-8, there are "safe" macros without a suffix,
  85  * and U8_..._UNSAFE versions.
  86  * The performance differences are much larger here because UTF-8 provides so
  87  * many opportunities for malformed sequences.
  88  * The unsafe UTF-8 macros are entirely implemented inside the macro definitions
  89  * and are fast, while the safe UTF-8 macros call functions for all but the
  90  * trivial (ASCII) cases.
  91  * (ICU 3.6 optimizes U8_NEXT() and U8_APPEND() to handle most other common
  92  * characters inline as well.)
  93  *
  94  * Unlike with UTF-16, malformed sequences cannot be expressed with distinct
  95  * code point values (0..U+10ffff). They are indicated with negative values instead.
  96  *
  97  * For more information see the ICU User Guide Strings chapter
  98  * (http://icu-project.org/userguide/strings.html).
  99  *
 100  * <em>Usage:</em>
 101  * ICU coding guidelines for if() statements should be followed when using these macros.
 102  * Compound statements (curly braces {}) must be used  for if-else-while...
 103  * bodies and all macro statements should be terminated with semicolon.
 104  *
 105  * @stable ICU 2.4
 106  */
 107
 108 #ifndef __UTF_H__
 109 #define __UTF_H__
 110
 111 #include "unicode/utypes.h"
 112 /* include the utfXX.h after the following definitions */
 113
 114 /* single-code point definitions -------------------------------------------- */
 115
 116 /**
 117  * This value is intended for sentinel values for APIs that
 118  * (take or) return single code points (UChar32).
 119  * It is outside of the Unicode code point range 0..0x10ffff.
 120  *
 121  * For example, a "done" or "error" value in a new API
 122  * could be indicated with U_SENTINEL.
 123  *
 124  * ICU APIs designed before ICU 2.4 usually define service-specific "done"
 125  * values, mostly 0xffff.
 126  * Those may need to be distinguished from
 127  * actual U+ffff text contents by calling functions like
 128  * CharacterIterator::hasNext() or UnicodeString::length().
 129  *
 130  * @return -1
 131  * @see UChar32
 132  * @stable ICU 2.4
 133  */
 134 #define U_SENTINEL (-1)
 135
 136 /**
 137  * Is this code point a Unicode noncharacter?
 138  * @param c 32-bit code point
 139  * @return TRUE or FALSE
 140  * @stable ICU 2.4
 141  */
 142 #define U_IS_UNICODE_NONCHAR(c) \
 143     ((c)>=0xfdd0 && \
 144      ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \
 145      (uint32_t)(c)<=0x10ffff)
 146
 147 /**
 148  * Is c a Unicode code point value (0..U+10ffff)
 149  * that can be assigned a character?
 150  *
 151  * Code points that are not characters include:
 152  * - single surrogate code points (U+d800..U+dfff, 2048 code points)
 153  * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)
 154  * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)
 155  * - the highest Unicode code point value is U+10ffff
 156  *
 157  * This means that all code points below U+d800 are character code points,
 158  * and that boundary is tested first for performance.
 159  *
 160  * @param c 32-bit code point
 161  * @return TRUE or FALSE
 162  * @stable ICU 2.4
 163  */
 164 #define U_IS_UNICODE_CHAR(c) \
 165     ((uint32_t)(c)<0xd800 || \
 166         ((uint32_t)(c)>0xdfff && \
 167          (uint32_t)(c)<=0x10ffff && \
 168          !U_IS_UNICODE_NONCHAR(c)))
 169
 170 /**
 171  * Is this code point a BMP code point (U+0000..U+ffff)?
 172  * @param c 32-bit code point
 173  * @return TRUE or FALSE
 174  * @stable ICU 2.8
 175  */
 176 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff)
 177
 178 /**
 179  * Is this code point a supplementary code point (U+10000..U+10ffff)?
 180  * @param c 32-bit code point
 181  * @return TRUE or FALSE
 182  * @stable ICU 2.8
 183  */
 184 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff)
 185
 186 /**
 187  * Is this code point a lead surrogate (U+d800..U+dbff)?
 188  * @param c 32-bit code point
 189  * @return TRUE or FALSE
 190  * @stable ICU 2.4
 191  */
 192 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
 193
 194 /**
 195  * Is this code point a trail surrogate (U+dc00..U+dfff)?
 196  * @param c 32-bit code point
 197  * @return TRUE or FALSE
 198  * @stable ICU 2.4
 199  */
 200 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
 201
 202 /**
 203  * Is this code point a surrogate (U+d800..U+dfff)?
 204  * @param c 32-bit code point
 205  * @return TRUE or FALSE
 206  * @stable ICU 2.4
 207  */
 208 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800)
 209
 210 /**
 211  * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
 212  * is it a lead surrogate?
 213  * @param c 32-bit code point
 214  * @return TRUE or FALSE
 215  * @stable ICU 2.4
 216  */
 217 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
 218
 219 /**
 220  * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
 221  * is it a trail surrogate?
 222  * @param c 32-bit code point
 223  * @return TRUE or FALSE
 224  * @stable ICU 4.2
 225  */
 226 #define U_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0)
 227
 228 /* include the utfXX.h ------------------------------------------------------ */
 229
 230 #include "unicode/utf8.h"
 231 #include "unicode/utf16.h"
 232
 233 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */
 234 #include "unicode/utf_old.h"
 235
 236 #endif