locale/utf2-fbsd.c

   1 /*-
   2  * Copyright (c) 2002-2004 Tim J. Robbins
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/param.h>
  28 __FBSDID("$FreeBSD: src/lib/libc/locale/utf8.c,v 1.16 2007/10/15 09:51:30 ache Exp $");
  29
  30 #include "xlocale_private.h"
  31
  32 #include <errno.h>
  33 #include <limits.h>
  34 #include <runetype.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <wchar.h>
  38 #include "mblocal.h"
  39
  40 #define UTF2_MB_CUR_MAX         3
  41
  42 static size_t   _UTF2_mbrtowc(wchar_t * __restrict, const char * __restrict,
  43                     size_t, mbstate_t * __restrict, locale_t);
  44 static int      _UTF2_mbsinit(const mbstate_t *, locale_t);
  45 static size_t   _UTF2_mbsnrtowcs(wchar_t * __restrict,
  46                     const char ** __restrict, size_t, size_t,
  47                     mbstate_t * __restrict, locale_t);
  48 static size_t   _UTF2_wcrtomb(char * __restrict, wchar_t,
  49                     mbstate_t * __restrict, locale_t);
  50 static size_t   _UTF2_wcsnrtombs(char * __restrict, const wchar_t ** __restrict,
  51                     size_t, size_t, mbstate_t * __restrict, locale_t);
  52
  53 typedef struct {
  54         wchar_t ch;
  55         int     want;
  56         wchar_t lbound;
  57 } _UTF2State;
  58
  59 __private_extern__ int
  60 _UTF2_init(struct __xlocale_st_runelocale *xrl)
  61 {
  62
  63         xrl->__mbrtowc = _UTF2_mbrtowc;
  64         xrl->__wcrtomb = _UTF2_wcrtomb;
  65         xrl->__mbsinit = _UTF2_mbsinit;
  66         xrl->__mbsnrtowcs = _UTF2_mbsnrtowcs;
  67         xrl->__wcsnrtombs = _UTF2_wcsnrtombs;
  68         xrl->__mb_cur_max = UTF2_MB_CUR_MAX;
  69         /*
  70          * UCS-4 encoding used as the internal representation, so
  71          * slots 0x0080-0x00FF are occuped and must be excluded
  72          * from the single byte ctype by setting the limit.
  73          */
  74         xrl->__mb_sb_limit = 128;
  75
  76         return (0);
  77 }
  78
  79 static int
  80 _UTF2_mbsinit(const mbstate_t *ps, locale_t loc)
  81 {
  82
  83         return (ps == NULL || ((const _UTF2State *)ps)->want == 0);
  84 }
  85
  86 static size_t
  87 _UTF2_mbrtowc(wchar_t * __restrict pwc, const char * __restrict s, size_t n,
  88     mbstate_t * __restrict ps, locale_t loc)
  89 {
  90         _UTF2State *us;
  91         int ch, i, mask, want;
  92         wchar_t lbound, wch;
  93
  94         us = (_UTF2State *)ps;
  95
  96         if (us->want < 0 || us->want > 6) {
  97                 errno = EINVAL;
  98                 return ((size_t)-1);
  99         }
 100
 101         if (s == NULL) {
 102                 s = "";
 103                 n = 1;
 104                 pwc = NULL;
 105         }
 106
 107         if (n == 0)
 108                 /* Incomplete multibyte sequence */
 109                 return ((size_t)-2);
 110
 111         if (us->want == 0 && ((ch = (unsigned char)*s) & ~0x7f) == 0) {
 112                 /* Fast path for plain ASCII characters. */
 113                 if (pwc != NULL)
 114                         *pwc = ch;
 115                 return (ch != '\0' ? 1 : 0);
 116         }
 117
 118         if (us->want == 0) {
 119                 /*
 120                  * Determine the number of octets that make up this character
 121                  * from the first octet, and a mask that extracts the
 122                  * interesting bits of the first octet. We already know
 123                  * the character is at least two bytes long.
 124                  *
 125                  * We also specify a lower bound for the character code to
 126                  * detect redundant, non-"shortest form" encodings. For
 127                  * example, the sequence C0 80 is _not_ a legal representation
 128                  * of the null character. This enforces a 1-to-1 mapping
 129                  * between character codes and their multibyte representations.
 130                  */
 131                 ch = (unsigned char)*s;
 132                 if ((ch & 0x80) == 0) {
 133                         mask = 0x7f;
 134                         want = 1;
 135                         lbound = 0;
 136                 } else if ((ch & 0xe0) == 0xc0) {
 137                         mask = 0x1f;
 138                         want = 2;
 139                         lbound = 0x80;
 140                 } else if ((ch & 0xf0) == 0xe0) {
 141                         mask = 0x0f;
 142                         want = 3;
 143                         lbound = 0x800;
 144                 } else {
 145                         /*
 146                          * Malformed input; input is not UTF2.
 147                          */
 148                         errno = EILSEQ;
 149                         return ((size_t)-1);
 150                 }
 151         } else {
 152                 want = us->want;
 153                 lbound = us->lbound;
 154         }
 155
 156         /*
 157          * Decode the octet sequence representing the character in chunks
 158          * of 6 bits, most significant first.
 159          */
 160         if (us->want == 0)
 161                 wch = (unsigned char)*s++ & mask;
 162         else
 163                 wch = us->ch;
 164         for (i = (us->want == 0) ? 1 : 0; i < MIN(want, n); i++) {
 165                 if ((*s & 0xc0) != 0x80) {
 166                         /*
 167                          * Malformed input; bad characters in the middle
 168                          * of a character.
 169                          */
 170                         errno = EILSEQ;
 171                         return ((size_t)-1);
 172                 }
 173                 wch <<= 6;
 174                 wch |= *s++ & 0x3f;
 175         }
 176         if (i < want) {
 177                 /* Incomplete multibyte sequence. */
 178                 us->want = want - i;
 179                 us->lbound = lbound;
 180                 us->ch = wch;
 181                 return ((size_t)-2);
 182         }
 183         if (wch < lbound) {
 184                 /*
 185                  * Malformed input; redundant encoding.
 186                  */
 187                 errno = EILSEQ;
 188                 return ((size_t)-1);
 189         }
 190         if (pwc != NULL)
 191                 *pwc = wch;
 192         us->want = 0;
 193         return (wch == L'\0' ? 0 : want);
 194 }
 195
 196 static size_t
 197 _UTF2_mbsnrtowcs(wchar_t * __restrict dst, const char ** __restrict src,
 198     size_t nms, size_t len, mbstate_t * __restrict ps, locale_t loc)
 199 {
 200         _UTF2State *us;
 201         const char *s;
 202         size_t nchr;
 203         wchar_t wc;
 204         size_t nb;
 205
 206         us = (_UTF2State *)ps;
 207
 208         s = *src;
 209         nchr = 0;
 210
 211         if (dst == NULL) {
 212                 /*
 213                  * The fast path in the loop below is not safe if an ASCII
 214                  * character appears as anything but the first byte of a
 215                  * multibyte sequence. Check now to avoid doing it in the loop.
 216                  */
 217                 if (nms > 0 && us->want > 0 && (signed char)*s > 0) {
 218                         errno = EILSEQ;
 219                         return ((size_t)-1);
 220                 }
 221                 for (;;) {
 222                         if (nms > 0 && (signed char)*s > 0)
 223                                 /*
 224                                  * Fast path for plain ASCII characters
 225                                  * excluding NUL.
 226                                  */
 227                                 nb = 1;
 228                         else if ((nb = _UTF2_mbrtowc(&wc, s, nms, ps, loc)) ==
 229                             (size_t)-1)
 230                                 /* Invalid sequence - mbrtowc() sets errno. */
 231                                 return ((size_t)-1);
 232                         else if (nb == 0 || nb == (size_t)-2)
 233                                 return (nchr);
 234                         s += nb;
 235                         nms -= nb;
 236                         nchr++;
 237                 }
 238                 /*NOTREACHED*/
 239         }
 240
 241         /*
 242          * The fast path in the loop below is not safe if an ASCII
 243          * character appears as anything but the first byte of a
 244          * multibyte sequence. Check now to avoid doing it in the loop.
 245          */
 246         if (nms > 0 && len > 0 && us->want > 0 && (signed char)*s > 0) {
 247                 errno = EILSEQ;
 248                 return ((size_t)-1);
 249         }
 250         while (len-- > 0) {
 251                 if (nms > 0 && (signed char)*s > 0) {
 252                         /*
 253                          * Fast path for plain ASCII characters
 254                          * excluding NUL.
 255                          */
 256                         *dst = (wchar_t)*s;
 257                         nb = 1;
 258                 } else if ((nb = _UTF2_mbrtowc(dst, s, nms, ps, loc)) ==
 259                     (size_t)-1) {
 260                         *src = s;
 261                         return ((size_t)-1);
 262                 } else if (nb == (size_t)-2) {
 263                         *src = s + nms;
 264                         return (nchr);
 265                 } else if (nb == 0) {
 266                         *src = NULL;
 267                         return (nchr);
 268                 }
 269                 s += nb;
 270                 nms -= nb;
 271                 nchr++;
 272                 dst++;
 273         }
 274         *src = s;
 275         return (nchr);
 276 }
 277
 278 static size_t
 279 _UTF2_wcrtomb(char * __restrict s, wchar_t wc, mbstate_t * __restrict ps, locale_t loc)
 280 {
 281         _UTF2State *us;
 282         unsigned char lead;
 283         int i, len;
 284
 285         us = (_UTF2State *)ps;
 286
 287         if (us->want != 0) {
 288                 errno = EINVAL;
 289                 return ((size_t)-1);
 290         }
 291
 292         if (s == NULL)
 293                 /* Reset to initial shift state (no-op) */
 294                 return (1);
 295
 296         if ((wc & ~0x7f) == 0) {
 297                 /* Fast path for plain ASCII characters. */
 298                 *s = (char)wc;
 299                 return (1);
 300         }
 301
 302         /*
 303          * Determine the number of octets needed to represent this character.
 304          * We always output the shortest sequence possible. Also specify the
 305          * first few bits of the first octet, which contains the information
 306          * about the sequence length.
 307          */
 308         if ((wc & ~0x7f) == 0) {
 309                 lead = 0;
 310                 len = 1;
 311         } else if ((wc & ~0x7ff) == 0) {
 312                 lead = 0xc0;
 313                 len = 2;
 314         } else if ((wc & ~0xffff) == 0) {
 315                 lead = 0xe0;
 316                 len = 3;
 317         } else {
 318                 errno = EILSEQ;
 319                 return ((size_t)-1);
 320         }
 321
 322         /*
 323          * Output the octets representing the character in chunks
 324          * of 6 bits, least significant last. The first octet is
 325          * a special case because it contains the sequence length
 326          * information.
 327          */
 328         for (i = len - 1; i > 0; i--) {
 329                 s[i] = (wc & 0x3f) | 0x80;
 330                 wc >>= 6;
 331         }
 332         *s = (wc & 0xff) | lead;
 333
 334         return (len);
 335 }
 336
 337 static size_t
 338 _UTF2_wcsnrtombs(char * __restrict dst, const wchar_t ** __restrict src,
 339     size_t nwc, size_t len, mbstate_t * __restrict ps, locale_t loc)
 340 {
 341         _UTF2State *us;
 342         char buf[MB_LEN_MAX];
 343         const wchar_t *s;
 344         size_t nbytes;
 345         size_t nb;
 346
 347         us = (_UTF2State *)ps;
 348
 349         if (us->want != 0) {
 350                 errno = EINVAL;
 351                 return ((size_t)-1);
 352         }
 353
 354         s = *src;
 355         nbytes = 0;
 356
 357         if (dst == NULL) {
 358                 while (nwc-- > 0) {
 359                         if (0 <= *s && *s < 0x80)
 360                                 /* Fast path for plain ASCII characters. */
 361                                 nb = 1;
 362                         else if ((nb = _UTF2_wcrtomb(buf, *s, ps, loc)) ==
 363                             (size_t)-1)
 364                                 /* Invalid character - wcrtomb() sets errno. */
 365                                 return ((size_t)-1);
 366                         if (*s == L'\0')
 367                                 return (nbytes + nb - 1);
 368                         s++;
 369                         nbytes += nb;
 370                 }
 371                 return (nbytes);
 372         }
 373
 374         while (len > 0 && nwc-- > 0) {
 375                 if (0 <= *s && *s < 0x80) {
 376                         /* Fast path for plain ASCII characters. */
 377                         nb = 1;
 378                         *dst = *s;
 379                 } else if (len > (size_t)UTF2_MB_CUR_MAX) {
 380                         /* Enough space to translate in-place. */
 381                         if ((nb = _UTF2_wcrtomb(dst, *s, ps, loc)) == (size_t)-1) {
 382                                 *src = s;
 383                                 return ((size_t)-1);
 384                         }
 385                 } else {
 386                         /*
 387                          * May not be enough space; use temp. buffer.
 388                          */
 389                         if ((nb = _UTF2_wcrtomb(buf, *s, ps, loc)) == (size_t)-1) {
 390                                 *src = s;
 391                                 return ((size_t)-1);
 392                         }
 393                         if (nb > (int)len)
 394                                 /* MB sequence for character won't fit. */
 395                                 break;
 396                         memcpy(dst, buf, nb);
 397                 }
 398                 if (*s == L'\0') {
 399                         *src = NULL;
 400                         return (nbytes + nb - 1);
 401                 }
 402                 s++;
 403                 dst += nb;
 404                 len -= nb;
 405                 nbytes += nb;
 406         }
 407         *src = s;
 408         return (nbytes);
 409 }