icuSources/common/dictionarydata.h

   1 /*
   2 *******************************************************************************
   3 * Copyright (C) 2012, International Business Machines
   4 * Corporation and others.  All Rights Reserved.
   5 *******************************************************************************
   6 * dictionarydata.h
   7 *
   8 * created on: 2012may31
   9 * created by: Markus W. Scherer & Maxime Serrano
  10 */
  11
  12 #ifndef __DICTIONARYDATA_H__
  13 #define __DICTIONARYDATA_H__
  14
  15 #include "unicode/utypes.h"
  16
  17 #if !UCONFIG_NO_BREAK_ITERATION
  18
  19 #include "unicode/utext.h"
  20 #include "unicode/udata.h"
  21 #include "udataswp.h"
  22 #include "unicode/uobject.h"
  23 #include "unicode/ustringtrie.h"
  24
  25 U_NAMESPACE_BEGIN
  26
  27 class UCharsTrie;
  28 class BytesTrie;
  29
  30 class U_COMMON_API DictionaryData : public UMemory {
  31 public:
  32     static const int32_t TRIE_TYPE_BYTES = 0;
  33     static const int32_t TRIE_TYPE_UCHARS = 1;
  34     static const int32_t TRIE_TYPE_MASK = 7;
  35     static const int32_t TRIE_HAS_VALUES = 8;
  36
  37     static const int32_t TRANSFORM_NONE = 0;
  38     static const int32_t TRANSFORM_TYPE_OFFSET = 0x1000000;
  39     static const int32_t TRANSFORM_TYPE_MASK = 0x7f000000;
  40     static const int32_t TRANSFORM_OFFSET_MASK = 0x1fffff;
  41
  42     enum {
  43         // Byte offsets from the start of the data, after the generic header.
  44         IX_STRING_TRIE_OFFSET,
  45         IX_RESERVED1_OFFSET,
  46         IX_RESERVED2_OFFSET,
  47         IX_TOTAL_SIZE,
  48
  49         // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc.
  50         IX_TRIE_TYPE,
  51         // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc.
  52         IX_TRANSFORM,
  53
  54         IX_RESERVED6,
  55         IX_RESERVED7,
  56         IX_COUNT
  57     };
  58 };
  59
  60 /**
  61  * Wrapper class around generic dictionaries, implementing matches().
  62  * getType() should return a TRIE_TYPE_??? constant from DictionaryData.
  63  *
  64  * All implementations of this interface must be thread-safe if they are to be used inside of the
  65  * dictionary-based break iteration code.
  66  */
  67 class U_COMMON_API DictionaryMatcher : public UMemory {
  68 public:
  69     virtual ~DictionaryMatcher();
  70     // this should emulate CompactTrieDictionary::matches()
  71     virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
  72                             int32_t limit, int32_t *values = NULL) const = 0;
  73     /** @return DictionaryData::TRIE_TYPE_XYZ */
  74     virtual int32_t getType() const = 0;
  75 };
  76
  77 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary
  78 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher {
  79 public:
  80     // constructs a new UCharsDictionaryMatcher.
  81     // The UDataMemory * will be closed on this object's destruction.
  82     UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { }
  83     virtual ~UCharsDictionaryMatcher();
  84     virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
  85                             int32_t limit, int32_t *values = NULL) const;
  86     virtual int32_t getType() const;
  87 private:
  88     const UChar *characters;
  89     UDataMemory *file;
  90 };
  91
  92 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary
  93 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher {
  94 public:
  95     // constructs a new BytesTrieDictionaryMatcher
  96     // the transform constant should be the constant read from the file, not a masked version!
  97     // the UDataMemory * fed in here will be closed on this object's destruction
  98     BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f)
  99             : characters(c), transformConstant(t), file(f) { }
 100     virtual ~BytesDictionaryMatcher();
 101     virtual int32_t matches(UText *text, int32_t maxLength, int32_t *lengths, int32_t &count,
 102                             int32_t limit, int32_t *values = NULL) const;
 103     virtual int32_t getType() const;
 104 private:
 105     UChar32 transform(UChar32 c) const;
 106
 107     const char *characters;
 108     int32_t transformConstant;
 109     UDataMemory *file;
 110 };
 111
 112 U_NAMESPACE_END
 113
 114 U_CAPI int32_t U_EXPORT2
 115 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode);
 116
 117 /**
 118  * Format of dictionary .dict data files.
 119  * Format version 1.0.
 120  *
 121  * A dictionary .dict data file contains a byte-serialized BytesTrie or
 122  * a UChars-serialized UCharsTrie.
 123  * Such files are used in dictionary-based break iteration (DBBI).
 124  *
 125  * For a BytesTrie, a transformation type is specified for
 126  * transforming Unicode strings into byte sequences.
 127  *
 128  * A .dict file begins with a standard ICU data file header
 129  * (DataHeader, see ucmndata.h and unicode/udata.h).
 130  * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0).
 131  *
 132  * After the header, the file contains the following parts.
 133  * Constants are defined in the DictionaryData class.
 134  *
 135  * For the data structure of BytesTrie & UCharsTrie see
 136  * http://site.icu-project.org/design/struct/tries
 137  * and the bytestrie.h and ucharstrie.h header files.
 138  *
 139  * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4;
 140  *
 141  *      The first four indexes are byte offsets in ascending order.
 142  *      Each byte offset marks the start of the next part in the data file,
 143  *      and the end of the previous one.
 144  *      When two consecutive byte offsets are the same, then the corresponding part is empty.
 145  *      Byte offsets are offsets from after the header,
 146  *      that is, from the beginning of the indexes[].
 147  *      Each part starts at an offset with proper alignment for its data.
 148  *      If necessary, the previous part may include padding bytes to achieve this alignment.
 149  *
 150  *      trieType=indexes[IX_TRIE_TYPE] defines the trie type.
 151  *      transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation.
 152  *          If the transformation type is TRANSFORM_TYPE_OFFSET,
 153  *          then the lower 21 bits contain the offset code point.
 154  *          Each code point c is mapped to byte b = (c - offset).
 155  *          Code points outside the range offset..(offset+0xff) cannot be mapped
 156  *          and do not occur in the dictionary.
 157  *
 158  * stringTrie; -- a serialized BytesTrie or UCharsTrie
 159  *
 160  *      The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType),
 161  *      or it maps all strings to 0 (TRIE_HAS_VALUES bit not set).
 162  */
 163
 164 #endif  /* !UCONFIG_NO_BREAK_ITERATION */
 165 #endif  /* __DICTIONARYDATA_H__ */