]> git.saurik.com Git - apple/icu.git/blame - icuSources/i18n/uspoof_impl.h
ICU-59173.0.1.tar.gz
[apple/icu.git] / icuSources / i18n / uspoof_impl.h
CommitLineData
f3c0d7a5
A
1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
729e4ab9
A
3/*
4***************************************************************************
51004dcb 5* Copyright (C) 2008-2013, International Business Machines Corporation
729e4ab9
A
6* and others. All Rights Reserved.
7***************************************************************************
8*
9* uspoof_impl.h
10*
11* Implemenation header for spoof detection
12*
13*/
14
15#ifndef USPOOFIM_H
16#define USPOOFIM_H
17
f3c0d7a5 18#include "uassert.h"
729e4ab9
A
19#include "unicode/utypes.h"
20#include "unicode/uspoof.h"
729e4ab9
A
21#include "unicode/uscript.h"
22#include "unicode/udata.h"
23
51004dcb 24#include "utrie2.h"
729e4ab9
A
25
26#if !UCONFIG_NO_NORMALIZATION
27
4388f060 28#ifdef __cplusplus
729e4ab9
A
29
30U_NAMESPACE_BEGIN
31
32// The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
33// a single input code point. This is function of the unicode.org data.
34#define USPOOF_MAX_SKELETON_EXPANSION 20
35
36// The default stack buffer size for copies or conversions or normalizations
37// of input strings being checked. (Used in multiple places.)
38#define USPOOF_STACK_BUFFER_SIZE 100
39
40// Magic number for sanity checking spoof data.
41#define USPOOF_MAGIC 0x3845fdef
42
f3c0d7a5
A
43// Magic number for sanity checking spoof checkers.
44#define USPOOF_CHECK_MAGIC 0x2734ecde
45
51004dcb 46class ScriptSet;
729e4ab9
A
47class SpoofData;
48struct SpoofDataHeader;
f3c0d7a5 49class ConfusableDataUtils;
729e4ab9
A
50
51/**
52 * Class SpoofImpl corresponds directly to the plain C API opaque type
53 * USpoofChecker. One can be cast to the other.
54 */
55class SpoofImpl : public UObject {
56public:
f3c0d7a5
A
57 SpoofImpl(SpoofData *data, UErrorCode& status);
58 SpoofImpl(UErrorCode& status);
59 SpoofImpl();
60 void construct(UErrorCode& status);
61 virtual ~SpoofImpl();
729e4ab9
A
62
63 /** Copy constructor, used by the user level uspoof_clone() function.
64 */
65 SpoofImpl(const SpoofImpl &src, UErrorCode &status);
66
f3c0d7a5 67 USpoofChecker *asUSpoofChecker();
729e4ab9
A
68 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
69 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
70
729e4ab9
A
71 /** Set and Get AllowedLocales, implementations of the corresponding API */
72 void setAllowedLocales(const char *localesList, UErrorCode &status);
73 const char * getAllowedLocales(UErrorCode &status);
74
75 // Add (union) to the UnicodeSet all of the characters for the scripts used for
76 // the specified locale. Part of the implementation of setAllowedLocales.
77 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
78
f3c0d7a5
A
79 // Functions implementing the features of UTS 39 section 5.
80 static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status);
81 void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const;
82 void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const;
83 void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const;
84 URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const;
729e4ab9
A
85
86 /** parse a hex number. Untility used by the builders. */
87 static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);
88
729e4ab9
A
89 static UClassID U_EXPORT2 getStaticClassID(void);
90 virtual UClassID getDynamicClassID(void) const;
91
92 //
93 // Data Members
94 //
95
96 int32_t fMagic; // Internal sanity check.
97 int32_t fChecks; // Bit vector of checks to perform.
98
99 SpoofData *fSpoofData;
100
729e4ab9 101 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters.
f3c0d7a5 102 // for this Spoof Checker. Defaults to all chars.
729e4ab9
A
103
104 const char *fAllowedLocales; // The list of allowed locales.
51004dcb 105 URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier.
729e4ab9
A
106};
107
f3c0d7a5
A
108/**
109 * Class CheckResult corresponds directly to the plain C API opaque type
110 * USpoofCheckResult. One can be cast to the other.
111 */
112class CheckResult : public UObject {
113public:
114 CheckResult();
115 virtual ~CheckResult();
116
117 USpoofCheckResult *asUSpoofCheckResult();
118 static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status);
119 static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status);
120
121 void clear();
122
123 // Used to convert this CheckResult to the older int32_t return value API
124 int32_t toCombinedBitmask(int32_t expectedChecks);
125
126 // Data Members
127 int32_t fMagic; // Internal sanity check.
128 int32_t fChecks; // Bit vector of checks that were failed.
129 UnicodeSet fNumerics; // Set of numerics found in the string.
130 URestrictionLevel fRestrictionLevel; // The restriction level of the string.
131};
729e4ab9
A
132
133
134//
f3c0d7a5 135// Confusable Mappings Data Structures, version 2.0
729e4ab9
A
136//
137// For the confusable data, we are essentially implementing a map,
138// key: a code point
139// value: a string. Most commonly one char in length, but can be more.
140//
141// The keys are stored as a sorted array of 32 bit ints.
142// bits 0-23 a code point value
f3c0d7a5 143// bits 24-31 length of value string, in UChars (between 1 and 256 UChars).
729e4ab9
A
144// The key table is sorted in ascending code point order. (not on the
145// 32 bit int value, the flag bits do not participate in the sorting.)
146//
147// Lookup is done by means of a binary search in the key table.
148//
149// The corresponding values are kept in a parallel array of 16 bit ints.
150// If the value string is of length 1, it is literally in the value array.
151// For longer strings, the value array contains an index into the strings table.
152//
153// String Table:
154// The strings table contains all of the value strings (those of length two or greater)
155// concatentated together into one long UChar (UTF-16) array.
156//
729e4ab9
A
157// There is no nul character or other mark between adjacent strings.
158//
f3c0d7a5 159//----------------------------------------------------------------------------
729e4ab9 160//
f3c0d7a5
A
161// Changes from format version 1 to format version 2:
162// 1) Removal of the whole-script confusable data tables.
163// 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask.
164// 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits.
165// 4) Removal of the string lengths table since 8 bits is sufficient for the
166// lengths of all entries in confusables.txt.
729e4ab9 167
729e4ab9
A
168
169
f3c0d7a5
A
170// Internal functions for manipulating confusable data table keys
171#define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58
172class ConfusableDataUtils {
173public:
174 inline static UChar32 keyToCodePoint(int32_t key) {
175 return key & 0x00ffffff;
176 }
177 inline static int32_t keyToLength(int32_t key) {
178 return ((key & 0xff000000) >> 24) + 1;
179 }
180 inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) {
181 U_ASSERT((codePoint & 0x00ffffff) == codePoint);
182 U_ASSERT(length <= 256);
183 return codePoint | ((length - 1) << 24);
184 }
729e4ab9
A
185};
186
187
729e4ab9
A
188//-------------------------------------------------------------------------------------
189//
190// SpoofData
191//
192// A small class that wraps the raw (usually memory mapped) spoof data.
193// Serves two primary functions:
194// 1. Convenience. Contains real pointers to the data, to avoid dealing with
195// the offsets in the raw data.
196// 2. Reference counting. When a spoof checker is cloned, the raw data is shared
197// and must be retained until all checkers using the data are closed.
198// Nothing in this struct includes state that is specific to any particular
199// USpoofDetector object.
200//
201//---------------------------------------------------------------------------------------
202class SpoofData: public UMemory {
203 public:
f3c0d7a5
A
204 static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data.
205 static void releaseDefault(); // Cleanup reference to default spoof data.
206
729e4ab9
A
207 SpoofData(UErrorCode &status); // Create new spoof data wrapper.
208 // Only used when building new data from rules.
209
210 // Constructor for use when creating from prebuilt default data.
211 // A UDataMemory is what the ICU internal data loading functions provide.
212 // The udm is adopted by the SpoofData.
213 SpoofData(UDataMemory *udm, UErrorCode &status);
214
215 // Constructor for use when creating from serialized data.
216 //
217 SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
218
219 // Check raw Spoof Data Version compatibility.
220 // Return TRUE it looks good.
f3c0d7a5
A
221 UBool validateDataVersion(UErrorCode &status) const;
222
729e4ab9
A
223 ~SpoofData(); // Destructor not normally used.
224 // Use removeReference() instead.
225 // Reference Counting functions.
226 // Clone of a user-level spoof detector increments the ref count on the data.
227 // Close of a user-level spoof detector decrements the ref count.
228 // If the data is owned by us, it will be deleted when count goes to zero.
229 SpoofData *addReference();
230 void removeReference();
231
f3c0d7a5
A
232 // Reset all fields to an initial state.
233 // Called from the top of all constructors.
234 void reset();
235
236 // Copy this instance's raw data buffer to the specified address.
237 int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const;
238
239 // Get the total number of bytes of data backed by this SpoofData.
240 // Not to be confused with length, which returns the number of confusable entries.
241 int32_t size() const;
242
243 // Get the confusable skeleton transform for a single code point.
244 // The result is a string with a length between 1 and 18 as of Unicode 9.
245 // This is the main public endpoint for this class.
246 // @return The length in UTF-16 code units of the substition string.
247 int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const;
248
249 // Get the number of confusable entries in this SpoofData.
250 int32_t length() const;
251
252 // Get the code point (key) at the specified index.
253 UChar32 codePointAt(int32_t index) const;
254
255 // Get the confusable skeleton (value) at the specified index.
256 // Append it to the specified UnicodeString&.
257 // @return The length in UTF-16 code units of the skeleton string.
258 int32_t appendValueTo(int32_t index, UnicodeString& dest) const;
259
260 private:
729e4ab9
A
261 // Reserve space in the raw data. For use by builder when putting together a
262 // new set of data. Init the new storage to zero, to prevent inconsistent
263 // results if it is not all otherwise set by the requester.
264 // Return:
265 // pointer to the new space that was added by this function.
266 void *reserveSpace(int32_t numBytes, UErrorCode &status);
267
268 // initialize the pointers from this object to the raw data.
269 void initPtrs(UErrorCode &status);
270
729e4ab9
A
271 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data
272 UBool fDataOwned; // True if the raw data is owned, and needs
273 // to be deleted when refcount goes to zero.
274 UDataMemory *fUDM; // If not NULL, our data came from a
275 // UDataMemory, which we must close when
4388f060 276 // we are done.
729e4ab9
A
277
278 uint32_t fMemLimit; // Limit of available raw data space
57a6839d 279 u_atomic_int32_t fRefCount;
729e4ab9
A
280
281 // Confusable data
282 int32_t *fCFUKeys;
283 uint16_t *fCFUValues;
729e4ab9
A
284 UChar *fCFUStrings;
285
f3c0d7a5
A
286 friend class ConfusabledataBuilder;
287};
729e4ab9
A
288
289//---------------------------------------------------------------------------------------
290//
291// Raw Binary Data Formats, as loaded from the ICU data file,
292// or as built by the builder.
293//
294//---------------------------------------------------------------------------------------
295struct SpoofDataHeader {
296 int32_t fMagic; // (0x3845fdef)
297 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo
298 // if there is one associated with this data.
299 int32_t fLength; // Total lenght in bytes of this spoof data,
300 // including all sections, not just the header.
301
302 // The following four sections refer to data representing the confusable data
303 // from the Unicode.org data from "confusables.txt"
304
305 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *)
306 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each)
307
308 // TODO: change name to fCFUValues, for consistency.
309 int32_t fCFUStringIndex; // byte offset to String Indexes table
310 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each)
311 // (number of entries must be same as in Keys table
312
313 int32_t fCFUStringTable; // byte offset of String table
314 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars)
315
729e4ab9 316 // The following sections are for data from xidmodifications.txt
729e4ab9 317
f3c0d7a5
A
318 int32_t unused[15]; // Padding, Room for Expansion
319};
729e4ab9 320
729e4ab9
A
321
322
323U_NAMESPACE_END
4388f060 324#endif /* __cplusplus */
729e4ab9
A
325
326/**
327 * Endianness swap function for binary spoof data.
328 * @internal
329 */
330U_CAPI int32_t U_EXPORT2
331uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
332 UErrorCode *status);
333
334
335#endif
336
337#endif /* USPOOFIM_H */
338