ICU-461.13.tar.gz

[apple/icu.git] / icuSources / i18n / uspoof_impl.h

/*
***************************************************************************
* Copyright (C) 2008-2011, International Business Machines Corporation
* and others. All Rights Reserved.
***************************************************************************
*
*  uspoof_impl.h
*
*    Implemenation header for spoof detection
*
*/

#ifndef USPOOFIM_H
#define USPOOFIM_H

#include "unicode/utypes.h"
#include "unicode/uspoof.h"
#include "utrie2.h"
#include "unicode/uscript.h"
#include "unicode/udata.h"


#if !UCONFIG_NO_NORMALIZATION

#ifdef XP_CPLUSPLUS

U_NAMESPACE_BEGIN

// The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
//   a single input code point.  This is function of the unicode.org data.
#define USPOOF_MAX_SKELETON_EXPANSION 20

// The default stack buffer size for copies or conversions or normalizations
// of input strings being checked.  (Used in multiple places.)
#define USPOOF_STACK_BUFFER_SIZE 100

// Magic number for sanity checking spoof data.
#define USPOOF_MAGIC 0x3845fdef

class SpoofData;
struct SpoofDataHeader;
struct SpoofStringLengthsElement;
class ScriptSet;

/**
  *  Class SpoofImpl corresponds directly to the plain C API opaque type
  *  USpoofChecker.  One can be cast to the other.
  */
class SpoofImpl : public UObject  {
public:
        SpoofImpl(SpoofData *data, UErrorCode &status);
        SpoofImpl();
        virtual ~SpoofImpl();

    /** Copy constructor, used by the user level uspoof_clone() function.
     */
    SpoofImpl(const SpoofImpl &src, UErrorCode &status);
    
    static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
    static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);

    /** Get the confusable skeleton transform for a single code point.
     *  The result is a string with a length between 1 and 18.
     *  @param    tableMask  bit flag specifying which confusable table to use.
     *                       One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc.
     *  @return   The length in UTF-16 code units of the substition string.
     */  
    int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UChar *destBuf) const;

    /** Set and Get AllowedLocales, implementations of the corresponding API */
    void setAllowedLocales(const char *localesList, UErrorCode &status);
    const char * getAllowedLocales(UErrorCode &status);

    // Add (union) to the UnicodeSet all of the characters for the scripts used for
    // the specified locale.  Part of the implementation of setAllowedLocales.
    void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);


    /** parse a hex number.  Untility used by the builders.   */
    static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);

    // Implementation for Whole Script tests.
    // Return the test bit flag to be ORed into the eventual user return value
    //    if a Spoof opportunity is detected.
    void wholeScriptCheck(
        const UChar *text, int32_t length, ScriptSet *result, UErrorCode &status) const;
            
    /** Scan a string to determine how many scripts it includes.
     * Ignore characters with script=Common and scirpt=Inherited.
     * @param    text     The UChar text to be scanned
     * @param    length   The length of the input text, -1 for nul termintated.
     * @param    pos      An out parameter, set to the first input postion at which
     *                    a second script was encountered, ignoring Common and Inherited.
     * @param    status   For errors.
     * @return            the number of (non-common,inherited) scripts encountered,
     *                    clipped to a max of two.
     */
    int32_t scriptScan(const UChar *text, int32_t length, int32_t &pos, UErrorCode &status) const;


    // WholeScript and MixedScript check implementation.
    //
    ScriptSet *WholeScriptCheck(const UChar *text, int32_t length, UErrorCode &status) const;
    
    static UClassID U_EXPORT2 getStaticClassID(void);
    virtual UClassID getDynamicClassID(void) const;

    //
    // Data Members
    //

    int32_t           fMagic;             // Internal sanity check.
    int32_t           fChecks;            // Bit vector of checks to perform.

    SpoofData        *fSpoofData;
    
    int32_t           fCheckMask;         // Spoof table selector.  f(Check Type)
        
    const UnicodeSet *fAllowedCharsSet;   // The UnicodeSet of allowed characters.
                                          //   for this Spoof Checker.  Defaults to all chars. 

    const char       *fAllowedLocales;    // The list of allowed locales.
};



//
//  Confusable Mappings Data Structures
//
//    For the confusable data, we are essentially implementing a map,
//       key:    a code point
//       value:  a string.  Most commonly one char in length, but can be more.
//
//    The keys are stored as a sorted array of 32 bit ints.
//             bits 0-23    a code point value
//             bits 24-31   flags
//                24:  1 if entry applies to SL table
//                25:  1 if entry applies to SA table
//                26:  1 if entry applies to ML table
//                27:  1 if entry applies to MA table
//                28:  1 if there are multiple entries for this code point.
//                29-30:  length of value string, in UChars.
//                         values are (1, 2, 3, other)
//        The key table is sorted in ascending code point order.  (not on the
//        32 bit int value, the flag bits do not participate in the sorting.)
//
//        Lookup is done by means of a binary search in the key table.
//
//    The corresponding values are kept in a parallel array of 16 bit ints.
//        If the value string is of length 1, it is literally in the value array.
//        For longer strings, the value array contains an index into the strings table.
//
//    String Table:
//       The strings table contains all of the value strings (those of length two or greater)
//       concatentated together into one long UChar (UTF-16) array.
//
//       The array is arranged by length of the strings - all strings of the same length
//       are stored together.  The sections are ordered by length of the strings -
//       all two char strings first, followed by all of the three Char strings, etc.
//
//       There is no nul character or other mark between adjacent strings.
//
//    String Lengths table
//       The length of strings from 1 to 3 is flagged in the key table.
//       For strings of length 4 or longer, the string length table provides a
//       mapping between an index into the string table and the corresponding length.
//       Strings of these lengths are rare, so lookup time is not an issue.
//       Each entry consists of
//            uint16_t      index of the _last_ string with this length
//            uint16_t      the length
//

// Flag bits in the Key entries
#define USPOOF_SL_TABLE_FLAG (1<<24)
#define USPOOF_SA_TABLE_FLAG (1<<25)
#define USPOOF_ML_TABLE_FLAG (1<<26)
#define USPOOF_MA_TABLE_FLAG (1<<27)
#define USPOOF_KEY_MULTIPLE_VALUES (1<<28)
#define USPOOF_KEY_LENGTH_SHIFT 29
#define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3)


struct SpoofStringLengthsElement {
    uint16_t      fLastString;         // index in string table of last string with this length
    uint16_t      fStrLength;           // Length of strings
};


//-------------------------------------------------------------------------------
//
//  ScriptSet - Wrapper class for the Script code bit sets that are part of the
//              whole script confusable data.
//
//              This class is used both at data build and at run time.
//              The constructor is only used at build time.
//              At run time, just point at the prebuilt data and go.
//  
//-------------------------------------------------------------------------------
class ScriptSet: public UMemory {
  public:
    ScriptSet();
    ~ScriptSet();

    UBool operator == (const ScriptSet &other);
    ScriptSet & operator = (const ScriptSet &other);

    void Union(const ScriptSet &other);
    void Union(UScriptCode script);
    void intersect(const ScriptSet &other);
    void intersect(UScriptCode script);
    void setAll();
    void resetAll();
    int32_t countMembers();

  private:
    uint32_t  bits[6];
};




//-------------------------------------------------------------------------------
//
//  NFDBuffer   A little class to handle the NFD normalization that is
//               needed on incoming identifiers to be checked.
//               Takes care of buffer handling and normalization
//
//               Instances of this class are intended to be stack-allocated.
//
//               TODO:  how to map position offsets back to user values?
//
//--------------------------------------------------------------------------------
class NFDBuffer: public UMemory {
public:
    NFDBuffer(const UChar *text, int32_t length, UErrorCode &status);
    ~NFDBuffer();
    const UChar *getBuffer();
    int32_t getLength();

  private:
    const UChar *fOriginalText;
    UChar       *fNormalizedText;
    int32_t      fNormalizedTextLength;
    UChar        fSmallBuf[USPOOF_STACK_BUFFER_SIZE];
};





//-------------------------------------------------------------------------------------
//
//  SpoofData
//
//    A small class that wraps the raw (usually memory mapped) spoof data.
//    Serves two primary functions:
//      1.  Convenience.  Contains real pointers to the data, to avoid dealing with
//          the offsets in the raw data.
//      2.  Reference counting.  When a spoof checker is cloned, the raw data is shared
//          and must be retained until all checkers using the data are closed.
//    Nothing in this struct includes state that is specific to any particular
//    USpoofDetector object.
//
//---------------------------------------------------------------------------------------
class SpoofData: public UMemory {
  public:
    static SpoofData *getDefault(UErrorCode &status);   // Load standard ICU spoof data.
    SpoofData(UErrorCode &status);   // Create new spoof data wrapper.
                                     // Only used when building new data from rules.
    
    // Constructor for use when creating from prebuilt default data.
    //   A UDataMemory is what the ICU internal data loading functions provide.
    //   The udm is adopted by the SpoofData.
    SpoofData(UDataMemory *udm, UErrorCode &status);

    // Constructor for use when creating from serialized data.
    //
    SpoofData(const void *serializedData, int32_t length, UErrorCode &status);

    //  Check raw Spoof Data Version compatibility.
    //  Return TRUE it looks good.
    static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status);
    ~SpoofData();                    // Destructor not normally used.
                                     // Use removeReference() instead.
    // Reference Counting functions.
    //    Clone of a user-level spoof detector increments the ref count on the data.
    //    Close of a user-level spoof detector decrements the ref count.
    //    If the data is owned by us, it will be deleted when count goes to zero.
    SpoofData *addReference(); 
    void removeReference();

    // Reserve space in the raw data.  For use by builder when putting together a
    //   new set of data.  Init the new storage to zero, to prevent inconsistent
    //   results if it is not all otherwise set by the requester.
    //  Return:
    //    pointer to the new space that was added by this function.
    void *reserveSpace(int32_t numBytes, UErrorCode &status);

    // initialize the pointers from this object to the raw data.
    void initPtrs(UErrorCode &status);

    // Reset all fields to an initial state.
    // Called from the top of all constructors.
    void reset();
    
    SpoofDataHeader             *fRawData;          // Ptr to the raw memory-mapped data
    UBool                       fDataOwned;         // True if the raw data is owned, and needs
                                                    //  to be deleted when refcount goes to zero.
    UDataMemory                 *fUDM;              // If not NULL, our data came from a
                                                    //   UDataMemory, which we must close when
                                                    //   we're done.

    uint32_t                    fMemLimit;          // Limit of available raw data space
    int32_t                     fRefCount;

    // Confusable data
    int32_t                     *fCFUKeys;
    uint16_t                    *fCFUValues;
    SpoofStringLengthsElement   *fCFUStringLengths;
    UChar                       *fCFUStrings;

    // Whole Script Confusable Data
    UTrie2                      *fAnyCaseTrie;
    UTrie2                      *fLowerCaseTrie;
    ScriptSet                   *fScriptSets;
    };
    

//---------------------------------------------------------------------------------------
//
//  Raw Binary Data Formats, as loaded from the ICU data file,
//    or as built by the builder.
//
//---------------------------------------------------------------------------------------
struct SpoofDataHeader {
    int32_t       fMagic;                // (0x3845fdef)
    uint8_t       fFormatVersion[4];     // Data Format. Same as the value in struct UDataInfo
                                         //   if there is one associated with this data.
    int32_t       fLength;               // Total lenght in bytes of this spoof data,
                                         //   including all sections, not just the header.

    // The following four sections refer to data representing the confusable data
    //   from the Unicode.org data from "confusables.txt"

    int32_t       fCFUKeys;               // byte offset to Keys table (from SpoofDataHeader *)
    int32_t       fCFUKeysSize;           // number of entries in keys table  (32 bits each)

    // TODO: change name to fCFUValues, for consistency.
    int32_t       fCFUStringIndex;        // byte offset to String Indexes table
    int32_t       fCFUStringIndexSize;    // number of entries in String Indexes table (16 bits each)
                                          //     (number of entries must be same as in Keys table

    int32_t       fCFUStringTable;        // byte offset of String table
    int32_t       fCFUStringTableLen;     // length of string table (in 16 bit UChars)

    int32_t       fCFUStringLengths;      // byte offset to String Lengths table
    int32_t       fCFUStringLengthsSize;  // number of entries in lengths table. (2 x 16 bits each)


    // The following sections are for data from confusablesWholeScript.txt
    
    int32_t       fAnyCaseTrie;           // byte offset to the serialized Any Case Trie
    int32_t       fAnyCaseTrieLength;     // Length (bytes) of the serialized Any Case Trie
    
    int32_t       fLowerCaseTrie;         // byte offset to the serialized Lower Case Trie
    int32_t       fLowerCaseTrieLength;   // Length (bytes) of the serialized Lower Case Trie

    int32_t       fScriptSets;            // byte offset to array of ScriptSets
    int32_t       fScriptSetsLength;      // Number of ScriptSets (24 bytes each)
    

    // The following sections are for data from xidmodifications.txt
    
    
    int32_t       unused[15];              // Padding, Room for Expansion
    
 }; 



    
//
//  Structure for the Whole Script Confusable Data
//    See Unicode UAX-39, Unicode Security Mechanisms, for a description of the
//    Whole Script confusable data
//
//  The data provides mappings from code points to a set of scripts
//    that contain characters that might be confused with the code point.
//  There are two mappings, one for lower case only, and one for characters
//    of any case.
//
//  The actual data consists of a utrie2 to map from a code point to an offset,
//  and an array of UScriptSets (essentially bit maps) that is indexed
//  by the offsets obtained from the Trie.
//
//


U_NAMESPACE_END
#endif /* XP_CPLUSPLUS */

/**
  * Endianness swap function for binary spoof data.
  * @internal
  */
U_CAPI int32_t U_EXPORT2
uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
            UErrorCode *status);


#endif

#endif  /* USPOOFIM_H */