[apple/icu.git] / icuSources / tools / toolutil / uparse.c

/*
*******************************************************************************
*
*   Copyright (C) 2000-2010, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
*******************************************************************************
*   file name:  uparse.c
*   encoding:   US-ASCII
*   tab size:   8 (not used)
*   indentation:4
*
*   created on: 2000apr18
*   created by: Markus W. Scherer
*
*   This file provides a parser for files that are delimited by one single
*   character like ';' or TAB. Example: the Unicode Character Properties files
*   like UnicodeData.txt are semicolon-delimited.
*/

#include "unicode/utypes.h"
#include "cstring.h"
#include "filestrm.h"
#include "uparse.h"
#include "unicode/uchar.h"
#include "unicode/ustring.h"
#include "ustr_imp.h"

#include <stdio.h>

U_CAPI const char * U_EXPORT2
u_skipWhitespace(const char *s) {
    while(U_IS_INV_WHITESPACE(*s)) {
        ++s;
    }
    return s;
}

U_CAPI char * U_EXPORT2
u_rtrim(char *s) {
    char *end=uprv_strchr(s, 0);
    while(s<end && U_IS_INV_WHITESPACE(*(end-1))) {
        *--end = 0;
    }
    return end;
}

/*
 * If the string starts with # @missing: then return the pointer to the
 * following non-whitespace character.
 * Otherwise return the original pointer.
 * Unicode 5.0 adds such lines in some data files to document
 * default property values.
 * Poor man's regex for variable amounts of white space.
 */
static const char *
getMissingLimit(const char *s) {
    const char *s0=s;
    if(
        *(s=u_skipWhitespace(s))=='#' &&
        *(s=u_skipWhitespace(s+1))=='@' &&
        0==strncmp((s=u_skipWhitespace(s+1)), "missing", 7) &&
        *(s=u_skipWhitespace(s+7))==':'
    ) {
        return u_skipWhitespace(s+1);
    } else {
        return s0;
    }
}

U_CAPI void U_EXPORT2
u_parseDelimitedFile(const char *filename, char delimiter,
                     char *fields[][2], int32_t fieldCount,
                     UParseLineFn *lineFn, void *context,
                     UErrorCode *pErrorCode) {
    FileStream *file;
    char line[300];
    char *start, *limit;
    int32_t i, length;

    if(U_FAILURE(*pErrorCode)) {
        return;
    }

    if(fields==NULL || lineFn==NULL || fieldCount<=0) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
        return;
    }

    if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) {
        filename=NULL;
        file=T_FileStream_stdin();
    } else {
        file=T_FileStream_open(filename, "r");
    }
    if(file==NULL) {
        *pErrorCode=U_FILE_ACCESS_ERROR;
        return;
    }

    while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
        /* remove trailing newline characters */
        length=(int32_t)(u_rtrim(line)-line);

        /*
         * detect a line with # @missing:
         * start parsing after that, or else from the beginning of the line
         * set the default warning for @missing lines
         */
        start=(char *)getMissingLimit(line);
        if(start==line) {
            *pErrorCode=U_ZERO_ERROR;
        } else {
            *pErrorCode=U_USING_DEFAULT_WARNING;
        }

        /* skip this line if it is empty or a comment */
        if(*start==0 || *start=='#') {
            continue;
        }

        /* remove in-line comments */
        limit=uprv_strchr(start, '#');
        if(limit!=NULL) {
            /* get white space before the pound sign */
            while(limit>start && U_IS_INV_WHITESPACE(*(limit-1))) {
                --limit;
            }

            /* truncate the line */
            *limit=0;
        }

        /* skip lines with only whitespace */
        if(u_skipWhitespace(start)[0]==0) {
            continue;
        }

        /* for each field, call the corresponding field function */
        for(i=0; i<fieldCount; ++i) {
            /* set the limit pointer of this field */
            limit=start;
            while(*limit!=delimiter && *limit!=0) {
                ++limit;
            }

            /* set the field start and limit in the fields array */
            fields[i][0]=start;
            fields[i][1]=limit;

            /* set start to the beginning of the next field, if any */
            start=limit;
            if(*start!=0) {
                ++start;
            } else if(i+1<fieldCount) {
                *pErrorCode=U_PARSE_ERROR;
                limit=line+length;
                i=fieldCount;
                break;
            }
        }

        /* error in a field function? */
        if(U_FAILURE(*pErrorCode)) {
            break;
        }

        /* call the field function */
        lineFn(context, fields, fieldCount, pErrorCode);
        if(U_FAILURE(*pErrorCode)) {
            break;
        }
    }

    if(filename!=NULL) {
        T_FileStream_close(file);
    }
}

/*
 * parse a list of code points
 * store them as a UTF-32 string in dest[destCapacity]
 * return the number of code points
 */
U_CAPI int32_t U_EXPORT2
u_parseCodePoints(const char *s,
                  uint32_t *dest, int32_t destCapacity,
                  UErrorCode *pErrorCode) {
    char *end;
    uint32_t value;
    int32_t count;

    if(U_FAILURE(*pErrorCode)) {
        return 0;
    }
    if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
        return 0;
    }

    count=0;
    for(;;) {
        s=u_skipWhitespace(s);
        if(*s==';' || *s==0) {
            return count;
        }

        /* read one code point */
        value=(uint32_t)uprv_strtoul(s, &end, 16);
        if(end<=s || (!U_IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0) || value>=0x110000) {
            *pErrorCode=U_PARSE_ERROR;
            return 0;
        }

        /* append it to the destination array */
        if(count<destCapacity) {
            dest[count++]=value;
        } else {
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
        }

        /* go to the following characters */
        s=end;
    }
}

/*
 * parse a list of code points
 * store them as a string in dest[destCapacity]
 * set the first code point in *pFirst
 * @return The length of the string in numbers of UChars.
 */
U_CAPI int32_t U_EXPORT2
u_parseString(const char *s,
              UChar *dest, int32_t destCapacity,
              uint32_t *pFirst,
              UErrorCode *pErrorCode) {
    char *end;
    uint32_t value;
    int32_t destLength;

    if(U_FAILURE(*pErrorCode)) {
        return 0;
    }
    if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
    }

    if(pFirst!=NULL) {
        *pFirst=0xffffffff;
    }

    destLength=0;
    for(;;) {
        s=u_skipWhitespace(s);
        if(*s==';' || *s==0) {
            if(destLength<destCapacity) {
                dest[destLength]=0;
            } else if(destLength==destCapacity) {
                *pErrorCode=U_STRING_NOT_TERMINATED_WARNING;
            } else {
                *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
            }
            return destLength;
        }

        /* read one code point */
        value=(uint32_t)uprv_strtoul(s, &end, 16);
        if(end<=s || (!U_IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0) || value>=0x110000) {
            *pErrorCode=U_PARSE_ERROR;
            return 0;
        }

        /* store the first code point */
        if(pFirst!=NULL) {
            *pFirst=value;
            pFirst=NULL;
        }

        /* append it to the destination array */
        if((destLength+U16_LENGTH(value))<=destCapacity) {
            U16_APPEND_UNSAFE(dest, destLength, value);
        } else {
            destLength+=U16_LENGTH(value);
        }

        /* go to the following characters */
        s=end;
    }
}

/* read a range like start or start..end */
U_CAPI int32_t U_EXPORT2
u_parseCodePointRangeAnyTerminator(const char *s,
                                   uint32_t *pStart, uint32_t *pEnd,
                                   const char **terminator,
                                   UErrorCode *pErrorCode) {
    char *end;
    uint32_t value;

    if(U_FAILURE(*pErrorCode)) {
        return 0;
    }
    if(s==NULL || pStart==NULL || pEnd==NULL) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
        return 0;
    }

    /* read the start code point */
    s=u_skipWhitespace(s);
    value=(uint32_t)uprv_strtoul(s, &end, 16);
    if(end<=s || value>=0x110000) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }
    *pStart=*pEnd=value;

    /* is there a "..end"? */
    s=u_skipWhitespace(end);
    if(*s!='.' || s[1]!='.') {
        *terminator=end;
        return 1;
    }
    s=u_skipWhitespace(s+2);

    /* read the end code point */
    value=(uint32_t)uprv_strtoul(s, &end, 16);
    if(end<=s || value>=0x110000) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }
    *pEnd=value;

    /* is this a valid range? */
    if(value<*pStart) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }

    *terminator=end;
    return value-*pStart+1;
}

U_CAPI int32_t U_EXPORT2
u_parseCodePointRange(const char *s,
                      uint32_t *pStart, uint32_t *pEnd,
                      UErrorCode *pErrorCode) {
    const char *terminator;
    int32_t rangeLength=
        u_parseCodePointRangeAnyTerminator(s, pStart, pEnd, &terminator, pErrorCode);
    if(U_SUCCESS(*pErrorCode)) {
        terminator=u_skipWhitespace(terminator);
        if(*terminator!=';' && *terminator!=0) {
            *pErrorCode=U_PARSE_ERROR;
            return 0;
        }
    }
    return rangeLength;
}

U_CAPI int32_t U_EXPORT2
u_parseUTF8(const char *source, int32_t sLen, char *dest, int32_t destCapacity, UErrorCode *status) {
    const char *read = source;
    int32_t i = 0;
    unsigned int value = 0;
    if(sLen == -1) {
        sLen = (int32_t)strlen(source);
    }
    
    while(read < source+sLen) {
        sscanf(read, "%2x", &value);
        if(i < destCapacity) {
            dest[i] = (char)value;
        }
        i++;
        read += 2;
    }
    return u_terminateChars(dest, destCapacity, i, status);
}
Commit	Line	Data
b75a7d8f A	1	/*
	2	*******************************************************************************
	3	*
729e4ab9	4	* Copyright (C) 2000-2010, International Business Machines
b75a7d8f A	5	* Corporation and others. All Rights Reserved.
	6	*
	7	*******************************************************************************
	8	* file name: uparse.c
	9	* encoding: US-ASCII
	10	* tab size: 8 (not used)
	11	* indentation:4
	12	*
	13	* created on: 2000apr18
	14	* created by: Markus W. Scherer
	15	*
	16	* This file provides a parser for files that are delimited by one single
	17	* character like ';' or TAB. Example: the Unicode Character Properties files
	18	* like UnicodeData.txt are semicolon-delimited.
	19	*/
	20
	21	#include "unicode/utypes.h"
	22	#include "cstring.h"
	23	#include "filestrm.h"
	24	#include "uparse.h"
	25	#include "unicode/uchar.h"
	26	#include "unicode/ustring.h"
	27	#include "ustr_imp.h"
	28
	29	#include <stdio.h>
	30
	31	U_CAPI const char * U_EXPORT2
	32	u_skipWhitespace(const char *s) {
729e4ab9	33	while(U_IS_INV_WHITESPACE(*s)) {
b75a7d8f A	34	++s;
	35	}
	36	return s;
	37	}
	38
729e4ab9 A	39	U_CAPI char * U_EXPORT2
	40	u_rtrim(char *s) {
	41	char *end=uprv_strchr(s, 0);
	42	while(s<end && U_IS_INV_WHITESPACE(*(end-1))) {
	43	*--end = 0;
	44	}
	45	return end;
	46	}
	47
73c04bcf A	48	/*
	49	* If the string starts with # @missing: then return the pointer to the
	50	* following non-whitespace character.
	51	* Otherwise return the original pointer.
	52	* Unicode 5.0 adds such lines in some data files to document
	53	* default property values.
	54	* Poor man's regex for variable amounts of white space.
	55	*/
	56	static const char *
	57	getMissingLimit(const char *s) {
	58	const char *s0=s;
	59	if(
	60	*(s=u_skipWhitespace(s))=='#' &&
	61	*(s=u_skipWhitespace(s+1))=='@' &&
	62	0==strncmp((s=u_skipWhitespace(s+1)), "missing", 7) &&
	63	*(s=u_skipWhitespace(s+7))==':'
	64	) {
	65	return u_skipWhitespace(s+1);
	66	} else {
	67	return s0;
	68	}
	69	}
	70
b75a7d8f A	71	U_CAPI void U_EXPORT2
	72	u_parseDelimitedFile(const char *filename, char delimiter,
	73	char *fields[][2], int32_t fieldCount,
	74	UParseLineFn lineFn, void context,
	75	UErrorCode *pErrorCode) {
	76	FileStream *file;
	77	char line[300];
	78	char start, limit;
	79	int32_t i, length;
	80
729e4ab9	81	if(U_FAILURE(*pErrorCode)) {
b75a7d8f A	82	return;
	83	}
	84
	85	if(fields==NULL \|\| lineFn==NULL \|\| fieldCount<=0) {
	86	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
	87	return;
	88	}
	89
	90	if(filename==NULL \|\| filename==0 \|\| (filename=='-' && filename[1]==0)) {
	91	filename=NULL;
	92	file=T_FileStream_stdin();
	93	} else {
	94	file=T_FileStream_open(filename, "r");
	95	}
	96	if(file==NULL) {
	97	*pErrorCode=U_FILE_ACCESS_ERROR;
	98	return;
	99	}
	100
	101	while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
b75a7d8f	102	/* remove trailing newline characters */
729e4ab9	103	length=(int32_t)(u_rtrim(line)-line);
b75a7d8f	104
73c04bcf A	105	/*
	106	* detect a line with # @missing:
	107	* start parsing after that, or else from the beginning of the line
	108	* set the default warning for @missing lines
	109	*/
	110	start=(char *)getMissingLimit(line);
	111	if(start==line) {
	112	*pErrorCode=U_ZERO_ERROR;
	113	} else {
	114	*pErrorCode=U_USING_DEFAULT_WARNING;
	115	}
	116
b75a7d8f	117	/* skip this line if it is empty or a comment */
73c04bcf	118	if(start==0 \|\| start=='#') {
b75a7d8f A	119	continue;
	120	}
	121
	122	/* remove in-line comments */
73c04bcf	123	limit=uprv_strchr(start, '#');
b75a7d8f A	124	if(limit!=NULL) {
b75a7d8f A	125	/* get white space before the pound sign */
729e4ab9	126	while(limit>start && U_IS_INV_WHITESPACE(*(limit-1))) {
b75a7d8f A	127	--limit;
	128	}
	129
	130	/* truncate the line */
	131	*limit=0;
	132	}
	133
	134	/* skip lines with only whitespace */
73c04bcf	135	if(u_skipWhitespace(start)[0]==0) {
b75a7d8f A	136	continue;
	137	}
	138
	139	/* for each field, call the corresponding field function */
b75a7d8f A	140	for(i=0; i<fieldCount; ++i) {
	141	/* set the limit pointer of this field */
	142	limit=start;
	143	while(limit!=delimiter && limit!=0) {
	144	++limit;
	145	}
	146
	147	/* set the field start and limit in the fields array */
	148	fields[i][0]=start;
	149	fields[i][1]=limit;
	150
	151	/* set start to the beginning of the next field, if any */
	152	start=limit;
	153	if(*start!=0) {
	154	++start;
	155	} else if(i+1<fieldCount) {
	156	*pErrorCode=U_PARSE_ERROR;
	157	limit=line+length;
	158	i=fieldCount;
	159	break;
	160	}
	161	}
	162
	163	/* error in a field function? */
	164	if(U_FAILURE(*pErrorCode)) {
	165	break;
	166	}
	167
	168	/* call the field function */
	169	lineFn(context, fields, fieldCount, pErrorCode);
	170	if(U_FAILURE(*pErrorCode)) {
	171	break;
	172	}
	173	}
	174
	175	if(filename!=NULL) {
	176	T_FileStream_close(file);
	177	}
	178	}
	179
	180	/*
	181	* parse a list of code points
	182	* store them as a UTF-32 string in dest[destCapacity]
	183	* return the number of code points
	184	*/
	185	U_CAPI int32_t U_EXPORT2
	186	u_parseCodePoints(const char *s,
	187	uint32_t *dest, int32_t destCapacity,
	188	UErrorCode *pErrorCode) {
	189	char *end;
	190	uint32_t value;
	191	int32_t count;
	192
729e4ab9	193	if(U_FAILURE(*pErrorCode)) {
b75a7d8f A	194	return 0;
	195	}
	196	if(s==NULL \|\| destCapacity<0 \|\| (destCapacity>0 && dest==NULL)) {
	197	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
73c04bcf	198	return 0;
b75a7d8f A	199	}
	200
	201	count=0;
	202	for(;;) {
	203	s=u_skipWhitespace(s);
	204	if(s==';' \|\| s==0) {
	205	return count;
	206	}
	207
	208	/* read one code point */
	209	value=(uint32_t)uprv_strtoul(s, &end, 16);
729e4ab9	210	if(end<=s \|\| (!U_IS_INV_WHITESPACE(end) && end!=';' && *end!=0) \|\| value>=0x110000) {
b75a7d8f A	211	*pErrorCode=U_PARSE_ERROR;
	212	return 0;
	213	}
	214
	215	/* append it to the destination array */
	216	if(count<destCapacity) {
	217	dest[count++]=value;
	218	} else {
	219	*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
	220	}
	221
	222	/* go to the following characters */
	223	s=end;
	224	}
	225	}
	226
	227	/*
	228	* parse a list of code points
	229	* store them as a string in dest[destCapacity]
	230	* set the first code point in *pFirst
	231	* @return The length of the string in numbers of UChars.
	232	*/
	233	U_CAPI int32_t U_EXPORT2
	234	u_parseString(const char *s,
	235	UChar *dest, int32_t destCapacity,
	236	uint32_t *pFirst,
	237	UErrorCode *pErrorCode) {
	238	char *end;
	239	uint32_t value;
	240	int32_t destLength;
	241
729e4ab9	242	if(U_FAILURE(*pErrorCode)) {
b75a7d8f A	243	return 0;
	244	}
	245	if(s==NULL \|\| destCapacity<0 \|\| (destCapacity>0 && dest==NULL)) {
	246	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
	247	}
	248
	249	if(pFirst!=NULL) {
	250	*pFirst=0xffffffff;
	251	}
	252
	253	destLength=0;
	254	for(;;) {
	255	s=u_skipWhitespace(s);
	256	if(s==';' \|\| s==0) {
	257	if(destLength<destCapacity) {
	258	dest[destLength]=0;
	259	} else if(destLength==destCapacity) {
	260	*pErrorCode=U_STRING_NOT_TERMINATED_WARNING;
	261	} else {
	262	*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
	263	}
	264	return destLength;
	265	}
	266
	267	/* read one code point */
	268	value=(uint32_t)uprv_strtoul(s, &end, 16);
729e4ab9	269	if(end<=s \|\| (!U_IS_INV_WHITESPACE(end) && end!=';' && *end!=0) \|\| value>=0x110000) {
b75a7d8f A	270	*pErrorCode=U_PARSE_ERROR;
	271	return 0;
	272	}
	273
	274	/* store the first code point */
729e4ab9	275	if(pFirst!=NULL) {
b75a7d8f	276	*pFirst=value;
729e4ab9	277	pFirst=NULL;
b75a7d8f A	278	}
	279
	280	/* append it to the destination array */
729e4ab9 A	281	if((destLength+U16_LENGTH(value))<=destCapacity) {
729e4ab9 A	282	U16_APPEND_UNSAFE(dest, destLength, value);
b75a7d8f	283	} else {
729e4ab9	284	destLength+=U16_LENGTH(value);
b75a7d8f A	285	}
	286
	287	/* go to the following characters */
	288	s=end;
	289	}
	290	}
	291
	292	/* read a range like start or start..end */
	293	U_CAPI int32_t U_EXPORT2
729e4ab9 A	294	u_parseCodePointRangeAnyTerminator(const char *s,
	295	uint32_t pStart, uint32_t pEnd,
	296	const char **terminator,
	297	UErrorCode *pErrorCode) {
b75a7d8f A	298	char *end;
	299	uint32_t value;
	300
729e4ab9	301	if(U_FAILURE(*pErrorCode)) {
b75a7d8f A	302	return 0;
	303	}
	304	if(s==NULL \|\| pStart==NULL \|\| pEnd==NULL) {
	305	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
73c04bcf	306	return 0;
b75a7d8f A	307	}
b75a7d8f A	308
b75a7d8f	309	/* read the start code point */
729e4ab9	310	s=u_skipWhitespace(s);
b75a7d8f	311	value=(uint32_t)uprv_strtoul(s, &end, 16);
729e4ab9	312	if(end<=s \|\| value>=0x110000) {
b75a7d8f A	313	*pErrorCode=U_PARSE_ERROR;
	314	return 0;
	315	}
	316	pStart=pEnd=value;
	317
	318	/* is there a "..end"? */
	319	s=u_skipWhitespace(end);
b75a7d8f	320	if(*s!='.' \|\| s[1]!='.') {
729e4ab9 A	321	*terminator=end;
729e4ab9 A	322	return 1;
b75a7d8f	323	}
729e4ab9	324	s=u_skipWhitespace(s+2);
b75a7d8f A	325
	326	/* read the end code point */
	327	value=(uint32_t)uprv_strtoul(s, &end, 16);
729e4ab9	328	if(end<=s \|\| value>=0x110000) {
b75a7d8f A	329	*pErrorCode=U_PARSE_ERROR;
	330	return 0;
	331	}
	332	*pEnd=value;
	333
	334	/* is this a valid range? */
	335	if(value<*pStart) {
	336	*pErrorCode=U_PARSE_ERROR;
	337	return 0;
	338	}
	339
729e4ab9 A	340	*terminator=end;
	341	return value-*pStart+1;
	342	}
	343
	344	U_CAPI int32_t U_EXPORT2
	345	u_parseCodePointRange(const char *s,
	346	uint32_t pStart, uint32_t pEnd,
	347	UErrorCode *pErrorCode) {
	348	const char *terminator;
	349	int32_t rangeLength=
	350	u_parseCodePointRangeAnyTerminator(s, pStart, pEnd, &terminator, pErrorCode);
	351	if(U_SUCCESS(*pErrorCode)) {
	352	terminator=u_skipWhitespace(terminator);
	353	if(terminator!=';' && terminator!=0) {
	354	*pErrorCode=U_PARSE_ERROR;
	355	return 0;
	356	}
b75a7d8f	357	}
729e4ab9	358	return rangeLength;
b75a7d8f A	359	}
b75a7d8f A	360
b75a7d8f A	361	U_CAPI int32_t U_EXPORT2
	362	u_parseUTF8(const char source, int32_t sLen, char dest, int32_t destCapacity, UErrorCode *status) {
	363	const char *read = source;
	364	int32_t i = 0;
	365	unsigned int value = 0;
	366	if(sLen == -1) {
374ca955	367	sLen = (int32_t)strlen(source);
b75a7d8f A	368	}
	369
	370	while(read < source+sLen) {
	371	sscanf(read, "%2x", &value);
	372	if(i < destCapacity) {
	373	dest[i] = (char)value;
	374	}
	375	i++;
	376	read += 2;
	377	}
	378	return u_terminateChars(dest, destCapacity, i, status);
	379	}