[apple/icu.git] / icuSources / tools / toolutil / uparse.c

/*
*******************************************************************************
*
*   Copyright (C) 2000-2006, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
*******************************************************************************
*   file name:  uparse.c
*   encoding:   US-ASCII
*   tab size:   8 (not used)
*   indentation:4
*
*   created on: 2000apr18
*   created by: Markus W. Scherer
*
*   This file provides a parser for files that are delimited by one single
*   character like ';' or TAB. Example: the Unicode Character Properties files
*   like UnicodeData.txt are semicolon-delimited.
*/

#include "unicode/utypes.h"
#include "cstring.h"
#include "filestrm.h"
#include "uparse.h"
#include "unicode/uchar.h"
#include "unicode/ustring.h"
#include "ustr_imp.h"

#include <stdio.h>

U_CAPI const char * U_EXPORT2
u_skipWhitespace(const char *s) {
    while(*s==' ' || *s=='\t') {
        ++s;
    }
    return s;
}

/*
 * If the string starts with # @missing: then return the pointer to the
 * following non-whitespace character.
 * Otherwise return the original pointer.
 * Unicode 5.0 adds such lines in some data files to document
 * default property values.
 * Poor man's regex for variable amounts of white space.
 */
static const char *
getMissingLimit(const char *s) {
    const char *s0=s;
    if(
        *(s=u_skipWhitespace(s))=='#' &&
        *(s=u_skipWhitespace(s+1))=='@' &&
        0==strncmp((s=u_skipWhitespace(s+1)), "missing", 7) &&
        *(s=u_skipWhitespace(s+7))==':'
    ) {
        return u_skipWhitespace(s+1);
    } else {
        return s0;
    }
}

U_CAPI void U_EXPORT2
u_parseDelimitedFile(const char *filename, char delimiter,
                     char *fields[][2], int32_t fieldCount,
                     UParseLineFn *lineFn, void *context,
                     UErrorCode *pErrorCode) {
    FileStream *file;
    char line[300];
    char *start, *limit;
    int32_t i, length;

    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return;
    }

    if(fields==NULL || lineFn==NULL || fieldCount<=0) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
        return;
    }

    if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) {
        filename=NULL;
        file=T_FileStream_stdin();
    } else {
        file=T_FileStream_open(filename, "r");
    }
    if(file==NULL) {
        *pErrorCode=U_FILE_ACCESS_ERROR;
        return;
    }

    while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
        length=(int32_t)uprv_strlen(line);

        /* remove trailing newline characters */
        while(length>0 && (line[length-1]=='\r' || line[length-1]=='\n')) {
            line[--length]=0;
        }

        /*
         * detect a line with # @missing:
         * start parsing after that, or else from the beginning of the line
         * set the default warning for @missing lines
         */
        start=(char *)getMissingLimit(line);
        if(start==line) {
            *pErrorCode=U_ZERO_ERROR;
        } else {
            *pErrorCode=U_USING_DEFAULT_WARNING;
        }

        /* skip this line if it is empty or a comment */
        if(*start==0 || *start=='#') {
            continue;
        }

        /* remove in-line comments */
        limit=uprv_strchr(start, '#');
        if(limit!=NULL) {
            /* get white space before the pound sign */
            while(limit>start && (*(limit-1)==' ' || *(limit-1)=='\t')) {
                --limit;
            }

            /* truncate the line */
            *limit=0;
        }

        /* skip lines with only whitespace */
        if(u_skipWhitespace(start)[0]==0) {
            continue;
        }

        /* for each field, call the corresponding field function */
        for(i=0; i<fieldCount; ++i) {
            /* set the limit pointer of this field */
            limit=start;
            while(*limit!=delimiter && *limit!=0) {
                ++limit;
            }

            /* set the field start and limit in the fields array */
            fields[i][0]=start;
            fields[i][1]=limit;

            /* set start to the beginning of the next field, if any */
            start=limit;
            if(*start!=0) {
                ++start;
            } else if(i+1<fieldCount) {
                *pErrorCode=U_PARSE_ERROR;
                limit=line+length;
                i=fieldCount;
                break;
            }
        }

        /* error in a field function? */
        if(U_FAILURE(*pErrorCode)) {
            break;
        }

        /* call the field function */
        lineFn(context, fields, fieldCount, pErrorCode);
        if(U_FAILURE(*pErrorCode)) {
            break;
        }
    }

    if(filename!=NULL) {
        T_FileStream_close(file);
    }
}

/*
 * parse a list of code points
 * store them as a UTF-32 string in dest[destCapacity]
 * return the number of code points
 */
U_CAPI int32_t U_EXPORT2
u_parseCodePoints(const char *s,
                  uint32_t *dest, int32_t destCapacity,
                  UErrorCode *pErrorCode) {
    char *end;
    uint32_t value;
    int32_t count;

    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return 0;
    }
    if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
        return 0;
    }

    count=0;
    for(;;) {
        s=u_skipWhitespace(s);
        if(*s==';' || *s==0) {
            return count;
        }

        /* read one code point */
        value=(uint32_t)uprv_strtoul(s, &end, 16);
        if(end<=s || (*end!=' ' && *end!='\t' && *end!=';' && *end!=0) || value>=0x110000) {
            *pErrorCode=U_PARSE_ERROR;
            return 0;
        }

        /* append it to the destination array */
        if(count<destCapacity) {
            dest[count++]=value;
        } else {
            *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
        }

        /* go to the following characters */
        s=end;
    }
}

/*
 * parse a list of code points
 * store them as a string in dest[destCapacity]
 * set the first code point in *pFirst
 * @return The length of the string in numbers of UChars.
 */
U_CAPI int32_t U_EXPORT2
u_parseString(const char *s,
              UChar *dest, int32_t destCapacity,
              uint32_t *pFirst,
              UErrorCode *pErrorCode) {
    char *end;
    uint32_t value;
    int32_t destLength;

    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return 0;
    }
    if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
    }

    if(pFirst!=NULL) {
        *pFirst=0xffffffff;
    }

    destLength=0;
    for(;;) {
        s=u_skipWhitespace(s);
        if(*s==';' || *s==0) {
            if(destLength<destCapacity) {
                dest[destLength]=0;
            } else if(destLength==destCapacity) {
                *pErrorCode=U_STRING_NOT_TERMINATED_WARNING;
            } else {
                *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
            }
            return destLength;
        }

        /* read one code point */
        value=(uint32_t)uprv_strtoul(s, &end, 16);
        if(end<=s || (*end!=' ' && *end!='\t' && *end!=';' && *end!=0) || value>=0x110000) {
            *pErrorCode=U_PARSE_ERROR;
            return 0;
        }

        /* store the first code point */
        if(destLength==0 && pFirst!=NULL) {
            *pFirst=value;
        }

        /* append it to the destination array */
        if((destLength+UTF_CHAR_LENGTH(value))<=destCapacity) {
            UTF_APPEND_CHAR_UNSAFE(dest, destLength, value);
        } else {
            destLength+=UTF_CHAR_LENGTH(value);
        }

        /* go to the following characters */
        s=end;
    }
}

/* read a range like start or start..end */
U_CAPI int32_t U_EXPORT2
u_parseCodePointRange(const char *s,
                      uint32_t *pStart, uint32_t *pEnd,
                      UErrorCode *pErrorCode) {
    char *end;
    uint32_t value;

    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return 0;
    }
    if(s==NULL || pStart==NULL || pEnd==NULL) {
        *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
        return 0;
    }

    s=u_skipWhitespace(s);
    if(*s==';' || *s==0) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }

    /* read the start code point */
    value=(uint32_t)uprv_strtoul(s, &end, 16);
    if(end<=s || (*end!=' ' && *end!='\t' && *end!='.' && *end!=';') || value>=0x110000) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }
    *pStart=*pEnd=value;

    /* is there a "..end"? */
    s=u_skipWhitespace(end);
    if(*s==';' || *s==0) {
        return 1;
    }

    if(*s!='.' || s[1]!='.') {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }
    s+=2;

    /* read the end code point */
    value=(uint32_t)uprv_strtoul(s, &end, 16);
    if(end<=s || (*end!=' ' && *end!='\t' && *end!=';') || value>=0x110000) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }
    *pEnd=value;

    /* is this a valid range? */
    if(value<*pStart) {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }

    /* no garbage after that? */
    s=u_skipWhitespace(end);
    if(*s==';' || *s==0) {
        return value-*pStart+1;
    } else {
        *pErrorCode=U_PARSE_ERROR;
        return 0;
    }
}

U_CAPI int32_t U_EXPORT2
u_parseUTF8(const char *source, int32_t sLen, char *dest, int32_t destCapacity, UErrorCode *status) {
    const char *read = source;
    int32_t i = 0;
    unsigned int value = 0;
    if(sLen == -1) {
        sLen = (int32_t)strlen(source);
    }
    
    while(read < source+sLen) {
        sscanf(read, "%2x", &value);
        if(i < destCapacity) {
            dest[i] = (char)value;
        }
        i++;
        read += 2;
    }
    return u_terminateChars(dest, destCapacity, i, status);
}
Commit	Line	Data
b75a7d8f A	1	/*
	2	*******************************************************************************
	3	*
73c04bcf	4	* Copyright (C) 2000-2006, International Business Machines
b75a7d8f A	5	* Corporation and others. All Rights Reserved.
	6	*
	7	*******************************************************************************
	8	* file name: uparse.c
	9	* encoding: US-ASCII
	10	* tab size: 8 (not used)
	11	* indentation:4
	12	*
	13	* created on: 2000apr18
	14	* created by: Markus W. Scherer
	15	*
	16	* This file provides a parser for files that are delimited by one single
	17	* character like ';' or TAB. Example: the Unicode Character Properties files
	18	* like UnicodeData.txt are semicolon-delimited.
	19	*/
	20
	21	#include "unicode/utypes.h"
	22	#include "cstring.h"
	23	#include "filestrm.h"
	24	#include "uparse.h"
	25	#include "unicode/uchar.h"
	26	#include "unicode/ustring.h"
	27	#include "ustr_imp.h"
	28
	29	#include <stdio.h>
	30
	31	U_CAPI const char * U_EXPORT2
	32	u_skipWhitespace(const char *s) {
	33	while(s==' ' \|\| s=='\t') {
	34	++s;
	35	}
	36	return s;
	37	}
	38
73c04bcf A	39	/*
	40	* If the string starts with # @missing: then return the pointer to the
	41	* following non-whitespace character.
	42	* Otherwise return the original pointer.
	43	* Unicode 5.0 adds such lines in some data files to document
	44	* default property values.
	45	* Poor man's regex for variable amounts of white space.
	46	*/
	47	static const char *
	48	getMissingLimit(const char *s) {
	49	const char *s0=s;
	50	if(
	51	*(s=u_skipWhitespace(s))=='#' &&
	52	*(s=u_skipWhitespace(s+1))=='@' &&
	53	0==strncmp((s=u_skipWhitespace(s+1)), "missing", 7) &&
	54	*(s=u_skipWhitespace(s+7))==':'
	55	) {
	56	return u_skipWhitespace(s+1);
	57	} else {
	58	return s0;
	59	}
	60	}
	61
b75a7d8f A	62	U_CAPI void U_EXPORT2
	63	u_parseDelimitedFile(const char *filename, char delimiter,
	64	char *fields[][2], int32_t fieldCount,
	65	UParseLineFn lineFn, void context,
	66	UErrorCode *pErrorCode) {
	67	FileStream *file;
	68	char line[300];
	69	char start, limit;
	70	int32_t i, length;
	71
	72	if(pErrorCode==NULL \|\| U_FAILURE(*pErrorCode)) {
	73	return;
	74	}
	75
	76	if(fields==NULL \|\| lineFn==NULL \|\| fieldCount<=0) {
	77	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
	78	return;
	79	}
	80
	81	if(filename==NULL \|\| filename==0 \|\| (filename=='-' && filename[1]==0)) {
	82	filename=NULL;
	83	file=T_FileStream_stdin();
	84	} else {
	85	file=T_FileStream_open(filename, "r");
	86	}
	87	if(file==NULL) {
	88	*pErrorCode=U_FILE_ACCESS_ERROR;
	89	return;
	90	}
	91
	92	while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) {
	93	length=(int32_t)uprv_strlen(line);
	94
	95	/* remove trailing newline characters */
	96	while(length>0 && (line[length-1]=='\r' \|\| line[length-1]=='\n')) {
	97	line[--length]=0;
	98	}
	99
73c04bcf A	100	/*
	101	* detect a line with # @missing:
	102	* start parsing after that, or else from the beginning of the line
	103	* set the default warning for @missing lines
	104	*/
	105	start=(char *)getMissingLimit(line);
	106	if(start==line) {
	107	*pErrorCode=U_ZERO_ERROR;
	108	} else {
	109	*pErrorCode=U_USING_DEFAULT_WARNING;
	110	}
	111
b75a7d8f	112	/* skip this line if it is empty or a comment */
73c04bcf	113	if(start==0 \|\| start=='#') {
b75a7d8f A	114	continue;
	115	}
	116
	117	/* remove in-line comments */
73c04bcf	118	limit=uprv_strchr(start, '#');
b75a7d8f A	119	if(limit!=NULL) {
b75a7d8f A	120	/* get white space before the pound sign */
73c04bcf	121	while(limit>start && ((limit-1)==' ' \|\| (limit-1)=='\t')) {
b75a7d8f A	122	--limit;
	123	}
	124
	125	/* truncate the line */
	126	*limit=0;
	127	}
	128
	129	/* skip lines with only whitespace */
73c04bcf	130	if(u_skipWhitespace(start)[0]==0) {
b75a7d8f A	131	continue;
	132	}
	133
	134	/* for each field, call the corresponding field function */
b75a7d8f A	135	for(i=0; i<fieldCount; ++i) {
	136	/* set the limit pointer of this field */
	137	limit=start;
	138	while(limit!=delimiter && limit!=0) {
	139	++limit;
	140	}
	141
	142	/* set the field start and limit in the fields array */
	143	fields[i][0]=start;
	144	fields[i][1]=limit;
	145
	146	/* set start to the beginning of the next field, if any */
	147	start=limit;
	148	if(*start!=0) {
	149	++start;
	150	} else if(i+1<fieldCount) {
	151	*pErrorCode=U_PARSE_ERROR;
	152	limit=line+length;
	153	i=fieldCount;
	154	break;
	155	}
	156	}
	157
	158	/* error in a field function? */
	159	if(U_FAILURE(*pErrorCode)) {
	160	break;
	161	}
	162
	163	/* call the field function */
	164	lineFn(context, fields, fieldCount, pErrorCode);
	165	if(U_FAILURE(*pErrorCode)) {
	166	break;
	167	}
	168	}
	169
	170	if(filename!=NULL) {
	171	T_FileStream_close(file);
	172	}
	173	}
	174
	175	/*
	176	* parse a list of code points
	177	* store them as a UTF-32 string in dest[destCapacity]
	178	* return the number of code points
	179	*/
	180	U_CAPI int32_t U_EXPORT2
	181	u_parseCodePoints(const char *s,
	182	uint32_t *dest, int32_t destCapacity,
	183	UErrorCode *pErrorCode) {
	184	char *end;
	185	uint32_t value;
	186	int32_t count;
	187
	188	if(pErrorCode==NULL \|\| U_FAILURE(*pErrorCode)) {
	189	return 0;
	190	}
	191	if(s==NULL \|\| destCapacity<0 \|\| (destCapacity>0 && dest==NULL)) {
	192	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
73c04bcf	193	return 0;
b75a7d8f A	194	}
	195
	196	count=0;
	197	for(;;) {
	198	s=u_skipWhitespace(s);
	199	if(s==';' \|\| s==0) {
	200	return count;
	201	}
	202
	203	/* read one code point */
	204	value=(uint32_t)uprv_strtoul(s, &end, 16);
	205	if(end<=s \|\| (end!=' ' && end!='\t' && end!=';' && end!=0) \|\| value>=0x110000) {
	206	*pErrorCode=U_PARSE_ERROR;
	207	return 0;
	208	}
	209
	210	/* append it to the destination array */
	211	if(count<destCapacity) {
	212	dest[count++]=value;
	213	} else {
	214	*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
	215	}
	216
	217	/* go to the following characters */
	218	s=end;
	219	}
	220	}
	221
	222	/*
	223	* parse a list of code points
	224	* store them as a string in dest[destCapacity]
	225	* set the first code point in *pFirst
	226	* @return The length of the string in numbers of UChars.
	227	*/
	228	U_CAPI int32_t U_EXPORT2
	229	u_parseString(const char *s,
	230	UChar *dest, int32_t destCapacity,
	231	uint32_t *pFirst,
	232	UErrorCode *pErrorCode) {
	233	char *end;
	234	uint32_t value;
	235	int32_t destLength;
	236
	237	if(pErrorCode==NULL \|\| U_FAILURE(*pErrorCode)) {
	238	return 0;
	239	}
	240	if(s==NULL \|\| destCapacity<0 \|\| (destCapacity>0 && dest==NULL)) {
	241	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
	242	}
	243
	244	if(pFirst!=NULL) {
	245	*pFirst=0xffffffff;
	246	}
	247
	248	destLength=0;
	249	for(;;) {
	250	s=u_skipWhitespace(s);
	251	if(s==';' \|\| s==0) {
	252	if(destLength<destCapacity) {
	253	dest[destLength]=0;
	254	} else if(destLength==destCapacity) {
	255	*pErrorCode=U_STRING_NOT_TERMINATED_WARNING;
	256	} else {
	257	*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
258	}
259	return destLength;
260	}
261
262	/* read one code point */
263	value=(uint32_t)uprv_strtoul(s, &end, 16);
264	if(end<=s \|\| (end!=' ' && end!='\t' && end!=';' && end!=0) \|\| value>=0x110000) {
265	*pErrorCode=U_PARSE_ERROR;
266	return 0;
267	}
268
269	/* store the first code point */
270	if(destLength==0 && pFirst!=NULL) {
271	*pFirst=value;
272	}
273
274	/* append it to the destination array */
275	if((destLength+UTF_CHAR_LENGTH(value))<=destCapacity) {
276	UTF_APPEND_CHAR_UNSAFE(dest, destLength, value);
277	} else {
278	destLength+=UTF_CHAR_LENGTH(value);
279	}
280
281	/* go to the following characters */
282	s=end;
283	}
284	}
285
286	/* read a range like start or start..end */
287	U_CAPI int32_t U_EXPORT2
288	u_parseCodePointRange(const char *s,
289	uint32_t pStart, uint32_t pEnd,
290	UErrorCode *pErrorCode) {
291	char *end;
292	uint32_t value;
293
294	if(pErrorCode==NULL \|\| U_FAILURE(*pErrorCode)) {
295	return 0;
296	}
297	if(s==NULL \|\| pStart==NULL \|\| pEnd==NULL) {
298	*pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
73c04bcf	299	return 0;
b75a7d8f A	300	}
	301
	302	s=u_skipWhitespace(s);
	303	if(s==';' \|\| s==0) {
	304	*pErrorCode=U_PARSE_ERROR;
	305	return 0;
	306	}
	307
	308	/* read the start code point */
	309	value=(uint32_t)uprv_strtoul(s, &end, 16);
	310	if(end<=s \|\| (end!=' ' && end!='\t' && end!='.' && end!=';') \|\| value>=0x110000) {
	311	*pErrorCode=U_PARSE_ERROR;
	312	return 0;
	313	}
	314	pStart=pEnd=value;
	315
	316	/* is there a "..end"? */
	317	s=u_skipWhitespace(end);
	318	if(s==';' \|\| s==0) {
	319	return 1;
	320	}
	321
	322	if(*s!='.' \|\| s[1]!='.') {
	323	*pErrorCode=U_PARSE_ERROR;
	324	return 0;
	325	}
	326	s+=2;
	327
	328	/* read the end code point */
	329	value=(uint32_t)uprv_strtoul(s, &end, 16);
	330	if(end<=s \|\| (end!=' ' && end!='\t' && *end!=';') \|\| value>=0x110000) {
	331	*pErrorCode=U_PARSE_ERROR;
	332	return 0;
	333	}
	334	*pEnd=value;
	335
	336	/* is this a valid range? */
	337	if(value<*pStart) {
	338	*pErrorCode=U_PARSE_ERROR;
	339	return 0;
	340	}
	341
	342	/* no garbage after that? */
	343	s=u_skipWhitespace(end);
	344	if(s==';' \|\| s==0) {
	345	return value-*pStart+1;
	346	} else {
	347	*pErrorCode=U_PARSE_ERROR;
	348	return 0;
	349	}
	350	}
	351
b75a7d8f A	352	U_CAPI int32_t U_EXPORT2
	353	u_parseUTF8(const char source, int32_t sLen, char dest, int32_t destCapacity, UErrorCode *status) {
	354	const char *read = source;
	355	int32_t i = 0;
	356	unsigned int value = 0;
	357	if(sLen == -1) {
374ca955	358	sLen = (int32_t)strlen(source);
b75a7d8f A	359	}
	360
	361	while(read < source+sLen) {
	362	sscanf(read, "%2x", &value);
	363	if(i < destCapacity) {
	364	dest[i] = (char)value;
	365	}
	366	i++;
	367	read += 2;
	368	}
	369	return u_terminateChars(dest, destCapacity, i, status);
	370	}