X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/73c04bcfe1096173b00431f0cdc742894b15eef0..5ea0322b6ab2af986e4c764284141380031dd014:/icuSources/tools/tzcode/tz2icu.cpp?ds=sidebyside diff --git a/icuSources/tools/tzcode/tz2icu.cpp b/icuSources/tools/tzcode/tz2icu.cpp index eba0d3aa..aeae5893 100644 --- a/icuSources/tools/tzcode/tz2icu.cpp +++ b/icuSources/tools/tzcode/tz2icu.cpp @@ -1,7 +1,7 @@ /* ********************************************************************** -* Copyright (c) 2003-2006, International Business Machines +* Copyright (c) 2003-2014, International Business Machines * Corporation and others. All Rights Reserved. ********************************************************************** * Author: Alan Liu @@ -50,18 +50,23 @@ using namespace std; +bool ICU44PLUS = TRUE; +string TZ_RESOURCE_NAME = ICU_TZ_RESOURCE; + //-------------------------------------------------------------------- // Time utilities //-------------------------------------------------------------------- -const long SECS_PER_YEAR = 31536000; // 365 days -const long SECS_PER_LEAP_YEAR = 31622400; // 366 days +const int64_t SECS_PER_YEAR = 31536000; // 365 days +const int64_t SECS_PER_LEAP_YEAR = 31622400; // 366 days +const int64_t LOWEST_TIME32 = (int64_t)((int32_t)0x80000000); +const int64_t HIGHEST_TIME32 = (int64_t)((int32_t)0x7fffffff); -bool isLeap(int y) { +bool isLeap(int32_t y) { return (y%4 == 0) && ((y%100 != 0) || (y%400 == 0)); // Gregorian } -long secsPerYear(int y) { +int64_t secsPerYear(int32_t y) { return isLeap(y) ? SECS_PER_LEAP_YEAR : SECS_PER_YEAR; } @@ -69,10 +74,10 @@ long secsPerYear(int y) { * Given a calendar year, return the GMT epoch seconds for midnight * GMT of January 1 of that year. yearToSeconds(1970) == 0. */ -long yearToSeconds(int year) { +int64_t yearToSeconds(int32_t year) { // inefficient but foolproof - long s = 0; - int y = 1970; + int64_t s = 0; + int32_t y = 1970; while (y < year) { s += secsPerYear(y++); } @@ -86,10 +91,10 @@ long yearToSeconds(int year) { * Given 1970 GMT epoch seconds, return the calendar year containing * that time. secondsToYear(0) == 1970. */ -int secondsToYear(long seconds) { +int32_t secondsToYear(int64_t seconds) { // inefficient but foolproof - int y = 1970; - long s = 0; + int32_t y = 1970; + int64_t s = 0; if (seconds >= 0) { for (;;) { s += secsPerYear(y++); @@ -116,9 +121,9 @@ struct SimplifiedZoneType; // A transition from one ZoneType to another // Minimal size = 5 bytes (4+1) struct Transition { - long time; // seconds, 1970 epoch - int type; // index into 'ZoneInfo.types' 0..255 - Transition(long _time, int _type) { + int64_t time; // seconds, 1970 epoch + int32_t type; // index into 'ZoneInfo.types' 0..255 + Transition(int64_t _time, int32_t _type) { time = _time; type = _type; } @@ -128,12 +133,12 @@ struct Transition { // Minimal size = 6 bytes (4+1+3bits) // SEE: SimplifiedZoneType struct ZoneType { - long rawoffset; // raw seconds offset from GMT - long dstoffset; // dst seconds offset from GMT + int64_t rawoffset; // raw seconds offset from GMT + int64_t dstoffset; // dst seconds offset from GMT // We don't really need any of the following, but they are // retained for possible future use. See SimplifiedZoneType. - int abbr; // index into ZoneInfo.abbrs 0..n-1 + int32_t abbr; // index into ZoneInfo.abbrs 0..n-1 bool isdst; bool isstd; bool isgmt; @@ -161,16 +166,16 @@ struct ZoneInfo { vector abbrs; string finalRuleID; - int finalOffset; - int finalYear; // -1 if none + int32_t finalOffset; + int32_t finalYear; // -1 if none // If this is an alias, then all other fields are meaningless, and // this field will point to the "real" zone 0..n-1. - int aliasTo; // -1 if this is a "real" zone + int32_t aliasTo; // -1 if this is a "real" zone // If there are aliases TO this zone, then the following set will // contain their index numbers (each index >= 0). - set aliases; + set aliases; ZoneInfo() : finalYear(-1), aliasTo(-1) {} @@ -179,13 +184,13 @@ struct ZoneInfo { void optimizeTypeList(); // Set this zone to be an alias TO another zone. - void setAliasTo(int index); + void setAliasTo(int32_t index); // Clear the list of aliases OF this zone. void clearAliases(); // Add an alias to the list of aliases OF this zone. - void addAlias(int index); + void addAlias(int32_t index); // Is this an alias to another zone? bool isAlias() const { @@ -193,7 +198,7 @@ struct ZoneInfo { } // Retrieve alias list - const set& getAliases() const { + const set& getAliases() const { return aliases; } @@ -205,12 +210,12 @@ void ZoneInfo::clearAliases() { aliases.clear(); } -void ZoneInfo::addAlias(int index) { +void ZoneInfo::addAlias(int32_t index) { assert(aliasTo < 0 && index >= 0 && aliases.find(index) == aliases.end()); aliases.insert(index); } -void ZoneInfo::setAliasTo(int index) { +void ZoneInfo::setAliasTo(int32_t index) { assert(index >= 0); assert(aliases.size() == 0); aliasTo = index; @@ -232,12 +237,12 @@ ZoneMap ZONEINFO; //-------------------------------------------------------------------- // Read zic-coded 32-bit integer from file -long readcoded(ifstream& file, long minv=numeric_limits::min(), - long maxv=numeric_limits::max()) { +int64_t readcoded(ifstream& file, int64_t minv=numeric_limits::min(), + int64_t maxv=numeric_limits::max()) { unsigned char buf[4]; // must be UNSIGNED - long val=0; + int64_t val=0; file.read((char*)buf, 4); - for(int i=0,shift=24;i<4;++i,shift-=8) { + for(int32_t i=0,shift=24;i<4;++i,shift-=8) { val |= buf[i] << shift; } if (val < minv || val > maxv) { @@ -249,13 +254,31 @@ long readcoded(ifstream& file, long minv=numeric_limits::min(), return val; } +// Read zic-coded 64-bit integer from file +int64_t readcoded64(ifstream& file, int64_t minv=numeric_limits::min(), + int64_t maxv=numeric_limits::max()) { + unsigned char buf[8]; // must be UNSIGNED + int64_t val=0; + file.read((char*)buf, 8); + for(int32_t i=0,shift=56;i<8;++i,shift-=8) { + val |= (int64_t)buf[i] << shift; + } + if (val < minv || val > maxv) { + ostringstream os; + os << "coded value out-of-range: " << val << ", expected [" + << minv << ", " << maxv << "]"; + throw out_of_range(os.str()); + } + return val; +} + // Read a boolean value bool readbool(ifstream& file) { char c; file.read(&c, 1); if (c!=0 && c!=1) { ostringstream os; - os << "boolean value out-of-range: " << (int)c; + os << "boolean value out-of-range: " << (int32_t)c; throw out_of_range(os.str()); } return (c!=0); @@ -265,8 +288,8 @@ bool readbool(ifstream& file) { * Read the zoneinfo file structure (see tzfile.h) into a ZoneInfo * @param file an already-open file stream */ -void readzoneinfo(ifstream& file, ZoneInfo& info) { - int i; +void readzoneinfo(ifstream& file, ZoneInfo& info, bool is64bitData) { + int32_t i; // Check for TZ_ICU_MAGIC signature at file start. If we get a // signature mismatch, it means we're trying to read a file which @@ -281,9 +304,9 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { } // skip additional Olson byte version file.read(buf, 1); - // if '\0', we have just one copy of data, if '2', there is additional + // if '\0', we have just one copy of data, if '2' or '3', there is additional // 64 bit version at the end. - if(buf[0]!=0 && buf[0]!='2') { + if(buf[0]!=0 && buf[0]!='2' && buf[0]!='3') { throw invalid_argument("Bad Olson version info"); } @@ -294,12 +317,12 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { } // Read array sizes - long isgmtcnt = readcoded(file, 0); - long isdstcnt = readcoded(file, 0); - long leapcnt = readcoded(file, 0); - long timecnt = readcoded(file, 0); - long typecnt = readcoded(file, 0); - long charcnt = readcoded(file, 0); + int64_t isgmtcnt = readcoded(file, 0); + int64_t isdstcnt = readcoded(file, 0); + int64_t leapcnt = readcoded(file, 0); + int64_t timecnt = readcoded(file, 0); + int64_t typecnt = readcoded(file, 0); + int64_t charcnt = readcoded(file, 0); // Confirm sizes that we assume to be equal. These assumptions // are drawn from a reading of the zic source (2003a), so they @@ -311,19 +334,23 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { // Used temporarily to store transition times and types. We need // to do this because the times and types are stored in two // separate arrays. - vector transitionTimes(timecnt, -1); // temporary - vector transitionTypes(timecnt, -1); // temporary + vector transitionTimes(timecnt, -1); // temporary + vector transitionTypes(timecnt, -1); // temporary // Read transition times for (i=0; i= typecnt) { ostringstream os; os << "illegal type: " << t << ", expected [0, " << (typecnt-1) << "]"; @@ -333,8 +360,40 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { } // Build transitions vector out of corresponding times and types. - for (i=0; i 0) { + int32_t minidx = -1; + for (i=0; i transitionTimes[minidx]) { + // Preserve the latest transition before the 32bit minimum time + minidx = i; + } + } else if (transitionTimes[i] > HIGHEST_TIME32) { + // Skipping the rest of the transition data. We cannot put such + // transitions into zoneinfo.res, because data is limited to singed + // 32bit int by the ICU resource bundle. + break; + } else { + info.transitions.push_back(Transition(transitionTimes[i], transitionTypes[i])); + } + } + + if (minidx != -1) { + // If there are any transitions before the 32bit minimum time, + // put the type information with the 32bit minimum time + vector::iterator itr = info.transitions.begin(); + info.transitions.insert(itr, Transition(LOWEST_TIME32, transitionTypes[minidx])); + } else { + // Otherwise, we need insert the initial type later + insertInitial = true; + } + } + } else { + for (i=0; i 0); + assert(typecnt > 0); + + int32_t initialTypeIdx = -1; + + // Check if the first type is not dst + if (info.types.at(0).dstoffset != 0) { + // Initial type's rawoffset is same with the rawoffset after the + // first transition, but no DST is observed. + int64_t rawoffset0 = (info.types.at(info.transitions.at(0).type)).rawoffset; + // Look for matching type + for (i=0; i<(int32_t)info.types.size(); ++i) { + if (info.types.at(i).rawoffset == rawoffset0 + && info.types.at(i).dstoffset == 0) { + initialTypeIdx = i; + break; + } + } + } else { + initialTypeIdx = 0; + } + assert(initialTypeIdx >= 0); + // Add the initial type associated with the lowest int32 time + vector::iterator itr = info.transitions.begin(); + info.transitions.insert(itr, Transition(LOWEST_TIME32, initialTypeIdx)); + } + + // Read the abbreviation string if (charcnt) { // All abbreviations are concatenated together, with a 0 at @@ -366,7 +455,7 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { // Split abbreviations apart into individual strings. Record // offset of each abbr in a vector. - vector abbroffset; + vector abbroffset; char *limit=str+charcnt; for (char* p=str; p::iterator it=info.types.begin(); it!=info.types.end(); ++it) { - vector::const_iterator x= + vector::const_iterator x= find(abbroffset.begin(), abbroffset.end(), it->abbr); if (x==abbroffset.end()) { // TODO: Modify code to add a new string to the end of @@ -400,7 +489,7 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { ostringstream os; os << "Warning: unusual abbr offset " << it->abbr << ", expected one of"; - for (vector::const_iterator y=abbroffset.begin(); + for (vector::const_iterator y=abbroffset.begin(); y!=abbroffset.end(); ++y) { os << ' ' << *y; } @@ -408,13 +497,13 @@ void readzoneinfo(ifstream& file, ZoneInfo& info) { #endif it->abbr = 0; } else { - int index = x - abbroffset.begin(); + int32_t index = x - abbroffset.begin(); it->abbr = index; abbrseen[index] = true; } } - for (int ii=0;ii<(int) abbrseen.size();++ii) { + for (int32_t ii=0;ii<(int32_t) abbrseen.size();++ii) { if (!abbrseen[ii]) { cerr << "Warning: unused abbreviation: " << ii << endl; } @@ -456,33 +545,53 @@ void handleFile(string path, string id) { if (!file) { throw invalid_argument("can't open file"); } + + // eat 32bit data part ZoneInfo info; - readzoneinfo(file, info); + readzoneinfo(file, info, false); // Check for errors if (!file) { throw invalid_argument("read error"); } - // Check eof-relative pos (there may be a cleaner way to do this) - long eofPos = (long) file.tellg(); - char buf[32]; - file.read(buf, 4); - file.seekg(0, ios::end); - eofPos = eofPos - (long) file.tellg(); - if (eofPos) { - // 2006c merged 32 and 64 bit versions in a fat binary - // 64 version starts at the end of 32 bit version. - // Therefore, if the file is *not* consumed, check - // if it is maybe being restarted. - if (strncmp(buf, TZ_ICU_MAGIC, 4) != 0) { + // we only use 64bit part + ZoneInfo info64; + readzoneinfo(file, info64, true); + + bool alldone = false; + int64_t eofPos = (int64_t) file.tellg(); + + // '\n' + + '\n' after the 64bit version data + char ch = file.get(); + if (ch == 0x0a) { + bool invalidchar = false; + while (file.get(ch)) { + if (ch == 0x0a) { + break; + } + if (ch < 0x20) { + // must be printable ascii + invalidchar = true; + break; + } + } + if (!invalidchar) { + eofPos = (int64_t) file.tellg(); + file.seekg(0, ios::end); + eofPos = eofPos - (int64_t) file.tellg(); + if (eofPos == 0) { + alldone = true; + } + } + } + if (!alldone) { ostringstream os; os << (-eofPos) << " unprocessed bytes at end"; throw invalid_argument(os.str()); - } } - ZONEINFO[id] = info; + ZONEINFO[id] = info64; } /** @@ -575,7 +684,7 @@ void scandir(string dir, string prefix="") { closedir(dp); chdir(pwd); - for(int i=0;i<(int)subfiles.size();i+=2) { + for(int32_t i=0;i<(int32_t)subfiles.size();i+=2) { try { handleFile(subfiles[i], subfiles[i+1]); } catch (const exception& e) { @@ -584,7 +693,7 @@ void scandir(string dir, string prefix="") { exit(1); } } - for(int i=0;i<(int)subdirs.size();i+=2) { + for(int32_t i=0;i<(int32_t)subdirs.size();i+=2) { scandir(subdirs[i], subdirs[i+1]); } } @@ -599,7 +708,7 @@ void scandir(string dir, string prefix="") { * Read and discard the current line. */ void consumeLine(istream& in) { - int c; + int32_t c; do { c = in.get(); } while (c != EOF && c != '\n'); @@ -615,16 +724,16 @@ const char* TIME_MODE[] = {"w", "s", "u"}; // Allow 29 days in February because zic outputs February 29 // for rules like "last Sunday in February". -const int MONTH_LEN[] = {31,29,31,30,31,30,31,31,30,31,30,31}; +const int32_t MONTH_LEN[] = {31,29,31,30,31,30,31,31,30,31,30,31}; -const int HOUR = 3600; +const int32_t HOUR = 3600; struct FinalZone { - int offset; // raw offset - int year; // takes effect for y >= year + int32_t offset; // raw offset + int32_t year; // takes effect for y >= year string ruleid; set aliases; - FinalZone(int _offset, int _year, const string& _ruleid) : + FinalZone(int32_t _offset, int32_t _year, const string& _ruleid) : offset(_offset), year(_year), ruleid(_ruleid) { if (offset <= -16*HOUR || offset >= 16*HOUR) { ostringstream os; @@ -633,7 +742,7 @@ struct FinalZone { << " and rule ID " << ruleid; throw invalid_argument(os.str()); } - if (year < 1900 || year >= 2050) { + if (year < 1900) { ostringstream os; os << "Invalid input year " << year << " with offset " << offset @@ -653,12 +762,12 @@ struct FinalZone { }; struct FinalRulePart { - int mode; - int month; - int dom; - int dow; - int time; - int offset; // dst offset, usually either 0 or 1:00 + int32_t mode; + int32_t month; + int32_t dom; + int32_t dow; + int32_t time; + int32_t offset; // dst offset, usually either 0 or 1:00 // Isstd and isgmt only have 3 valid states, corresponding to local // wall time, local standard time, and GMT standard time. @@ -682,13 +791,13 @@ struct FinalRulePart { FinalRulePart() : isset(false) {} void set(const string& id, const string& _mode, - int _month, - int _dom, - int _dow, - int _time, + int32_t _month, + int32_t _dom, + int32_t _dow, + int32_t _time, bool _isstd, bool _isgmt, - int _offset) { + int32_t _offset) { if (isset) { throw invalid_argument("FinalRulePart set twice"); } @@ -720,7 +829,7 @@ struct FinalRulePart { if (mode != DOM && (dow < 0 || dow >= 7)) { os << "Invalid input day of week " << dow; } - if (offset < 0 || offset > HOUR) { + if (offset < 0 || offset > (2 * HOUR)) { os << "Invalid input offset " << offset; } if (isgmt && !isstd) { @@ -741,7 +850,7 @@ struct FinalRulePart { * Return the time mode as an ICU SimpleTimeZone int from 0..2; * see simpletz.h. */ - int timemode() const { + int32_t timemode() const { if (isgmt) { assert(isstd); return 2; // gmt standard @@ -766,14 +875,14 @@ struct FinalRulePart { /** * Return a "dowim" param suitable for SimpleTimeZone. */ - int stz_dowim() const { + int32_t stz_dowim() const { return (mode == DOWLEQ) ? -dom : dom; } /** * Return a "dow" param suitable for SimpleTimeZone. */ - int stz_dow() const { + int32_t stz_dow() const { return (mode == DOM) ? 0 : -(dow+1); } }; @@ -840,7 +949,7 @@ void readFinalZonesAndRules(istream& in) { } else if (token == "zone") { // zone Africa/Cairo 7200 1995 Egypt # zone Africa/Cairo, offset 7200, year >= 1995, rule Egypt (0) string id, ruleid; - int offset, year; + int32_t offset, year; in >> id >> offset >> year >> ruleid; consumeLine(in); finalZones[id] = FinalZone(offset, year, ruleid); @@ -848,12 +957,12 @@ void readFinalZonesAndRules(istream& in) { // rule US DOWGEQ 3 1 0 7200 0 0 3600 # 52: US, file data/northamerica, line 119, mode DOWGEQ, April, dom 1, Sunday, time 7200, isstd 0, isgmt 0, offset 3600 // rule US DOWLEQ 9 31 0 7200 0 0 0 # 53: US, file data/northamerica, line 114, mode DOWLEQ, October, dom 31, Sunday, time 7200, isstd 0, isgmt 0, offset 0 string id, mode; - int month, dom, dow, time, offset; + int32_t month, dom, dow, time, offset; bool isstd, isgmt; in >> id >> mode >> month >> dom >> dow >> time >> isstd >> isgmt >> offset; consumeLine(in); FinalRule& fr = finalRules[id]; - int p = fr.part[0].isset ? 1 : 0; + int32_t p = fr.part[0].isset ? 1 : 0; fr.part[p].set(id, mode, month, dom, dow, time, isstd, isgmt, offset); } else if (token == "link") { string fromid, toid; // fromid == "real" zone, toid == alias @@ -906,7 +1015,6 @@ void readFinalZonesAndRules(istream& in) { void ZoneInfo::print(ostream& os, const string& id) const { // Implement compressed format #2: - os << " /* " << id << " */ "; if (aliasTo >= 0) { @@ -915,22 +1023,75 @@ void ZoneInfo::print(ostream& os, const string& id) const { return; } - os << ":array {" << endl; + if (ICU44PLUS) { + os << ":table {" << endl; + } else { + os << ":array {" << endl; + } vector::const_iterator trn; vector::const_iterator typ; - bool first=true; - os << " :intvector { "; - for (trn = transitions.begin(); trn != transitions.end(); ++trn) { - if (!first) os << ", "; - first = false; - os << trn->time; + bool first; + + if (ICU44PLUS) { + trn = transitions.begin(); + + // pre 32bit transitions + if (trn != transitions.end() && trn->time < LOWEST_TIME32) { + os << " transPre32:intvector { "; + for (first = true; trn != transitions.end() && trn->time < LOWEST_TIME32; ++trn) { + if (!first) { + os<< ", "; + } + first = false; + os << (int32_t)(trn->time >> 32) << ", " << (int32_t)(trn->time & 0x00000000ffffffff); + } + os << " }" << endl; + } + + // 32bit transtions + if (trn != transitions.end() && trn->time < HIGHEST_TIME32) { + os << " trans:intvector { "; + for (first = true; trn != transitions.end() && trn->time < HIGHEST_TIME32; ++trn) { + if (!first) { + os << ", "; + } + first = false; + os << trn->time; + } + os << " }" << endl; + } + + // post 32bit transitons + if (trn != transitions.end()) { + os << " transPost32:intvector { "; + for (first = true; trn != transitions.end(); ++trn) { + if (!first) { + os<< ", "; + } + first = false; + os << (int32_t)(trn->time >> 32) << ", " << (int32_t)(trn->time & 0x00000000ffffffff); + } + os << " }" << endl; + } + } else { + os << " :intvector { "; + for (trn = transitions.begin(), first = true; trn != transitions.end(); ++trn) { + if (!first) os << ", "; + first = false; + os << trn->time; + } + os << " }" << endl; } - os << " }" << endl; + first=true; - os << " :intvector { "; + if (ICU44PLUS) { + os << " typeOffsets:intvector { "; + } else { + os << " :intvector { "; + } for (typ = types.begin(); typ != types.end(); ++typ) { if (!first) os << ", "; first = false; @@ -938,24 +1099,44 @@ void ZoneInfo::print(ostream& os, const string& id) const { } os << " }" << endl; - os << " :bin { \"" << hex << setfill('0'); - for (trn = transitions.begin(); trn != transitions.end(); ++trn) { - os << setw(2) << trn->type; + if (ICU44PLUS) { + if (transitions.size() != 0) { + os << " typeMap:bin { \"" << hex << setfill('0'); + for (trn = transitions.begin(); trn != transitions.end(); ++trn) { + os << setw(2) << trn->type; + } + os << dec << "\" }" << endl; + } + } else { + os << " :bin { \"" << hex << setfill('0'); + for (trn = transitions.begin(); trn != transitions.end(); ++trn) { + os << setw(2) << trn->type; + } + os << dec << "\" }" << endl; } - os << dec << "\" }" << endl; // Final zone info, if any if (finalYear != -1) { - os << " \"" << finalRuleID << "\"" << endl; - os << " :intvector { " << finalOffset << ", " - << finalYear << " }" << endl; + if (ICU44PLUS) { + os << " finalRule { \"" << finalRuleID << "\" }" << endl; + os << " finalRaw:int { " << finalOffset << " }" << endl; + os << " finalYear:int { " << finalYear << " }" << endl; + } else { + os << " \"" << finalRuleID << "\"" << endl; + os << " :intvector { " << finalOffset << ", " + << finalYear << " }" << endl; + } } // Alias list, if any if (aliases.size() != 0) { first = true; - os << " :intvector { "; - for (set::const_iterator i=aliases.begin(); i!=aliases.end(); ++i) { + if (ICU44PLUS) { + os << " links:intvector { "; + } else { + os << " :intvector { "; + } + for (set::const_iterator i=aliases.begin(); i!=aliases.end(); ++i) { if (!first) os << ", "; first = false; os << *i; @@ -968,11 +1149,11 @@ void ZoneInfo::print(ostream& os, const string& id) const { inline ostream& operator<<(ostream& os, const ZoneMap& zoneinfo) { - int c = 0; + int32_t c = 0; for (ZoneMapIter it = zoneinfo.begin(); it != zoneinfo.end(); ++it) { - if(c) os << ","; + if(c && !ICU44PLUS) os << ","; it->second.print(os, it->first); os << "//Z#" << c++ << endl; } @@ -981,8 +1162,8 @@ operator<<(ostream& os, const ZoneMap& zoneinfo) { // print the string list ostream& printStringList( ostream& os, const ZoneMap& zoneinfo) { - int n = 0; // count - int col = 0; // column + int32_t n = 0; // count + int32_t col = 0; // column os << " Names {" << endl << " "; for (ZoneMapIter it = zoneinfo.begin(); @@ -1013,7 +1194,7 @@ ostream& printStringList( ostream& os, const ZoneMap& zoneinfo) { //-------------------------------------------------------------------- // Unary predicate for finding transitions after a given time -bool isAfter(const Transition t, long thresh) { +bool isAfter(const Transition t, int64_t thresh) { return t.time >= thresh; } @@ -1022,8 +1203,8 @@ bool isAfter(const Transition t, long thresh) { * optimizeTypeList() method. */ struct SimplifiedZoneType { - long rawoffset; - long dstoffset; + int64_t rawoffset; + int64_t dstoffset; SimplifiedZoneType() : rawoffset(-1), dstoffset(-1) {} SimplifiedZoneType(const ZoneType& t) : rawoffset(t.rawoffset), dstoffset(t.dstoffset) {} @@ -1061,50 +1242,136 @@ void ZoneInfo::optimizeTypeList() { if (aliasTo >= 0) return; // Nothing to do for aliases - // If there are zero transitions and one type, then leave that as-is. - if (transitions.size() == 0) { - if (types.size() != 1) { - cerr << "Error: transition count = 0, type count = " << types.size() << endl; + if (!ICU44PLUS) { + // This is the old logic which has a bug, which occasionally removes + // the type before the first transition. The problem was fixed + // by inserting the dummy transition indirectly. + + // If there are zero transitions and one type, then leave that as-is. + if (transitions.size() == 0) { + if (types.size() != 1) { + cerr << "Error: transition count = 0, type count = " << types.size() << endl; + } + return; } - return; - } - set simpleset; - for (vector::const_iterator i=transitions.begin(); - i!=transitions.end(); ++i) { - assert(i->type < (int)types.size()); - simpleset.insert(types[i->type]); - } + set simpleset; + for (vector::const_iterator i=transitions.begin(); + i!=transitions.end(); ++i) { + assert(i->type < (int32_t)types.size()); + simpleset.insert(types[i->type]); + } - // Map types to integer indices - map simplemap; - int n=0; - for (set::const_iterator i=simpleset.begin(); - i!=simpleset.end(); ++i) { - simplemap[*i] = n++; - } + // Map types to integer indices + map simplemap; + int32_t n=0; + for (set::const_iterator i=simpleset.begin(); + i!=simpleset.end(); ++i) { + simplemap[*i] = n++; + } + + // Remap transitions + for (vector::iterator i=transitions.begin(); + i!=transitions.end(); ++i) { + assert(i->type < (int32_t)types.size()); + ZoneType oldtype = types[i->type]; + SimplifiedZoneType newtype(oldtype); + assert(simplemap.find(newtype) != simplemap.end()); + i->type = simplemap[newtype]; + } + + // Replace type list + types.clear(); + copy(simpleset.begin(), simpleset.end(), back_inserter(types)); + + } else { + if (types.size() > 1) { + // Note: localtime uses the very first non-dst type as initial offsets. + // If all types are DSTs, the very first type is treated as the initial offsets. + + // Decide a type used as the initial offsets. ICU put the type at index 0. + ZoneType initialType = types[0]; + for (vector::const_iterator i=types.begin(); i!=types.end(); ++i) { + if (i->dstoffset == 0) { + initialType = *i; + break; + } + } - // Remap transitions - for (vector::iterator i=transitions.begin(); - i!=transitions.end(); ++i) { - assert(i->type < (int)types.size()); - ZoneType oldtype = types[i->type]; - SimplifiedZoneType newtype(oldtype); - assert(simplemap.find(newtype) != simplemap.end()); - i->type = simplemap[newtype]; + SimplifiedZoneType initialSimplifiedType(initialType); + + // create a set of unique types, but ignoring fields which we're not interested in + set simpleset; + simpleset.insert(initialSimplifiedType); + for (vector::const_iterator i=transitions.begin(); i!=transitions.end(); ++i) { + assert(i->type < (int32_t)types.size()); + simpleset.insert(types[i->type]); + } + + // Map types to integer indices, however, keeping the first type at offset 0 + map simplemap; + simplemap[initialSimplifiedType] = 0; + int32_t n = 1; + for (set::const_iterator i=simpleset.begin(); i!=simpleset.end(); ++i) { + if (*i < initialSimplifiedType || initialSimplifiedType < *i) { + simplemap[*i] = n++; + } + } + + // Remap transitions + for (vector::iterator i=transitions.begin(); + i!=transitions.end(); ++i) { + assert(i->type < (int32_t)types.size()); + ZoneType oldtype = types[i->type]; + SimplifiedZoneType newtype(oldtype); + assert(simplemap.find(newtype) != simplemap.end()); + i->type = simplemap[newtype]; + } + + // Replace type list + types.clear(); + types.push_back(initialSimplifiedType); + for (set::const_iterator i=simpleset.begin(); i!=simpleset.end(); ++i) { + if (*i < initialSimplifiedType || initialSimplifiedType < *i) { + types.push_back(*i); + } + } + + // Reiterating transitions to remove any transitions which + // do not actually change the raw/dst offsets + int32_t prevTypeIdx = 0; + for (vector::iterator i=transitions.begin(); i!=transitions.end();) { + if (i->type == prevTypeIdx) { + // this is not a time transition, probably just name change + // e.g. America/Resolute after 2006 in 2010b + transitions.erase(i); + } else { + prevTypeIdx = i->type; + i++; + } + } + } } - // Replace type list - types.clear(); - copy(simpleset.begin(), simpleset.end(), back_inserter(types)); } /** * Merge final zone data into this zone. */ void ZoneInfo::mergeFinalData(const FinalZone& fz) { - int year = fz.year; - long seconds = yearToSeconds(year); + int32_t year = fz.year; + int64_t seconds = yearToSeconds(year); + + if (!ICU44PLUS) { + if (seconds > HIGHEST_TIME32) { + // Avoid transitions beyond signed 32bit max second. + // This may result incorrect offset computation around + // HIGHEST_TIME32. This is a limitation of ICU + // before 4.4. + seconds = HIGHEST_TIME32; + } + } + vector::iterator it = find_if(transitions.begin(), transitions.end(), bind2nd(ptr_fun(isAfter), seconds)); @@ -1148,12 +1415,12 @@ void mergeFinalZone(const pair& p) { void FinalRule::print(ostream& os) const { // First print the rule part that enters DST; then the rule part // that exits it. - int whichpart = (part[0].offset != 0) ? 0 : 1; + int32_t whichpart = (part[0].offset != 0) ? 0 : 1; assert(part[whichpart].offset != 0); assert(part[1-whichpart].offset == 0); os << " "; - for (int i=0; i<2; ++i) { + for (int32_t i=0; i<2; ++i) { const FinalRulePart& p = part[whichpart]; whichpart = 1-whichpart; os << p.month << ", " << p.stz_dowim() << ", " << p.stz_dow() << ", " @@ -1164,22 +1431,35 @@ void FinalRule::print(ostream& os) const { int main(int argc, char *argv[]) { string rootpath, zonetab, version; + bool validArgs = FALSE; - if (argc != 4) { - cout << "Usage: tz2icu " << endl - << " path to zoneinfo file tree generated by" << endl - << " ICU-patched version of zic" << endl - << " country map, from tzdata archive," << endl - << " typically named \"zone.tab\"" << endl - << " version string, such as \"2003e\"" << endl; - exit(1); - } else { + if (argc == 4 || argc == 5) { + validArgs = TRUE; rootpath = argv[1]; zonetab = argv[2]; version = argv[3]; + if (argc == 5) { + if (strcmp(argv[4], "--old") == 0) { + ICU44PLUS = FALSE; + TZ_RESOURCE_NAME = ICU_TZ_RESOURCE_OLD; + } else { + validArgs = FALSE; + } + } + } + if (!validArgs) { + cout << "Usage: tz2icu [--old]" << endl + << " path to zoneinfo file tree generated by" << endl + << " ICU-patched version of zic" << endl + << " country map, from tzdata archive," << endl + << " typically named \"zone.tab\"" << endl + << " version string, such as \"2003e\"" << endl + << " --old generating resource format before ICU4.4" << endl; + exit(1); } cout << "Olson data version: " << version << endl; + cout << "ICU 4.4+ format: " << (ICU44PLUS ? "Yes" : "No") << endl; try { ifstream finals(ICU_ZONE_FILE); @@ -1198,64 +1478,6 @@ int main(int argc, char *argv[]) { return 1; } - // Read the legacy alias list and process it. Treat the legacy mappings - // like links, but also record them in the "legacy" hash. - try { - ifstream aliases(ICU_TZ_ALIAS); - if (!aliases) { - cerr << "Error: Unable to open " ICU_TZ_ALIAS << endl; - return 1; - } - int n = 0; - string line; - while (getline(aliases, line)) { - string::size_type lb = line.find('#'); - if (lb != string::npos) { - line.resize(lb); // trim comments - } - vector a; - istringstream is(line); - copy(istream_iterator(is),istream_iterator(), - back_inserter(a)); - if (a.size() == 0) continue; // blank line - if (a.size() != 2) { - cerr << "Error: Can't parse \"" << line << "\" in " - ICU_TZ_ALIAS << endl; - exit(1); - } - ++n; - - string alias(a[0]), olson(a[1]); - if (links.find(alias) != links.end()) { - cerr << "Error: Alias \"" << alias - << "\" is an Olson zone in " - ICU_TZ_ALIAS << endl; - return 1; - } - if (reverseLinks.find(alias) != reverseLinks.end()) { - cerr << "Error: Alias \"" << alias - << "\" is an Olson link to \"" << reverseLinks[olson] - << "\" in " << ICU_TZ_ALIAS << endl; - return 1; - } - - // Record source for error reporting - if (linkSource.find(olson) == linkSource.end()) { - linkSource[olson] = "ICU alias"; - } - assert(linkSource.find(alias) == linkSource.end()); - linkSource[alias] = "ICU alias"; - - links[olson].insert(alias); - reverseLinks[alias] = olson; - } - cout << "Finished reading " << n - << " aliases from " ICU_TZ_ALIAS << endl; - } catch (const exception& error) { - cerr << "Error: While reading " ICU_TZ_ALIAS ": " << error.what() << endl; - return 1; - } - try { // Recursively scan all files below the given path, accumulating // their data into ZONEINFO. All files must be TZif files. Any @@ -1298,14 +1520,14 @@ int main(int argc, char *argv[]) { } // 2. Create a mapping from zones to index numbers 0..n-1. - map zoneIDs; + map zoneIDs; vector zoneIDlist; - int z=0; + int32_t z=0; for (ZoneMap::iterator i=ZONEINFO.begin(); i!=ZONEINFO.end(); ++i) { zoneIDs[i->first] = z++; zoneIDlist.push_back(i->first); } - assert(z == (int) ZONEINFO.size()); + assert(z == (int32_t) ZONEINFO.size()); // 3. Merge aliases. Sometimes aliases link to other aliases; we // resolve these into simplest possible sets. @@ -1359,15 +1581,43 @@ int main(int argc, char *argv[]) { } // Create the country map + map icuRegions; // ICU's custom zone -> country override map > countryMap; // country -> set of zones map reverseCountryMap; // zone -> country + + try { + // Read icuregions file to collect ICU's own zone-region mapping data. + ifstream frg(ICU_REGIONS); + if (frg) { + string line; + while (getline(frg, line)) { + if (line[0] == '#') continue; + + string zone, country; + istringstream is(line); + is >> zone >> country; + if (zone.size() == 0) continue; + if (country.size() < 2) { + cerr << "Error: Can't parse " << line << " in " << ICU_REGIONS << endl; + return 1; + } + icuRegions[zone] = country; + } + } else { + cout << "No custom region map [icuregions]" << endl; + } + } catch (const exception& error) { + cerr << "Error: While reading " << ICU_REGIONS << ": " << error.what() << endl; + return 1; + } + try { ifstream f(zonetab.c_str()); if (!f) { cerr << "Error: Unable to open " << zonetab << endl; return 1; } - int n = 0; + int32_t n = 0; string line; while (getline(f, line)) { string::size_type lb = line.find('#'); @@ -1387,6 +1637,13 @@ int main(int argc, char *argv[]) { << " in " << zonetab << endl; return 1; } + if (icuRegions.find(zone) != icuRegions.end()) { + // Custom override + string customCountry = icuRegions[zone]; + cout << "Region Mapping: custom override for " << zone + << " " << country << " -> " << customCountry << endl; + country = customCountry; + } countryMap[country].insert(zone); reverseCountryMap[zone] = country; //cerr << (n+1) << ": " << country << " <=> " << zone << endl; @@ -1399,6 +1656,20 @@ int main(int argc, char *argv[]) { return 1; } + // Merge ICU's own zone-region mapping data + for (map::const_iterator i = icuRegions.begin(); + i != icuRegions.end(); ++i) { + const string& zid(i->first); + if (reverseCountryMap.find(zid) != reverseCountryMap.end()) { + continue; + } + cout << "Region Mapping: custom data zone=" << zid + << ", region=" << i->second << endl; + + reverseCountryMap[zid] = i->second; + countryMap[i->second].insert(zid); + } + // Merge ICU aliases into country map. Don't merge any alias // that already has a country map, since that doesn't make sense. // E.g. "Link Europe/Oslo Arctic/Longyearbyen" doesn't mean we @@ -1434,11 +1705,12 @@ int main(int argc, char *argv[]) { time_t sec; time(&sec); struct tm* now = localtime(&sec); - int thisYear = now->tm_year + 1900; + int32_t thisYear = now->tm_year + 1900; + string filename = TZ_RESOURCE_NAME + ".txt"; // Write out a resource-bundle source file containing data for // all zones. - ofstream file(ICU_TZ_RESOURCE ".txt"); + ofstream file(filename.c_str()); if (file) { file << "//---------------------------------------------------------" << endl << "// Copyright (C) 2003"; @@ -1448,17 +1720,17 @@ int main(int argc, char *argv[]) { file << ", International Business Machines" << endl << "// Corporation and others. All Rights Reserved." << endl << "//---------------------------------------------------------" << endl - << "// Build tool: tz2icu" << endl - << "// Build date: " << asctime(now) /* << endl -- asctime emits CR */ - << "// Olson source: ftp://elsie.nci.nih.gov/pub/" << endl - << "// Olson version: " << version << endl + << "// Build tool: tz2icu" << endl + << "// Build date: " << asctime(now) /* << endl -- asctime emits CR */ + << "// tz database: ftp://ftp.iana.org/tz/" << endl + << "// tz version: " << version << endl << "// ICU version: " << U_ICU_VERSION << endl << "//---------------------------------------------------------" << endl << "// >> !!! >> THIS IS A MACHINE-GENERATED FILE << !!! <<" << endl << "// >> !!! >>> DO NOT EDIT <<< !!! <<" << endl << "//---------------------------------------------------------" << endl << endl - << ICU_TZ_RESOURCE ":table(nofallback) {" << endl + << TZ_RESOURCE_NAME << ":table(nofallback) {" << endl << " TZVersion { \"" << version << "\" }" << endl << " Zones:array { " << endl << ZONEINFO // Zones (the actual data) @@ -1470,7 +1742,7 @@ int main(int argc, char *argv[]) { // Final Rules are used if requested by the zone file << " Rules { " << endl; // Emit final rules - int frc = 0; + int32_t frc = 0; for(map::iterator i=finalRules.begin(); i!=finalRules.end(); ++i) { const string& id = i->first; @@ -1481,35 +1753,47 @@ int main(int argc, char *argv[]) { } file << " }" << endl; - // Emit country (region) map. Emitting the string zone IDs results - // in a 188 kb binary resource; emitting the zone index numbers - // trims this to 171 kb. More work for the runtime code, but - // a smaller data footprint. - file << " Regions { " << endl; - int rc = 0; - for (map >::const_iterator i=countryMap.begin(); - i != countryMap.end(); ++i) { - string country = i->first; - const set& zones(i->second); - file << " "; - if(country[0]==0) { - file << "Default"; + // Emit country (region) map. + if (ICU44PLUS) { + file << " Regions:array {" << endl; + int32_t zn = 0; + for (ZoneMap::iterator i=ZONEINFO.begin(); i!=ZONEINFO.end(); ++i) { + map::iterator cit = reverseCountryMap.find(i->first); + if (cit == reverseCountryMap.end()) { + file << " \"001\","; + } else { + file << " \"" << cit->second << "\", "; + } + file << "//Z#" << zn++ << " " << i->first << endl; } - file << country << ":intvector { "; - bool first = true; - for (set::const_iterator j=zones.begin(); - j != zones.end(); ++j) { - if (!first) file << ", "; - first = false; - if (zoneIDs.find(*j) == zoneIDs.end()) { - cerr << "Error: Nonexistent zone in country map: " << *j << endl; - return 1; + file << " }" << endl; + } else { + file << " Regions { " << endl; + int32_t rc = 0; + for (map >::const_iterator i=countryMap.begin(); + i != countryMap.end(); ++i) { + string country = i->first; + const set& zones(i->second); + file << " "; + if(country[0]==0) { + file << "Default"; + } + file << country << ":intvector { "; + bool first = true; + for (set::const_iterator j=zones.begin(); + j != zones.end(); ++j) { + if (!first) file << ", "; + first = false; + if (zoneIDs.find(*j) == zoneIDs.end()) { + cerr << "Error: Nonexistent zone in country map: " << *j << endl; + return 1; + } + file << zoneIDs[*j]; // emit the zone's index number } - file << zoneIDs[*j]; // emit the zone's index number + file << " } //R#" << rc++ << endl; } - file << " } //R#" << rc++ << endl; + file << " }" << endl; } - file << " }" << endl; file << "}" << endl; } @@ -1517,100 +1801,10 @@ int main(int argc, char *argv[]) { file.close(); if (file) { // recheck error bit - cout << "Finished writing " ICU_TZ_RESOURCE ".txt" << endl; + cout << "Finished writing " << TZ_RESOURCE_NAME << ".txt" << endl; } else { - cerr << "Error: Unable to open/write to " ICU_TZ_RESOURCE ".txt" << endl; + cerr << "Error: Unable to open/write to " << TZ_RESOURCE_NAME << ".txt" << endl; return 1; } - -#define ICU4J_TZ_CLASS "ZoneMetaData" - - // Write out a Java source file containing only a few pieces of - // meta-data missing from the core JDK: the equivalency lists and - // the country map. - ofstream java(ICU4J_TZ_CLASS ".java"); - if (java) { - java << "//---------------------------------------------------------" << endl - << "// Copyright (C) 2003"; - if (thisYear > 2003) { - java << "-" << thisYear; - } - java << ", International Business Machines" << endl - << "// Corporation and others. All Rights Reserved." << endl - << "//---------------------------------------------------------" << endl - << "// Build tool: tz2icu" << endl - << "// Build date: " << asctime(now) /* << endl -- asctime emits CR */ - << "// Olson source: ftp://elsie.nci.nih.gov/pub/" << endl - << "// Olson version: " << version << endl - << "// ICU version: " << U_ICU_VERSION << endl - << "//---------------------------------------------------------" << endl - << "// >> !!! >> THIS IS A MACHINE-GENERATED FILE << !!! <<" << endl - << "// >> !!! >>> DO NOT EDIT <<< !!! <<" << endl - << "//---------------------------------------------------------" << endl - << endl - << "package com.ibm.icu.impl;" << endl - << endl - << "public final class " ICU4J_TZ_CLASS " {" << endl; - - // Emit equivalency lists - bool first1 = true; - java << " public static final String VERSION = \"" + version + "\";" << endl; - java << " public static final String[][] EQUIV = {" << endl; - for (ZoneMap::const_iterator i=ZONEINFO.begin(); i!=ZONEINFO.end(); ++i) { - if (i->second.isAlias() || i->second.getAliases().size() == 0) { - continue; - } - if (!first1) java << "," << endl; - first1 = false; - // The ID of this zone (the canonical zone, to which the - // aliases point) will be sorted into the list, so it - // won't be at position 0. If we want to know which is - // the canonical zone, we should move it to position 0. - java << " { "; - bool first2 = true; - const set& s = i->second.getAliases(); - for (set::const_iterator j=s.begin(); j!=s.end(); ++j) { - if (!first2) java << ", "; - java << '"' << zoneIDlist[*j] << '"'; - first2 = false; - } - java << " }"; - } - java << endl - << " };" << endl; - - // Emit country map. - first1 = true; - java << " public static final String[][] COUNTRY = {" << endl; - for (map >::const_iterator i=countryMap.begin(); - i != countryMap.end(); ++i) { - if (!first1) java << "," << endl; - first1 = false; - string country = i->first; - const set& zones(i->second); - java << " { \"" << country << '"'; - for (set::const_iterator j=zones.begin(); - j != zones.end(); ++j) { - java << ", \"" << *j << '"'; - } - java << " }"; - } - java << endl - << " };" << endl; - - java << "}" << endl; - } - - java.close(); - - if (java) { // recheck error bit - cout << "Finished writing " ICU4J_TZ_CLASS ".java" << endl; - } else { - cerr << "Error: Unable to open/write to " ICU4J_TZ_CLASS ".java" << endl; - return 1; - } - - return 0; } - //eof