From 788c5b99e8c1d32d5cd274020a0bd937ecad0f4c Mon Sep 17 00:00:00 2001 From: "Jay Freeman (saurik)" Date: Wed, 21 Jan 2009 11:17:34 +0000 Subject: [PATCH] Fixed Dl-Limit (long standing APT bug that they won't acknowledge), specified error messages for HTTP status codes, and added support for download timeouts. M cfnetwork.diff A dllimit.diff A parallel.diff git-svn-id: http://svn.telesphoreo.org/trunk@534 514c082c-b64e-11dc-b46d-3d985efe055d --- apt-pkg/acquire.cc | 24 +++++++-- methods/http.cc | 115 ++++++++++++++++++++++++++++++++++++++++--- methods/http.cc.orig | 115 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 237 insertions(+), 17 deletions(-) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index fff1b2b..b1463dc 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -238,9 +238,27 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config) /* Single-Instance methods get exactly one queue per URI. This is also used for the Access queue method */ if (Config->SingleInstance == true || QueueMode == QueueAccess) - return U.Access; + return U.Access; + string name(U.Access + ':' + U.Host); - return U.Access + ':' + U.Host; + int parallel(_config->FindI("Acquire::"+U.Access+"::MaxParallel",0)); + if (parallel <= 0) + return name; + + typedef map indexmap; + static indexmap indices; + + pair cache(indices.insert(indexmap::value_type(name, -1))); + if (cache.second || cache.first->second == -1) { + int &index(indices[U.Access]); + if (index >= parallel) + index = 0; + cache.first->second = index++; + } + + ostringstream value; + value << U.Access << "::" << cache.first->second; + return value.str(); } /*}}}*/ // Acquire::GetConfig - Fetch the configuration information /*{{{*/ @@ -268,7 +286,7 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) return 0; /* if a method uses DownloadLimit, we switch to SingleInstance mode */ - if(_config->FindI("Acquire::"+Access+"::DlLimit",0) > 0) + if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0) Conf->SingleInstance = true; return Conf; diff --git a/methods/http.cc b/methods/http.cc index 5d4848e..7df6f93 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -45,6 +45,7 @@ extern "C" { #include #include #include +#include // Internet stuff #include @@ -686,6 +687,51 @@ bool ServerState::HeaderLine(string Line) } /*}}}*/ +static const CFOptionFlags kNetworkEvents = + kCFStreamEventOpenCompleted | + kCFStreamEventHasBytesAvailable | + kCFStreamEventEndEncountered | + kCFStreamEventErrorOccurred | +0; + +static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) { + switch (event) { + case kCFStreamEventOpenCompleted: + break; + + case kCFStreamEventHasBytesAvailable: + case kCFStreamEventEndEncountered: + *reinterpret_cast(arg) = 1; + CFRunLoopStop(CFRunLoopGetCurrent()); + break; + + case kCFStreamEventErrorOccurred: + *reinterpret_cast(arg) = -1; + CFRunLoopStop(CFRunLoopGetCurrent()); + break; + } +} + +/* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */ +int CFReadStreamOpen(CFReadStreamRef stream, double timeout) { + CFStreamClientContext context; + int value(0); + + memset(&context, 0, sizeof(context)); + context.info = &value; + + if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) { + CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes); + if (CFReadStreamOpen(stream)) + CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false); + else + value = -1; + CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL); + } + + return value; +} + // HttpMethod::SendReq - Send the HTTP request /*{{{*/ // --------------------------------------------------------------------- /* This places the http request in the outbound buffer */ @@ -1099,6 +1145,8 @@ int HttpMethod::Loop() signal(SIGINT,SigTerm); Server = 0; + + std::set cached; int FailCounter = 0; while (1) @@ -1126,6 +1174,14 @@ int HttpMethod::Loop() URI uri = std::string(url); std::string hs = uri.Host; + if (cached.find(hs) != cached.end()) { + _error->Error("Cached Failure"); + Fail(true); + free(url); + FailCounter = 0; + continue; + } + std::string urs = uri; for (;;) { @@ -1172,11 +1228,25 @@ int HttpMethod::Loop() if (UniqueID_ != NULL) CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_); - CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.484")); + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.534")); CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm); CFRelease(hm); +#define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout") +#define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout") +#define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut") + + /*SInt32 to(TimeOut); + CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/ + double to(TimeOut); + CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to)); + + CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm); + CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm); + CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm); + CFRelease(nm); + CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL); CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr); CFRelease(dr); @@ -1193,9 +1263,22 @@ int HttpMethod::Loop() Status("Connecting to %s", hs.c_str()); - if (!CFReadStreamOpen(rs)) { - CfrsError("Open", rs); - Fail(true); + switch (CFReadStreamOpen(rs, to)) { + case -1: + CfrsError("Open", rs); + goto fail; + + case 0: + _error->Error("Host Unreachable"); + cached.insert(hs); + goto fail; + + case 1: + /* success */ + break; + + fail: + Fail(true); goto done; } @@ -1203,6 +1286,7 @@ int HttpMethod::Loop() if (rd == -1) { CfrsError(uri.Host.c_str(), rs); + cached.insert(hs); Fail(true); goto done; } @@ -1284,6 +1368,25 @@ int HttpMethod::Loop() } } + if (sc < 200 || sc >= 300 && sc != 304) { + sr = CFHTTPMessageCopyResponseStatusLine(hm); + + size_t ln = CFStringGetLength(sr) + 1; + char cr[ln]; + + if (!CFStringGetCString(sr, cr, ln, se)) { + Fail(); + goto done; + } + + CFRelease(sr); + + _error->Error("%s", cr); + + Fail(); + goto done_; + } + CFRelease(hm); if (sc == 304) { @@ -1291,9 +1394,7 @@ int HttpMethod::Loop() Res.IMSHit = true; Res.LastModified = Queue->LastModified; URIDone(Res); - } else if (sc < 200 || sc >= 300) - Fail(); - else { + } else { Hashes hash; File = new FileFd(Queue->DestFile, FileFd::WriteAny); diff --git a/methods/http.cc.orig b/methods/http.cc.orig index 2e69a79..b720a07 100644 --- a/methods/http.cc.orig +++ b/methods/http.cc.orig @@ -41,6 +41,7 @@ #include #include #include +#include // Internet stuff #include @@ -682,6 +683,51 @@ bool ServerState::HeaderLine(string Line) } /*}}}*/ +static const CFOptionFlags kNetworkEvents = + kCFStreamEventOpenCompleted | + kCFStreamEventHasBytesAvailable | + kCFStreamEventEndEncountered | + kCFStreamEventErrorOccurred | +0; + +static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) { + switch (event) { + case kCFStreamEventOpenCompleted: + break; + + case kCFStreamEventHasBytesAvailable: + case kCFStreamEventEndEncountered: + *reinterpret_cast(arg) = 1; + CFRunLoopStop(CFRunLoopGetCurrent()); + break; + + case kCFStreamEventErrorOccurred: + *reinterpret_cast(arg) = -1; + CFRunLoopStop(CFRunLoopGetCurrent()); + break; + } +} + +/* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */ +int CFReadStreamOpen(CFReadStreamRef stream, double timeout) { + CFStreamClientContext context; + int value(0); + + memset(&context, 0, sizeof(context)); + context.info = &value; + + if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) { + CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes); + if (CFReadStreamOpen(stream)) + CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false); + else + value = -1; + CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL); + } + + return value; +} + // HttpMethod::SendReq - Send the HTTP request /*{{{*/ // --------------------------------------------------------------------- /* This places the http request in the outbound buffer */ @@ -1095,6 +1141,8 @@ int HttpMethod::Loop() signal(SIGINT,SigTerm); Server = 0; + + std::set cached; int FailCounter = 0; while (1) @@ -1122,6 +1170,14 @@ int HttpMethod::Loop() URI uri = std::string(url); std::string hs = uri.Host; + if (cached.find(hs) != cached.end()) { + _error->Error("Cached Failure"); + Fail(true); + free(url); + FailCounter = 0; + continue; + } + std::string urs = uri; for (;;) { @@ -1168,11 +1224,25 @@ int HttpMethod::Loop() if (UniqueID_ != NULL) CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_); - CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.484")); + CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.534")); CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm); CFRelease(hm); +#define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout") +#define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout") +#define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut") + + /*SInt32 to(TimeOut); + CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/ + double to(TimeOut); + CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to)); + + CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm); + CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm); + CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm); + CFRelease(nm); + CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL); CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr); CFRelease(dr); @@ -1189,9 +1259,22 @@ int HttpMethod::Loop() Status("Connecting to %s", hs.c_str()); - if (!CFReadStreamOpen(rs)) { - CfrsError("Open", rs); - Fail(true); + switch (CFReadStreamOpen(rs, to)) { + case -1: + CfrsError("Open", rs); + goto fail; + + case 0: + _error->Error("Host Unreachable"); + cached.insert(hs); + goto fail; + + case 1: + /* success */ + break; + + fail: + Fail(true); goto done; } @@ -1199,6 +1282,7 @@ int HttpMethod::Loop() if (rd == -1) { CfrsError(uri.Host.c_str(), rs); + cached.insert(hs); Fail(true); goto done; } @@ -1280,6 +1364,25 @@ int HttpMethod::Loop() } } + if (sc < 200 || sc >= 300 && sc != 304) { + sr = CFHTTPMessageCopyResponseStatusLine(hm); + + size_t ln = CFStringGetLength(sr) + 1; + char cr[ln]; + + if (!CFStringGetCString(sr, cr, ln, se)) { + Fail(); + goto done; + } + + CFRelease(sr); + + _error->Error("%s", cr); + + Fail(); + goto done_; + } + CFRelease(hm); if (sc == 304) { @@ -1287,9 +1390,7 @@ int HttpMethod::Loop() Res.IMSHit = true; Res.LastModified = Queue->LastModified; URIDone(Res); - } else if (sc < 200 || sc >= 300) - Fail(); - else { + } else { Hashes hash; File = new FileFd(Queue->DestFile, FileFd::WriteAny); -- 2.47.2