Fixed Dl-Limit (long standing APT bug that they won't acknowledge), specified error...

M cfnetwork.diff
A dllimit.diff
A parallel.diff

git-svn-id: http://svn.telesphoreo.org/trunk@534 514c082c-b64e-11dc-b46d-3d985efe055d

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index fff1b2b6ad379b28603de987933b4523dc5c3f36..b1463dc67fc2f0efa32520ba85c8684192200fd3 100644 (file)
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -238,9 +238,27 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
    /* Single-Instance methods get exactly one queue per URI. This is
       also used for the Access queue method  */
    if (Config->SingleInstance == true || QueueMode == QueueAccess)
-       return U.Access;
+      return U.Access;
+   string name(U.Access + ':' + U.Host);
 
-   return U.Access + ':' + U.Host;
+   int parallel(_config->FindI("Acquire::"+U.Access+"::MaxParallel",0));
+   if (parallel <= 0)
+      return name;
+
+   typedef map<string, int> indexmap;
+   static indexmap indices;
+
+   pair<indexmap::iterator, bool> cache(indices.insert(indexmap::value_type(name, -1)));
+   if (cache.second || cache.first->second == -1) {
+      int &index(indices[U.Access]);
+      if (index >= parallel)
+         index = 0;
+      cache.first->second = index++;
+   }
+
+   ostringstream value;
+   value << U.Access << "::" << cache.first->second;
+   return value.str();
 }
                                                                        /*}}}*/
 // Acquire::GetConfig - Fetch the configuration information            /*{{{*/
@@ -268,7 +286,7 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
       return 0;
 
    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
-   if(_config->FindI("Acquire::"+Access+"::DlLimit",0) > 0)
+   if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
       Conf->SingleInstance = true;
     
    return Conf;

diff --git a/methods/http.cc b/methods/http.cc
index 5d4848e034ab80351a6a29670e25d6d8eb022fcd..7df6f9387fb57def9371c802d7847f613eab76c1 100644 (file)
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -45,6 +45,7 @@ extern "C" {
 #include <string.h>
 #include <iostream>
 #include <apti18n.h>
+#include <set>
 
 // Internet stuff
 #include <netdb.h>
@@ -686,6 +687,51 @@ bool ServerState::HeaderLine(string Line)
 }
                                                                        /*}}}*/
 
+static const CFOptionFlags kNetworkEvents =
+    kCFStreamEventOpenCompleted |
+    kCFStreamEventHasBytesAvailable |
+    kCFStreamEventEndEncountered |
+    kCFStreamEventErrorOccurred |
+0;
+
+static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
+    switch (event) {
+        case kCFStreamEventOpenCompleted:
+        break;
+
+        case kCFStreamEventHasBytesAvailable:
+        case kCFStreamEventEndEncountered:
+            *reinterpret_cast<int *>(arg) = 1;
+            CFRunLoopStop(CFRunLoopGetCurrent());
+        break;
+
+        case kCFStreamEventErrorOccurred:
+            *reinterpret_cast<int *>(arg) = -1;
+            CFRunLoopStop(CFRunLoopGetCurrent());
+        break;
+    }
+}
+
+/* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
+int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
+    CFStreamClientContext context;
+    int value(0);
+
+    memset(&context, 0, sizeof(context));
+    context.info = &value;
+
+    if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
+        CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
+        if (CFReadStreamOpen(stream))
+            CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
+        else
+            value = -1;
+        CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
+    }
+
+    return value;
+}
+
 // HttpMethod::SendReq - Send the HTTP request                         /*{{{*/
 // ---------------------------------------------------------------------
 /* This places the http request in the outbound buffer */
@@ -1099,6 +1145,8 @@ int HttpMethod::Loop()
    signal(SIGINT,SigTerm);
    
    Server = 0;
+
+   std::set<std::string> cached;
    
    int FailCounter = 0;
    while (1)
@@ -1126,6 +1174,14 @@ int HttpMethod::Loop()
       URI uri = std::string(url);
       std::string hs = uri.Host;
 
+      if (cached.find(hs) != cached.end()) {
+         _error->Error("Cached Failure");
+         Fail(true);
+         free(url);
+         FailCounter = 0;
+         continue;
+      }
+
       std::string urs = uri;
 
       for (;;) {
@@ -1172,11 +1228,25 @@ int HttpMethod::Loop()
       if (UniqueID_ != NULL)
          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
 
-      CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.484"));
+      CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.534"));
 
       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
       CFRelease(hm);
 
+#define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
+#define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
+#define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
+
+      /*SInt32 to(TimeOut);
+      CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/
+      double to(TimeOut);
+      CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
+
+      CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
+      CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
+      CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
+      CFRelease(nm);
+
       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
       CFRelease(dr);
@@ -1193,9 +1263,22 @@ int HttpMethod::Loop()
 
       Status("Connecting to %s", hs.c_str());
 
-      if (!CFReadStreamOpen(rs)) {
-         CfrsError("Open", rs);
-         Fail(true);
+      switch (CFReadStreamOpen(rs, to)) {
+         case -1:
+            CfrsError("Open", rs);
+         goto fail;
+
+         case 0:
+            _error->Error("Host Unreachable");
+            cached.insert(hs);
+         goto fail;
+
+         case 1:
+            /* success */
+         break;
+
+         fail:
+            Fail(true);
          goto done;
       }
 
@@ -1203,6 +1286,7 @@ int HttpMethod::Loop()
 
       if (rd == -1) {
          CfrsError(uri.Host.c_str(), rs);
+         cached.insert(hs);
          Fail(true);
          goto done;
       }
@@ -1284,6 +1368,25 @@ int HttpMethod::Loop()
          }
       }
 
+      if (sc < 200 || sc >= 300 && sc != 304) {
+         sr = CFHTTPMessageCopyResponseStatusLine(hm);
+
+         size_t ln = CFStringGetLength(sr) + 1;
+         char cr[ln];
+
+         if (!CFStringGetCString(sr, cr, ln, se)) {
+            Fail();
+            goto done;
+         }
+
+         CFRelease(sr);
+
+         _error->Error("%s", cr);
+
+         Fail();
+         goto done_;
+      }
+
       CFRelease(hm);
 
       if (sc == 304) {
@@ -1291,9 +1394,7 @@ int HttpMethod::Loop()
          Res.IMSHit = true;
          Res.LastModified = Queue->LastModified;
         URIDone(Res);
-      } else if (sc < 200 || sc >= 300)
-        Fail();
-      else {
+      } else {
          Hashes hash;
 
          File = new FileFd(Queue->DestFile, FileFd::WriteAny);

diff --git a/methods/http.cc.orig b/methods/http.cc.orig
index 2e69a79298abab3039534d7d44ae44ca1ae68870..b720a07c6cdf2f9ebdb065f70a3ce51aa576e648 100644 (file)
--- a/methods/http.cc.orig
+++ b/methods/http.cc.orig
@@ -41,6 +41,7 @@
 #include <string.h>
 #include <iostream>
 #include <apti18n.h>
+#include <set>
 
 // Internet stuff
 #include <netdb.h>
@@ -682,6 +683,51 @@ bool ServerState::HeaderLine(string Line)
 }
                                                                        /*}}}*/
 
+static const CFOptionFlags kNetworkEvents =
+    kCFStreamEventOpenCompleted |
+    kCFStreamEventHasBytesAvailable |
+    kCFStreamEventEndEncountered |
+    kCFStreamEventErrorOccurred |
+0;
+
+static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
+    switch (event) {
+        case kCFStreamEventOpenCompleted:
+        break;
+
+        case kCFStreamEventHasBytesAvailable:
+        case kCFStreamEventEndEncountered:
+            *reinterpret_cast<int *>(arg) = 1;
+            CFRunLoopStop(CFRunLoopGetCurrent());
+        break;
+
+        case kCFStreamEventErrorOccurred:
+            *reinterpret_cast<int *>(arg) = -1;
+            CFRunLoopStop(CFRunLoopGetCurrent());
+        break;
+    }
+}
+
+/* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
+int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
+    CFStreamClientContext context;
+    int value(0);
+
+    memset(&context, 0, sizeof(context));
+    context.info = &value;
+
+    if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
+        CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
+        if (CFReadStreamOpen(stream))
+            CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
+        else
+            value = -1;
+        CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
+    }
+
+    return value;
+}
+
 // HttpMethod::SendReq - Send the HTTP request                         /*{{{*/
 // ---------------------------------------------------------------------
 /* This places the http request in the outbound buffer */
@@ -1095,6 +1141,8 @@ int HttpMethod::Loop()
    signal(SIGINT,SigTerm);
    
    Server = 0;
+
+   std::set<std::string> cached;
    
    int FailCounter = 0;
    while (1)
@@ -1122,6 +1170,14 @@ int HttpMethod::Loop()
       URI uri = std::string(url);
       std::string hs = uri.Host;
 
+      if (cached.find(hs) != cached.end()) {
+         _error->Error("Cached Failure");
+         Fail(true);
+         free(url);
+         FailCounter = 0;
+         continue;
+      }
+
       std::string urs = uri;
 
       for (;;) {
@@ -1168,11 +1224,25 @@ int HttpMethod::Loop()
       if (UniqueID_ != NULL)
          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
 
-      CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.484"));
+      CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.534"));
 
       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
       CFRelease(hm);
 
+#define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
+#define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
+#define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
+
+      /*SInt32 to(TimeOut);
+      CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/
+      double to(TimeOut);
+      CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
+
+      CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
+      CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
+      CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
+      CFRelease(nm);
+
       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
       CFRelease(dr);
@@ -1189,9 +1259,22 @@ int HttpMethod::Loop()
 
       Status("Connecting to %s", hs.c_str());
 
-      if (!CFReadStreamOpen(rs)) {
-         CfrsError("Open", rs);
-         Fail(true);
+      switch (CFReadStreamOpen(rs, to)) {
+         case -1:
+            CfrsError("Open", rs);
+         goto fail;
+
+         case 0:
+            _error->Error("Host Unreachable");
+            cached.insert(hs);
+         goto fail;
+
+         case 1:
+            /* success */
+         break;
+
+         fail:
+            Fail(true);
          goto done;
       }
 
@@ -1199,6 +1282,7 @@ int HttpMethod::Loop()
 
       if (rd == -1) {
          CfrsError(uri.Host.c_str(), rs);
+         cached.insert(hs);
          Fail(true);
          goto done;
       }
@@ -1280,6 +1364,25 @@ int HttpMethod::Loop()
          }
       }
 
+      if (sc < 200 || sc >= 300 && sc != 304) {
+         sr = CFHTTPMessageCopyResponseStatusLine(hm);
+
+         size_t ln = CFStringGetLength(sr) + 1;
+         char cr[ln];
+
+         if (!CFStringGetCString(sr, cr, ln, se)) {
+            Fail();
+            goto done;
+         }
+
+         CFRelease(sr);
+
+         _error->Error("%s", cr);
+
+         Fail();
+         goto done_;
+      }
+
       CFRelease(hm);
 
       if (sc == 304) {
@@ -1287,9 +1390,7 @@ int HttpMethod::Loop()
          Res.IMSHit = true;
          Res.LastModified = Queue->LastModified;
         URIDone(Res);
-      } else if (sc < 200 || sc >= 300)
-        Fail();
-      else {
+      } else {
          Hashes hash;
 
          File = new FileFd(Queue->DestFile, FileFd::WriteAny);