methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32 #include <apt-pkg/netrc.h>
  33
  34 #include <sys/sysctl.h>
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42 #include <string.h>
  43 #include <iostream>
  44 #include <map>
  45 #include <set>
  46 #include <apti18n.h>
  47
  48
  49 // Internet stuff
  50 #include <netdb.h>
  51 #include <arpa/inet.h>
  52
  53 #include <dlfcn.h>
  54 #include <lockdown.h>
  55 #include <CoreFoundation/CoreFoundation.h>
  56 #include <CFNetwork/CFNetwork.h>
  57 #include <SystemConfiguration/SystemConfiguration.h>
  58
  59 #include "config.h"
  60 #include "connect.h"
  61 #include "rfc2553emu.h"
  62 #include "http.h"
  63                                                                         /*}}}*/
  64 using namespace std;
  65
  66 CFStringRef Firmware_;
  67 const char *Machine_;
  68 CFStringRef UniqueID_;
  69
  70 void CfrsError(const char *name, CFReadStreamRef rs) {
  71     CFStreamError se = CFReadStreamGetError(rs);
  72
  73     if (se.domain == kCFStreamErrorDomainCustom) {
  74     } else if (se.domain == kCFStreamErrorDomainPOSIX) {
  75         _error->Error("POSIX: %s", strerror(se.error));
  76     } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
  77         _error->Error("MacOSStatus: %ld", se.error);
  78     } else if (se.domain == kCFStreamErrorDomainNetDB) {
  79         _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
  80     } else if (se.domain == kCFStreamErrorDomainMach) {
  81         _error->Error("Mach: %ld", se.error);
  82     } else if (se.domain == kCFStreamErrorDomainHTTP) {
  83         switch (se.error) {
  84             case kCFStreamErrorHTTPParseFailure:
  85                 _error->Error("Parse failure");
  86             break;
  87
  88             case kCFStreamErrorHTTPRedirectionLoop:
  89                 _error->Error("Redirection loop");
  90             break;
  91
  92             case kCFStreamErrorHTTPBadURL:
  93                 _error->Error("Bad URL");
  94             break;
  95
  96             default:
  97                 _error->Error("Unknown HTTP error: %ld", se.error);
  98             break;
  99         }
 100     } else if (se.domain == kCFStreamErrorDomainSOCKS) {
 101         _error->Error("SOCKS: %ld", se.error);
 102     } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
 103         _error->Error("SystemConfiguration: %ld", se.error);
 104     } else if (se.domain == kCFStreamErrorDomainSSL) {
 105         _error->Error("SSL: %ld", se.error);
 106     } else {
 107         _error->Error("Domain #%ld: %ld", se.domain, se.error);
 108     }
 109 }
 110
 111 string HttpMethod::FailFile;
 112 int HttpMethod::FailFd = -1;
 113 time_t HttpMethod::FailTime = 0;
 114 unsigned long PipelineDepth = 10;
 115 unsigned long TimeOut = 120;
 116 bool AllowRedirect = false;
 117 bool Debug = false;
 118 URI Proxy;
 119
 120 static const CFOptionFlags kNetworkEvents =
 121     kCFStreamEventOpenCompleted |
 122     kCFStreamEventHasBytesAvailable |
 123     kCFStreamEventEndEncountered |
 124     kCFStreamEventErrorOccurred |
 125 0;
 126
 127 static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
 128     switch (event) {
 129         case kCFStreamEventOpenCompleted:
 130         break;
 131
 132         case kCFStreamEventHasBytesAvailable:
 133         case kCFStreamEventEndEncountered:
 134             *reinterpret_cast<int *>(arg) = 1;
 135             CFRunLoopStop(CFRunLoopGetCurrent());
 136         break;
 137
 138         case kCFStreamEventErrorOccurred:
 139             *reinterpret_cast<int *>(arg) = -1;
 140             CFRunLoopStop(CFRunLoopGetCurrent());
 141         break;
 142     }
 143 }
 144
 145 /* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
 146 int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
 147     CFStreamClientContext context;
 148     int value(0);
 149
 150     memset(&context, 0, sizeof(context));
 151     context.info = &value;
 152
 153     if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
 154         CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
 155         if (CFReadStreamOpen(stream))
 156             CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
 157         else
 158             value = -1;
 159         CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
 160     }
 161
 162     return value;
 163 }
 164
 165 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* This closes and timestamps the open file. This is neccessary to get
 168    resume behavoir on user abort */
 169 void HttpMethod::SigTerm(int)
 170 {
 171    if (FailFd == -1)
 172       _exit(100);
 173    close(FailFd);
 174
 175    // Timestamp
 176    struct utimbuf UBuf;
 177    UBuf.actime = FailTime;
 178    UBuf.modtime = FailTime;
 179    utime(FailFile.c_str(),&UBuf);
 180
 181    _exit(100);
 182 }
 183                                                                         /*}}}*/
 184 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 185 // ---------------------------------------------------------------------
 186 /* We stash the desired pipeline depth */
 187 bool HttpMethod::Configuration(string Message)
 188 {
 189    if (pkgAcqMethod::Configuration(Message) == false)
 190       return false;
 191
 192    AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
 193    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 194    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 195                                   PipelineDepth);
 196    Debug = _config->FindB("Debug::Acquire::http",false);
 197
 198    return true;
 199 }
 200                                                                         /*}}}*/
 201 // HttpMethod::Loop - Main loop                                         /*{{{*/
 202 // ---------------------------------------------------------------------
 203 /* */
 204 int HttpMethod::Loop()
 205 {
 206    typedef vector<string> StringVector;
 207    typedef vector<string>::iterator StringVectorIterator;
 208    map<string, StringVector> Redirected;
 209
 210    signal(SIGTERM,SigTerm);
 211    signal(SIGINT,SigTerm);
 212
 213    std::set<std::string> cached;
 214
 215    int FailCounter = 0;
 216    while (1)
 217    {
 218       // We have no commands, wait for some to arrive
 219       if (Queue == 0)
 220       {
 221          if (WaitFd(STDIN_FILENO) == false)
 222             return 0;
 223       }
 224
 225       /* Run messages, we can accept 0 (no message) if we didn't
 226          do a WaitFd above.. Otherwise the FD is closed. */
 227       int Result = Run(true);
 228       if (Result != -1 && (Result != 0 || Queue == 0))
 229          return 100;
 230
 231       if (Queue == 0)
 232          continue;
 233
 234       CFStringEncoding se = kCFStringEncodingUTF8;
 235
 236       char *url = strdup(Queue->Uri.c_str());
 237     url:
 238       URI uri = std::string(url);
 239       std::string hs = uri.Host;
 240
 241       if (cached.find(hs) != cached.end()) {
 242          _error->Error("Cached Failure");
 243          Fail(true);
 244          free(url);
 245          FailCounter = 0;
 246          continue;
 247       }
 248
 249       std::string urs = uri;
 250
 251       for (;;) {
 252          size_t bad = urs.find_first_of("+");
 253          if (bad == std::string::npos)
 254             break;
 255          // XXX: generalize
 256          urs = urs.substr(0, bad) + "%2b" + urs.substr(bad + 1);
 257       }
 258
 259       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
 260       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
 261       CFRelease(sr);
 262       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
 263       CFRelease(ur);
 264
 265       struct stat SBuf;
 266       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
 267          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
 268          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
 269          CFRelease(sr);
 270
 271          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
 272          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
 273          CFRelease(sr);
 274
 275          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
 276       } else if (Queue->LastModified != 0) {
 277          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified).c_str(), se);
 278          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
 279          CFRelease(sr);
 280
 281          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
 282       } else
 283          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("max-age=0"));
 284
 285       if (Firmware_ != NULL)
 286          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_);
 287
 288       sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
 289       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
 290       CFRelease(sr);
 291
 292       if (UniqueID_ != NULL)
 293          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
 294
 295       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.592"));
 296
 297       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
 298       CFRelease(hm);
 299
 300 #define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
 301 #define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
 302 #define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
 303
 304       /*SInt32 to(TimeOut);
 305       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/
 306       double to(TimeOut);
 307       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
 308
 309       CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
 310       CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
 311       CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
 312       CFRelease(nm);
 313
 314       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
 315       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
 316       CFRelease(dr);
 317
 318       //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
 319       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
 320
 321       FetchResult Res;
 322       CFIndex rd;
 323       UInt32 sc;
 324
 325       uint8_t data[10240];
 326       size_t offset = 0;
 327
 328       Status("Connecting to %s", hs.c_str());
 329
 330       switch (CFReadStreamOpen(rs, to)) {
 331          case -1:
 332             CfrsError("Open", rs);
 333          goto fail;
 334
 335          case 0:
 336             _error->Error("Host Unreachable");
 337             cached.insert(hs);
 338          goto fail;
 339
 340          case 1:
 341             /* success */
 342          break;
 343
 344          fail:
 345             Fail(true);
 346          goto done;
 347       }
 348
 349       rd = CFReadStreamRead(rs, data, sizeof(data));
 350
 351       if (rd == -1) {
 352          CfrsError(uri.Host.c_str(), rs);
 353          cached.insert(hs);
 354          Fail(true);
 355          goto done;
 356       }
 357
 358       Res.Filename = Queue->DestFile;
 359
 360       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
 361       sc = CFHTTPMessageGetResponseStatusCode(hm);
 362
 363       if (sc == 301 || sc == 302) {
 364          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
 365          if (sr == NULL) {
 366             Fail();
 367             goto done_;
 368          } else {
 369             size_t ln = CFStringGetLength(sr) + 1;
 370             free(url);
 371             url = static_cast<char *>(malloc(ln));
 372
 373             if (!CFStringGetCString(sr, url, ln, se)) {
 374                Fail();
 375                goto done_;
 376             }
 377
 378             CFRelease(sr);
 379             goto url;
 380          }
 381       }
 382
 383       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
 384       if (sr != NULL) {
 385          size_t ln = CFStringGetLength(sr) + 1;
 386          char cr[ln];
 387
 388          if (!CFStringGetCString(sr, cr, ln, se)) {
 389             Fail();
 390             goto done_;
 391          }
 392
 393          CFRelease(sr);
 394
 395          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
 396             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 397             Fail();
 398             goto done_;
 399          }
 400
 401          if (offset > Res.Size) {
 402             _error->Error(_("This HTTP server has broken range support"));
 403             Fail();
 404             goto done_;
 405          }
 406       } else {
 407          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
 408          if (sr != NULL) {
 409             Res.Size = CFStringGetIntValue(sr);
 410             CFRelease(sr);
 411          }
 412       }
 413
 414       time(&Res.LastModified);
 415
 416       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
 417       if (sr != NULL) {
 418          size_t ln = CFStringGetLength(sr) + 1;
 419          char cr[ln];
 420
 421          if (!CFStringGetCString(sr, cr, ln, se)) {
 422             Fail();
 423             goto done_;
 424          }
 425
 426          CFRelease(sr);
 427
 428          if (!StrToTime(cr, Res.LastModified)) {
 429             _error->Error(_("Unknown date format"));
 430             Fail();
 431             goto done_;
 432          }
 433       }
 434
 435       if (sc < 200 || sc >= 300 && sc != 304) {
 436          sr = CFHTTPMessageCopyResponseStatusLine(hm);
 437
 438          size_t ln = CFStringGetLength(sr) + 1;
 439          char cr[ln];
 440
 441          if (!CFStringGetCString(sr, cr, ln, se)) {
 442             Fail();
 443             goto done;
 444          }
 445
 446          CFRelease(sr);
 447
 448          _error->Error("%s", cr);
 449
 450          Fail();
 451          goto done_;
 452       }
 453
 454       CFRelease(hm);
 455
 456       if (sc == 304) {
 457          unlink(Queue->DestFile.c_str());
 458          Res.IMSHit = true;
 459          Res.LastModified = Queue->LastModified;
 460          URIDone(Res);
 461       } else {
 462          Hashes hash;
 463
 464          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
 465          if (_error->PendingError() == true) {
 466             delete File;
 467             File = NULL;
 468             Fail();
 469             goto done;
 470          }
 471
 472          FailFile = Queue->DestFile;
 473          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 474          FailFd = File->Fd();
 475          FailTime = Res.LastModified;
 476
 477          Res.ResumePoint = offset;
 478          ftruncate(File->Fd(), offset);
 479
 480          if (offset != 0) {
 481             lseek(File->Fd(), 0, SEEK_SET);
 482             if (!hash.AddFD(File->Fd(), offset)) {
 483                _error->Errno("read", _("Problem hashing file"));
 484                delete File;
 485                File = NULL;
 486                Fail();
 487                goto done;
 488             }
 489          }
 490
 491          lseek(File->Fd(), 0, SEEK_END);
 492
 493          URIStart(Res);
 494
 495          read: if (rd == -1) {
 496             CfrsError("rd", rs);
 497             Fail(true);
 498          } else if (rd == 0) {
 499             if (Res.Size == 0)
 500                Res.Size = File->Size();
 501
 502             struct utimbuf UBuf;
 503             time(&UBuf.actime);
 504             UBuf.actime = Res.LastModified;
 505             UBuf.modtime = Res.LastModified;
 506             utime(Queue->DestFile.c_str(), &UBuf);
 507
 508             Res.TakeHashes(hash);
 509             URIDone(Res);
 510          } else {
 511             hash.Add(data, rd);
 512
 513             uint8_t *dt = data;
 514             while (rd != 0) {
 515                int sz = write(File->Fd(), dt, rd);
 516
 517                if (sz == -1) {
 518                    delete File;
 519                    File = NULL;
 520                    Fail();
 521                    goto done;
 522                }
 523
 524                dt += sz;
 525                rd -= sz;
 526             }
 527
 528             rd = CFReadStreamRead(rs, data, sizeof(data));
 529             goto read;
 530          }
 531       }
 532
 533      goto done;
 534     done_:
 535       CFRelease(hm);
 536     done:
 537       CFReadStreamClose(rs);
 538       CFRelease(rs);
 539       free(url);
 540
 541       FailCounter = 0;
 542    }
 543
 544    return 0;
 545 }
 546                                                                         /*}}}*/
 547
 548 int main()
 549 {
 550    setlocale(LC_ALL, "");
 551    // ignore SIGPIPE, this can happen on write() if the socket
 552    // closes the connection (this is dealt with via ServerDie())
 553    signal(SIGPIPE, SIG_IGN);
 554
 555    HttpMethod Mth;
 556
 557     size_t size;
 558     sysctlbyname("hw.machine", NULL, &size, NULL, 0);
 559     char *machine = new char[size];
 560     sysctlbyname("hw.machine", machine, &size, NULL, 0);
 561     Machine_ = machine;
 562
 563     const char *path = "/System/Library/CoreServices/SystemVersion.plist";
 564     CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false);
 565
 566     CFPropertyListRef plist; {
 567         CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url);
 568         CFReadStreamOpen(stream);
 569         plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL);
 570         CFReadStreamClose(stream);
 571     }
 572
 573     CFRelease(url);
 574
 575     if (plist != NULL) {
 576         Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion")));
 577         CFRelease(plist);
 578     }
 579
 580     if (UniqueID_ == NULL)
 581     if (void *libMobileGestalt = dlopen("/usr/lib/libMobileGestalt.dylib", RTLD_GLOBAL | RTLD_LAZY))
 582     if (CFStringRef (*$MGCopyAnswer)(CFStringRef) = (CFStringRef (*)(CFStringRef)) dlsym(libMobileGestalt, "MGCopyAnswer"))
 583         UniqueID_ = $MGCopyAnswer(CFSTR("UniqueDeviceID"));
 584
 585     if (UniqueID_ == NULL)
 586     if (void *lockdown = lockdown_connect()) {
 587         UniqueID_ = lockdown_copy_value(lockdown, NULL, kLockdownUniqueDeviceIDKey);
 588         lockdown_disconnect(lockdown);
 589     }
 590
 591    return Mth.Loop();
 592 }