methods/http.cc.orig

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/sysctl.h>
  34 #include <sys/stat.h>
  35 #include <sys/time.h>
  36 #include <utime.h>
  37 #include <unistd.h>
  38 #include <signal.h>
  39 #include <stdio.h>
  40 #include <errno.h>
  41 #include <string.h>
  42 #include <iostream>
  43 #include <apti18n.h>
  44 #include <set>
  45
  46 // Internet stuff
  47 #include <netdb.h>
  48 #include <arpa/inet.h>
  49
  50 #include <lockdown.h>
  51 #include <CoreFoundation/CoreFoundation.h>
  52 #include <CoreServices/CoreServices.h>
  53 #include <SystemConfiguration/SystemConfiguration.h>
  54
  55 #include "connect.h"
  56 #include "rfc2553emu.h"
  57 #include "http.h"
  58
  59                                                                         /*}}}*/
  60 using namespace std;
  61
  62 CFStringRef Firmware_;
  63 const char *Machine_;
  64 CFStringRef UniqueID_;
  65
  66 void CfrsError(const char *name, CFReadStreamRef rs) {
  67     CFStreamError se = CFReadStreamGetError(rs);
  68
  69     if (se.domain == kCFStreamErrorDomainCustom) {
  70     } else if (se.domain == kCFStreamErrorDomainPOSIX) {
  71         _error->Error("POSIX: %s", strerror(se.error));
  72     } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
  73         _error->Error("MacOSStatus: %ld", se.error);
  74     } else if (se.domain == kCFStreamErrorDomainNetDB) {
  75         _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
  76     } else if (se.domain == kCFStreamErrorDomainMach) {
  77         _error->Error("Mach: %ld", se.error);
  78     } else if (se.domain == kCFStreamErrorDomainHTTP) {
  79         switch (se.error) {
  80             case kCFStreamErrorHTTPParseFailure:
  81                 _error->Error("Parse failure");
  82             break;
  83
  84             case kCFStreamErrorHTTPRedirectionLoop:
  85                 _error->Error("Redirection loop");
  86             break;
  87
  88             case kCFStreamErrorHTTPBadURL:
  89                 _error->Error("Bad URL");
  90             break;
  91
  92             default:
  93                 _error->Error("Unknown HTTP error: %ld", se.error);
  94             break;
  95         }
  96     } else if (se.domain == kCFStreamErrorDomainSOCKS) {
  97         _error->Error("SOCKS: %ld", se.error);
  98     } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
  99         _error->Error("SystemConfiguration: %ld", se.error);
 100     } else if (se.domain == kCFStreamErrorDomainSSL) {
 101         _error->Error("SSL: %ld", se.error);
 102     } else {
 103         _error->Error("Domain #%ld: %ld", se.domain, se.error);
 104     }
 105 }
 106
 107 string HttpMethod::FailFile;
 108 int HttpMethod::FailFd = -1;
 109 time_t HttpMethod::FailTime = 0;
 110 unsigned long PipelineDepth = 10;
 111 unsigned long TimeOut = 120;
 112 bool Debug = false;
 113
 114 unsigned long CircleBuf::BwReadLimit=0;
 115 unsigned long CircleBuf::BwTickReadData=0;
 116 struct timeval CircleBuf::BwReadTick={0,0};
 117 const unsigned int CircleBuf::BW_HZ=10;
 118
 119 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* */
 122 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
 123 {
 124    Buf = new unsigned char[Size];
 125    Reset();
 126
 127    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
 128 }
 129                                                                         /*}}}*/
 130 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* */
 133 void CircleBuf::Reset()
 134 {
 135    InP = 0;
 136    OutP = 0;
 137    StrPos = 0;
 138    MaxGet = (unsigned int)-1;
 139    OutQueue = string();
 140    if (Hash != 0)
 141    {
 142       delete Hash;
 143       Hash = new Hashes;
 144    }
 145 };
 146                                                                         /*}}}*/
 147 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
 148 // ---------------------------------------------------------------------
 149 /* This fills up the buffer with as much data as is in the FD, assuming it
 150    is non-blocking.. */
 151 bool CircleBuf::Read(int Fd)
 152 {
 153    unsigned long BwReadMax;
 154
 155    while (1)
 156    {
 157       // Woops, buffer is full
 158       if (InP - OutP == Size)
 159          return true;
 160
 161       // what's left to read in this tick
 162       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 163
 164       if(CircleBuf::BwReadLimit) {
 165          struct timeval now;
 166          gettimeofday(&now,0);
 167
 168          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 169             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 170          if(d > 1000000/BW_HZ) {
 171             CircleBuf::BwReadTick = now;
 172             CircleBuf::BwTickReadData = 0;
 173          }
 174
 175          if(CircleBuf::BwTickReadData >= BwReadMax) {
 176             usleep(1000000/BW_HZ);
 177             return true;
 178          }
 179       }
 180
 181       // Write the buffer segment
 182       int Res;
 183       if(CircleBuf::BwReadLimit) {
 184          Res = read(Fd,Buf + (InP%Size),
 185                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 186       } else
 187          Res = read(Fd,Buf + (InP%Size),LeftRead());
 188
 189       if(Res > 0 && BwReadLimit > 0)
 190          CircleBuf::BwTickReadData += Res;
 191
 192       if (Res == 0)
 193          return false;
 194       if (Res < 0)
 195       {
 196          if (errno == EAGAIN)
 197             return true;
 198          return false;
 199       }
 200
 201       if (InP == 0)
 202          gettimeofday(&Start,0);
 203       InP += Res;
 204    }
 205 }
 206                                                                         /*}}}*/
 207 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 208 // ---------------------------------------------------------------------
 209 /* This will hold the string in and fill the buffer with it as it empties */
 210 bool CircleBuf::Read(string Data)
 211 {
 212    OutQueue += Data;
 213    FillOut();
 214    return true;
 215 }
 216                                                                         /*}}}*/
 217 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 218 // ---------------------------------------------------------------------
 219 /* */
 220 void CircleBuf::FillOut()
 221 {
 222    if (OutQueue.empty() == true)
 223       return;
 224    while (1)
 225    {
 226       // Woops, buffer is full
 227       if (InP - OutP == Size)
 228          return;
 229
 230       // Write the buffer segment
 231       unsigned long Sz = LeftRead();
 232       if (OutQueue.length() - StrPos < Sz)
 233          Sz = OutQueue.length() - StrPos;
 234       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 235
 236       // Advance
 237       StrPos += Sz;
 238       InP += Sz;
 239       if (OutQueue.length() == StrPos)
 240       {
 241          StrPos = 0;
 242          OutQueue = "";
 243          return;
 244       }
 245    }
 246 }
 247                                                                         /*}}}*/
 248 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* This empties the buffer into the FD. */
 251 bool CircleBuf::Write(int Fd)
 252 {
 253    while (1)
 254    {
 255       FillOut();
 256
 257       // Woops, buffer is empty
 258       if (OutP == InP)
 259          return true;
 260
 261       if (OutP == MaxGet)
 262          return true;
 263
 264       // Write the buffer segment
 265       int Res;
 266       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 267
 268       if (Res == 0)
 269          return false;
 270       if (Res < 0)
 271       {
 272          if (errno == EAGAIN)
 273             return true;
 274
 275          return false;
 276       }
 277
 278       if (Hash != 0)
 279          Hash->Add(Buf + (OutP%Size),Res);
 280
 281       OutP += Res;
 282    }
 283 }
 284                                                                         /*}}}*/
 285 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 286 // ---------------------------------------------------------------------
 287 /* This copies till the first empty line */
 288 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 289 {
 290    // We cheat and assume it is unneeded to have more than one buffer load
 291    for (unsigned long I = OutP; I < InP; I++)
 292    {
 293       if (Buf[I%Size] != '\n')
 294          continue;
 295       ++I;
 296
 297       if (Single == false)
 298       {
 299          if (I < InP  && Buf[I%Size] == '\r')
 300             ++I;
 301          if (I >= InP || Buf[I%Size] != '\n')
 302             continue;
 303          ++I;
 304       }
 305
 306       Data = "";
 307       while (OutP < I)
 308       {
 309          unsigned long Sz = LeftWrite();
 310          if (Sz == 0)
 311             return false;
 312          if (I - OutP < Sz)
 313             Sz = I - OutP;
 314          Data += string((char *)(Buf + (OutP%Size)),Sz);
 315          OutP += Sz;
 316       }
 317       return true;
 318    }
 319    return false;
 320 }
 321                                                                         /*}}}*/
 322 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 323 // ---------------------------------------------------------------------
 324 /* */
 325 void CircleBuf::Stats()
 326 {
 327    if (InP == 0)
 328       return;
 329
 330    struct timeval Stop;
 331    gettimeofday(&Stop,0);
 332 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 333              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 334    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 335 }
 336                                                                         /*}}}*/
 337
 338 // ServerState::ServerState - Constructor                               /*{{{*/
 339 // ---------------------------------------------------------------------
 340 /* */
 341 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 342                         In(64*1024), Out(4*1024),
 343                         ServerName(Srv)
 344 {
 345    Reset();
 346 }
 347                                                                         /*}}}*/
 348 // ServerState::Open - Open a connection to the server                  /*{{{*/
 349 // ---------------------------------------------------------------------
 350 /* This opens a connection to the server. */
 351 bool ServerState::Open()
 352 {
 353    // Use the already open connection if possible.
 354    if (ServerFd != -1)
 355       return true;
 356
 357    Close();
 358    In.Reset();
 359    Out.Reset();
 360    Persistent = true;
 361
 362    // Determine the proxy setting
 363    if (getenv("http_proxy") == 0)
 364    {
 365       string DefProxy = _config->Find("Acquire::http::Proxy");
 366       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 367       if (SpecificProxy.empty() == false)
 368       {
 369          if (SpecificProxy == "DIRECT")
 370             Proxy = "";
 371          else
 372             Proxy = SpecificProxy;
 373       }
 374       else
 375          Proxy = DefProxy;
 376    }
 377    else
 378       Proxy = getenv("http_proxy");
 379
 380    // Parse no_proxy, a , separated list of domains
 381    if (getenv("no_proxy") != 0)
 382    {
 383       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 384          Proxy = "";
 385    }
 386
 387    // Determine what host and port to use based on the proxy settings
 388    int Port = 0;
 389    string Host;
 390    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 391    {
 392       if (ServerName.Port != 0)
 393          Port = ServerName.Port;
 394       Host = ServerName.Host;
 395    }
 396    else
 397    {
 398       if (Proxy.Port != 0)
 399          Port = Proxy.Port;
 400       Host = Proxy.Host;
 401    }
 402
 403    // Connect to the remote server
 404    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 405       return false;
 406
 407    return true;
 408 }
 409                                                                         /*}}}*/
 410 // ServerState::Close - Close a connection to the server                /*{{{*/
 411 // ---------------------------------------------------------------------
 412 /* */
 413 bool ServerState::Close()
 414 {
 415    close(ServerFd);
 416    ServerFd = -1;
 417    return true;
 418 }
 419                                                                         /*}}}*/
 420 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 421 // ---------------------------------------------------------------------
 422 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 423    parse error occured */
 424 int ServerState::RunHeaders()
 425 {
 426    State = Header;
 427
 428    Owner->Status(_("Waiting for headers"));
 429
 430    Major = 0;
 431    Minor = 0;
 432    Result = 0;
 433    Size = 0;
 434    StartPos = 0;
 435    Encoding = Closes;
 436    HaveContent = false;
 437    time(&Date);
 438
 439    do
 440    {
 441       string Data;
 442       if (In.WriteTillEl(Data) == false)
 443          continue;
 444
 445       if (Debug == true)
 446          clog << Data;
 447
 448       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 449       {
 450          string::const_iterator J = I;
 451          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 452          if (HeaderLine(string(I,J)) == false)
 453             return 2;
 454          I = J;
 455       }
 456
 457       // 100 Continue is a Nop...
 458       if (Result == 100)
 459          continue;
 460
 461       // Tidy up the connection persistance state.
 462       if (Encoding == Closes && HaveContent == true)
 463          Persistent = false;
 464
 465       return 0;
 466    }
 467    while (Owner->Go(false,this) == true);
 468
 469    return 1;
 470 }
 471                                                                         /*}}}*/
 472 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 bool ServerState::RunData()
 476 {
 477    State = Data;
 478
 479    // Chunked transfer encoding is fun..
 480    if (Encoding == Chunked)
 481    {
 482       while (1)
 483       {
 484          // Grab the block size
 485          bool Last = true;
 486          string Data;
 487          In.Limit(-1);
 488          do
 489          {
 490             if (In.WriteTillEl(Data,true) == true)
 491                break;
 492          }
 493          while ((Last = Owner->Go(false,this)) == true);
 494
 495          if (Last == false)
 496             return false;
 497
 498          // See if we are done
 499          unsigned long Len = strtol(Data.c_str(),0,16);
 500          if (Len == 0)
 501          {
 502             In.Limit(-1);
 503
 504             // We have to remove the entity trailer
 505             Last = true;
 506             do
 507             {
 508                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 509                   break;
 510             }
 511             while ((Last = Owner->Go(false,this)) == true);
 512             if (Last == false)
 513                return false;
 514             return !_error->PendingError();
 515          }
 516
 517          // Transfer the block
 518          In.Limit(Len);
 519          while (Owner->Go(true,this) == true)
 520             if (In.IsLimit() == true)
 521                break;
 522
 523          // Error
 524          if (In.IsLimit() == false)
 525             return false;
 526
 527          // The server sends an extra new line before the next block specifier..
 528          In.Limit(-1);
 529          Last = true;
 530          do
 531          {
 532             if (In.WriteTillEl(Data,true) == true)
 533                break;
 534          }
 535          while ((Last = Owner->Go(false,this)) == true);
 536          if (Last == false)
 537             return false;
 538       }
 539    }
 540    else
 541    {
 542       /* Closes encoding is used when the server did not specify a size, the
 543          loss of the connection means we are done */
 544       if (Encoding == Closes)
 545          In.Limit(-1);
 546       else
 547          In.Limit(Size - StartPos);
 548
 549       // Just transfer the whole block.
 550       do
 551       {
 552          if (In.IsLimit() == false)
 553             continue;
 554
 555          In.Limit(-1);
 556          return !_error->PendingError();
 557       }
 558       while (Owner->Go(true,this) == true);
 559    }
 560
 561    return Owner->Flush(this) && !_error->PendingError();
 562 }
 563                                                                         /*}}}*/
 564 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 565 // ---------------------------------------------------------------------
 566 /* */
 567 bool ServerState::HeaderLine(string Line)
 568 {
 569    if (Line.empty() == true)
 570       return true;
 571
 572    // The http server might be trying to do something evil.
 573    if (Line.length() >= MAXLEN)
 574       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 575
 576    string::size_type Pos = Line.find(' ');
 577    if (Pos == string::npos || Pos+1 > Line.length())
 578    {
 579       // Blah, some servers use "connection:closes", evil.
 580       Pos = Line.find(':');
 581       if (Pos == string::npos || Pos + 2 > Line.length())
 582          return _error->Error(_("Bad header line"));
 583       Pos++;
 584    }
 585
 586    // Parse off any trailing spaces between the : and the next word.
 587    string::size_type Pos2 = Pos;
 588    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 589       Pos2++;
 590
 591    string Tag = string(Line,0,Pos);
 592    string Val = string(Line,Pos2);
 593
 594    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 595    {
 596       // Evil servers return no version
 597       if (Line[4] == '/')
 598       {
 599          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 600                     &Result,Code) != 4)
 601             return _error->Error(_("The HTTP server sent an invalid reply header"));
 602       }
 603       else
 604       {
 605          Major = 0;
 606          Minor = 9;
 607          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 608             return _error->Error(_("The HTTP server sent an invalid reply header"));
 609       }
 610
 611       /* Check the HTTP response header to get the default persistance
 612          state. */
 613       if (Major < 1)
 614          Persistent = false;
 615       else
 616       {
 617          if (Major == 1 && Minor <= 0)
 618             Persistent = false;
 619          else
 620             Persistent = true;
 621       }
 622
 623       return true;
 624    }
 625
 626    if (stringcasecmp(Tag,"Content-Length:") == 0)
 627    {
 628       if (Encoding == Closes)
 629          Encoding = Stream;
 630       HaveContent = true;
 631
 632       // The length is already set from the Content-Range header
 633       if (StartPos != 0)
 634          return true;
 635
 636       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 637          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 638       return true;
 639    }
 640
 641    if (stringcasecmp(Tag,"Content-Type:") == 0)
 642    {
 643       HaveContent = true;
 644       return true;
 645    }
 646
 647    if (stringcasecmp(Tag,"Content-Range:") == 0)
 648    {
 649       HaveContent = true;
 650
 651       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 652          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 653       if ((unsigned)StartPos > Size)
 654          return _error->Error(_("This HTTP server has broken range support"));
 655       return true;
 656    }
 657
 658    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 659    {
 660       HaveContent = true;
 661       if (stringcasecmp(Val,"chunked") == 0)
 662          Encoding = Chunked;
 663       return true;
 664    }
 665
 666    if (stringcasecmp(Tag,"Connection:") == 0)
 667    {
 668       if (stringcasecmp(Val,"close") == 0)
 669          Persistent = false;
 670       if (stringcasecmp(Val,"keep-alive") == 0)
 671          Persistent = true;
 672       return true;
 673    }
 674
 675    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 676    {
 677       if (StrToTime(Val,Date) == false)
 678          return _error->Error(_("Unknown date format"));
 679       return true;
 680    }
 681
 682    return true;
 683 }
 684                                                                         /*}}}*/
 685
 686 static const CFOptionFlags kNetworkEvents =
 687     kCFStreamEventOpenCompleted |
 688     kCFStreamEventHasBytesAvailable |
 689     kCFStreamEventEndEncountered |
 690     kCFStreamEventErrorOccurred |
 691 0;
 692
 693 static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
 694     switch (event) {
 695         case kCFStreamEventOpenCompleted:
 696         break;
 697
 698         case kCFStreamEventHasBytesAvailable:
 699         case kCFStreamEventEndEncountered:
 700             *reinterpret_cast<int *>(arg) = 1;
 701             CFRunLoopStop(CFRunLoopGetCurrent());
 702         break;
 703
 704         case kCFStreamEventErrorOccurred:
 705             *reinterpret_cast<int *>(arg) = -1;
 706             CFRunLoopStop(CFRunLoopGetCurrent());
 707         break;
 708     }
 709 }
 710
 711 /* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
 712 int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
 713     CFStreamClientContext context;
 714     int value(0);
 715
 716     memset(&context, 0, sizeof(context));
 717     context.info = &value;
 718
 719     if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
 720         CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
 721         if (CFReadStreamOpen(stream))
 722             CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
 723         else
 724             value = -1;
 725         CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
 726     }
 727
 728     return value;
 729 }
 730
 731 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 732 // ---------------------------------------------------------------------
 733 /* This places the http request in the outbound buffer */
 734 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 735 {
 736    URI Uri = Itm->Uri;
 737
 738    // The HTTP server expects a hostname with a trailing :port
 739    char Buf[1000];
 740    string ProperHost = Uri.Host;
 741    if (Uri.Port != 0)
 742    {
 743       sprintf(Buf,":%u",Uri.Port);
 744       ProperHost += Buf;
 745    }
 746
 747    // Just in case.
 748    if (Itm->Uri.length() >= sizeof(Buf))
 749        abort();
 750
 751    /* Build the request. We include a keep-alive header only for non-proxy
 752       requests. This is to tweak old http/1.0 servers that do support keep-alive
 753       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 754       will glitch HTTP/1.0 proxies because they do not filter it out and
 755       pass it on, HTTP/1.1 says the connection should default to keep alive
 756       and we expect the proxy to do this */
 757    if (Proxy.empty() == true || Proxy.Host.empty())
 758       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 759               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 760    else
 761    {
 762       /* Generate a cache control header if necessary. We place a max
 763          cache age on index files, optionally set a no-cache directive
 764          and a no-store directive for archives. */
 765       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 766               Itm->Uri.c_str(),ProperHost.c_str());
 767       // only generate a cache control header if we actually want to
 768       // use a cache
 769       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 770       {
 771          if (Itm->IndexFile == true)
 772             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 773                     _config->FindI("Acquire::http::Max-Age",0));
 774          else
 775          {
 776             if (_config->FindB("Acquire::http::No-Store",false) == true)
 777                strcat(Buf,"Cache-Control: no-store\r\n");
 778          }
 779       }
 780    }
 781    // generate a no-cache header if needed
 782    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 783       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 784
 785
 786    string Req = Buf;
 787
 788    // Check for a partial file
 789    struct stat SBuf;
 790    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 791    {
 792       // In this case we send an if-range query with a range header
 793       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 794               TimeRFC1123(SBuf.st_mtime).c_str());
 795       Req += Buf;
 796    }
 797    else
 798    {
 799       if (Itm->LastModified != 0)
 800       {
 801          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 802          Req += Buf;
 803       }
 804    }
 805
 806    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 807       Req += string("Proxy-Authorization: Basic ") +
 808           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 809
 810    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 811       Req += string("Authorization: Basic ") +
 812           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 813
 814    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 815
 816    if (Debug == true)
 817       cerr << Req << endl;
 818
 819    Out.Read(Req);
 820 }
 821                                                                         /*}}}*/
 822 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 823 // ---------------------------------------------------------------------
 824 /* This runs the select loop over the server FDs, Output file FDs and
 825    stdin. */
 826 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 827 {
 828    // Server has closed the connection
 829    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 830                                ToFile == false))
 831       return false;
 832
 833    fd_set rfds,wfds;
 834    FD_ZERO(&rfds);
 835    FD_ZERO(&wfds);
 836
 837    /* Add the server. We only send more requests if the connection will
 838       be persisting */
 839    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 840        && Srv->Persistent == true)
 841       FD_SET(Srv->ServerFd,&wfds);
 842    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 843       FD_SET(Srv->ServerFd,&rfds);
 844
 845    // Add the file
 846    int FileFD = -1;
 847    if (File != 0)
 848       FileFD = File->Fd();
 849
 850    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 851       FD_SET(FileFD,&wfds);
 852
 853    // Add stdin
 854    FD_SET(STDIN_FILENO,&rfds);
 855
 856    // Figure out the max fd
 857    int MaxFd = FileFD;
 858    if (MaxFd < Srv->ServerFd)
 859       MaxFd = Srv->ServerFd;
 860
 861    // Select
 862    struct timeval tv;
 863    tv.tv_sec = TimeOut;
 864    tv.tv_usec = 0;
 865    int Res = 0;
 866    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 867    {
 868       if (errno == EINTR)
 869          return true;
 870       return _error->Errno("select",_("Select failed"));
 871    }
 872
 873    if (Res == 0)
 874    {
 875       _error->Error(_("Connection timed out"));
 876       return ServerDie(Srv);
 877    }
 878
 879    // Handle server IO
 880    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 881    {
 882       errno = 0;
 883       if (Srv->In.Read(Srv->ServerFd) == false)
 884          return ServerDie(Srv);
 885    }
 886
 887    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 888    {
 889       errno = 0;
 890       if (Srv->Out.Write(Srv->ServerFd) == false)
 891          return ServerDie(Srv);
 892    }
 893
 894    // Send data to the file
 895    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 896    {
 897       if (Srv->In.Write(FileFD) == false)
 898          return _error->Errno("write",_("Error writing to output file"));
 899    }
 900
 901    // Handle commands from APT
 902    if (FD_ISSET(STDIN_FILENO,&rfds))
 903    {
 904       if (Run(true) != -1)
 905          exit(100);
 906    }
 907
 908    return true;
 909 }
 910                                                                         /*}}}*/
 911 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 912 // ---------------------------------------------------------------------
 913 /* This takes the current input buffer from the Server FD and writes it
 914    into the file */
 915 bool HttpMethod::Flush(ServerState *Srv)
 916 {
 917    if (File != 0)
 918    {
 919       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 920       // can't be set
 921       if (File->Name() != "/dev/null")
 922          SetNonBlock(File->Fd(),false);
 923       if (Srv->In.WriteSpace() == false)
 924          return true;
 925
 926       while (Srv->In.WriteSpace() == true)
 927       {
 928          if (Srv->In.Write(File->Fd()) == false)
 929             return _error->Errno("write",_("Error writing to file"));
 930          if (Srv->In.IsLimit() == true)
 931             return true;
 932       }
 933
 934       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 935          return true;
 936    }
 937    return false;
 938 }
 939                                                                         /*}}}*/
 940 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 941 // ---------------------------------------------------------------------
 942 /* */
 943 bool HttpMethod::ServerDie(ServerState *Srv)
 944 {
 945    unsigned int LErrno = errno;
 946
 947    // Dump the buffer to the file
 948    if (Srv->State == ServerState::Data)
 949    {
 950       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 951       // can't be set
 952       if (File->Name() != "/dev/null")
 953          SetNonBlock(File->Fd(),false);
 954       while (Srv->In.WriteSpace() == true)
 955       {
 956          if (Srv->In.Write(File->Fd()) == false)
 957             return _error->Errno("write",_("Error writing to the file"));
 958
 959          // Done
 960          if (Srv->In.IsLimit() == true)
 961             return true;
 962       }
 963    }
 964
 965    // See if this is because the server finished the data stream
 966    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 967        Srv->Encoding != ServerState::Closes)
 968    {
 969       Srv->Close();
 970       if (LErrno == 0)
 971          return _error->Error(_("Error reading from server. Remote end closed connection"));
 972       errno = LErrno;
 973       return _error->Errno("read",_("Error reading from server"));
 974    }
 975    else
 976    {
 977       Srv->In.Limit(-1);
 978
 979       // Nothing left in the buffer
 980       if (Srv->In.WriteSpace() == false)
 981          return false;
 982
 983       // We may have got multiple responses back in one packet..
 984       Srv->Close();
 985       return true;
 986    }
 987
 988    return false;
 989 }
 990                                                                         /*}}}*/
 991 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 992 // ---------------------------------------------------------------------
 993 /* We look at the header data we got back from the server and decide what
 994    to do. Returns
 995      0 - File is open,
 996      1 - IMS hit
 997      3 - Unrecoverable error
 998      4 - Error with error content page
 999      5 - Unrecoverable non-server error (close the connection) */
1000 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
1001 {
1002    // Not Modified
1003    if (Srv->Result == 304)
1004    {
1005       unlink(Queue->DestFile.c_str());
1006       Res.IMSHit = true;
1007       Res.LastModified = Queue->LastModified;
1008       return 1;
1009    }
1010
1011    /* We have a reply we dont handle. This should indicate a perm server
1012       failure */
1013    if (Srv->Result < 200 || Srv->Result >= 300)
1014    {
1015       _error->Error("%u %s",Srv->Result,Srv->Code);
1016       if (Srv->HaveContent == true)
1017          return 4;
1018       return 3;
1019    }
1020
1021    // This is some sort of 2xx 'data follows' reply
1022    Res.LastModified = Srv->Date;
1023    Res.Size = Srv->Size;
1024
1025    // Open the file
1026    delete File;
1027    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
1028    if (_error->PendingError() == true)
1029       return 5;
1030
1031    FailFile = Queue->DestFile;
1032    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1033    FailFd = File->Fd();
1034    FailTime = Srv->Date;
1035
1036    // Set the expected size
1037    if (Srv->StartPos >= 0)
1038    {
1039       Res.ResumePoint = Srv->StartPos;
1040       ftruncate(File->Fd(),Srv->StartPos);
1041    }
1042
1043    // Set the start point
1044    lseek(File->Fd(),0,SEEK_END);
1045
1046    delete Srv->In.Hash;
1047    Srv->In.Hash = new Hashes;
1048
1049    // Fill the Hash if the file is non-empty (resume)
1050    if (Srv->StartPos > 0)
1051    {
1052       lseek(File->Fd(),0,SEEK_SET);
1053       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1054       {
1055          _error->Errno("read",_("Problem hashing file"));
1056          return 5;
1057       }
1058       lseek(File->Fd(),0,SEEK_END);
1059    }
1060
1061    SetNonBlock(File->Fd(),true);
1062    return 0;
1063 }
1064                                                                         /*}}}*/
1065 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
1066 // ---------------------------------------------------------------------
1067 /* This closes and timestamps the open file. This is neccessary to get
1068    resume behavoir on user abort */
1069 void HttpMethod::SigTerm(int)
1070 {
1071    if (FailFd == -1)
1072       _exit(100);
1073    close(FailFd);
1074
1075    // Timestamp
1076    struct utimbuf UBuf;
1077    UBuf.actime = FailTime;
1078    UBuf.modtime = FailTime;
1079    utime(FailFile.c_str(),&UBuf);
1080
1081    _exit(100);
1082 }
1083                                                                         /*}}}*/
1084 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
1085 // ---------------------------------------------------------------------
1086 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1087    depth. */
1088 bool HttpMethod::Fetch(FetchItem *)
1089 {
1090    if (Server == 0)
1091       return true;
1092
1093    // Queue the requests
1094    int Depth = -1;
1095    bool Tail = false;
1096    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1097         I = I->Next, Depth++)
1098    {
1099       // If pipelining is disabled, we only queue 1 request
1100       if (Server->Pipeline == false && Depth >= 0)
1101          break;
1102
1103       // Make sure we stick with the same server
1104       if (Server->Comp(I->Uri) == false)
1105          break;
1106       if (QueueBack == I)
1107          Tail = true;
1108       if (Tail == true)
1109       {
1110          QueueBack = I->Next;
1111          SendReq(I,Server->Out);
1112          continue;
1113       }
1114    }
1115
1116    return true;
1117 };
1118                                                                         /*}}}*/
1119 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1120 // ---------------------------------------------------------------------
1121 /* We stash the desired pipeline depth */
1122 bool HttpMethod::Configuration(string Message)
1123 {
1124    if (pkgAcqMethod::Configuration(Message) == false)
1125       return false;
1126
1127    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1128    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1129                                   PipelineDepth);
1130    Debug = _config->FindB("Debug::Acquire::http",false);
1131
1132    return true;
1133 }
1134                                                                         /*}}}*/
1135 // HttpMethod::Loop - Main loop                                         /*{{{*/
1136 // ---------------------------------------------------------------------
1137 /* */
1138 int HttpMethod::Loop()
1139 {
1140    signal(SIGTERM,SigTerm);
1141    signal(SIGINT,SigTerm);
1142
1143    Server = 0;
1144
1145    std::set<std::string> cached;
1146
1147    int FailCounter = 0;
1148    while (1)
1149    {
1150       // We have no commands, wait for some to arrive
1151       if (Queue == 0)
1152       {
1153          if (WaitFd(STDIN_FILENO) == false)
1154             return 0;
1155       }
1156
1157       /* Run messages, we can accept 0 (no message) if we didn't
1158          do a WaitFd above.. Otherwise the FD is closed. */
1159       int Result = Run(true);
1160       if (Result != -1 && (Result != 0 || Queue == 0))
1161          return 100;
1162
1163       if (Queue == 0)
1164          continue;
1165
1166       CFStringEncoding se = kCFStringEncodingUTF8;
1167
1168       char *url = strdup(Queue->Uri.c_str());
1169     url:
1170       URI uri = std::string(url);
1171       std::string hs = uri.Host;
1172
1173       if (cached.find(hs) != cached.end()) {
1174          _error->Error("Cached Failure");
1175          Fail(true);
1176          free(url);
1177          FailCounter = 0;
1178          continue;
1179       }
1180
1181       std::string urs = uri;
1182
1183       for (;;) {
1184          size_t bad = urs.find_first_of("+");
1185          if (bad == std::string::npos)
1186             break;
1187          // XXX: generalize
1188          urs = urs.substr(0, bad) + "%2b" + urs.substr(bad + 1);
1189       }
1190
1191       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
1192       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1193       CFRelease(sr);
1194       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1195       CFRelease(ur);
1196
1197       struct stat SBuf;
1198       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1199          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1200          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1201          CFRelease(sr);
1202
1203          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1204          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1205          CFRelease(sr);
1206
1207          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
1208       } else if (Queue->LastModified != 0) {
1209          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified).c_str(), se);
1210          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1211          CFRelease(sr);
1212
1213          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
1214       } else
1215          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("max-age=0"));
1216
1217       if (Firmware_ != NULL)
1218          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_);
1219
1220       sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
1221       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
1222       CFRelease(sr);
1223
1224       if (UniqueID_ != NULL)
1225          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
1226
1227       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.534"));
1228
1229       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1230       CFRelease(hm);
1231
1232 #define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
1233 #define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
1234 #define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
1235
1236       /*SInt32 to(TimeOut);
1237       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/
1238       double to(TimeOut);
1239       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
1240
1241       CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
1242       CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
1243       CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
1244       CFRelease(nm);
1245
1246       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1247       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1248       CFRelease(dr);
1249
1250       //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1251       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1252
1253       FetchResult Res;
1254       CFIndex rd;
1255       UInt32 sc;
1256
1257       uint8_t data[10240];
1258       size_t offset = 0;
1259
1260       Status("Connecting to %s", hs.c_str());
1261
1262       switch (CFReadStreamOpen(rs, to)) {
1263          case -1:
1264             CfrsError("Open", rs);
1265          goto fail;
1266
1267          case 0:
1268             _error->Error("Host Unreachable");
1269             cached.insert(hs);
1270          goto fail;
1271
1272          case 1:
1273             /* success */
1274          break;
1275
1276          fail:
1277             Fail(true);
1278          goto done;
1279       }
1280
1281       rd = CFReadStreamRead(rs, data, sizeof(data));
1282
1283       if (rd == -1) {
1284          CfrsError(uri.Host.c_str(), rs);
1285          cached.insert(hs);
1286          Fail(true);
1287          goto done;
1288       }
1289
1290       Res.Filename = Queue->DestFile;
1291
1292       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1293       sc = CFHTTPMessageGetResponseStatusCode(hm);
1294
1295       if (sc == 301 || sc == 302) {
1296          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1297          if (sr == NULL) {
1298             Fail();
1299             goto done_;
1300          } else {
1301             size_t ln = CFStringGetLength(sr) + 1;
1302             free(url);
1303             url = static_cast<char *>(malloc(ln));
1304
1305             if (!CFStringGetCString(sr, url, ln, se)) {
1306                Fail();
1307                goto done_;
1308             }
1309
1310             CFRelease(sr);
1311             goto url;
1312          }
1313       }
1314
1315       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1316       if (sr != NULL) {
1317          size_t ln = CFStringGetLength(sr) + 1;
1318          char cr[ln];
1319
1320          if (!CFStringGetCString(sr, cr, ln, se)) {
1321             Fail();
1322             goto done_;
1323          }
1324
1325          CFRelease(sr);
1326
1327          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1328             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1329             Fail();
1330             goto done_;
1331          }
1332
1333          if (offset > Res.Size) {
1334             _error->Error(_("This HTTP server has broken range support"));
1335             Fail();
1336             goto done_;
1337          }
1338       } else {
1339          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1340          if (sr != NULL) {
1341             Res.Size = CFStringGetIntValue(sr);
1342             CFRelease(sr);
1343          }
1344       }
1345
1346       time(&Res.LastModified);
1347
1348       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1349       if (sr != NULL) {
1350          size_t ln = CFStringGetLength(sr) + 1;
1351          char cr[ln];
1352
1353          if (!CFStringGetCString(sr, cr, ln, se)) {
1354             Fail();
1355             goto done_;
1356          }
1357
1358          CFRelease(sr);
1359
1360          if (!StrToTime(cr, Res.LastModified)) {
1361             _error->Error(_("Unknown date format"));
1362             Fail();
1363             goto done_;
1364          }
1365       }
1366
1367       if (sc < 200 || sc >= 300 && sc != 304) {
1368          sr = CFHTTPMessageCopyResponseStatusLine(hm);
1369
1370          size_t ln = CFStringGetLength(sr) + 1;
1371          char cr[ln];
1372
1373          if (!CFStringGetCString(sr, cr, ln, se)) {
1374             Fail();
1375             goto done;
1376          }
1377
1378          CFRelease(sr);
1379
1380          _error->Error("%s", cr);
1381
1382          Fail();
1383          goto done_;
1384       }
1385
1386       CFRelease(hm);
1387
1388       if (sc == 304) {
1389          unlink(Queue->DestFile.c_str());
1390          Res.IMSHit = true;
1391          Res.LastModified = Queue->LastModified;
1392          URIDone(Res);
1393       } else {
1394          Hashes hash;
1395
1396          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1397          if (_error->PendingError() == true) {
1398             delete File;
1399             File = NULL;
1400             Fail();
1401             goto done;
1402          }
1403
1404          FailFile = Queue->DestFile;
1405          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1406          FailFd = File->Fd();
1407          FailTime = Res.LastModified;
1408
1409          Res.ResumePoint = offset;
1410          ftruncate(File->Fd(), offset);
1411
1412          if (offset != 0) {
1413             lseek(File->Fd(), 0, SEEK_SET);
1414             if (!hash.AddFD(File->Fd(), offset)) {
1415                _error->Errno("read", _("Problem hashing file"));
1416                delete File;
1417                File = NULL;
1418                Fail();
1419                goto done;
1420             }
1421          }
1422
1423          lseek(File->Fd(), 0, SEEK_END);
1424
1425          URIStart(Res);
1426
1427          read: if (rd == -1) {
1428             CfrsError("rd", rs);
1429             Fail(true);
1430          } else if (rd == 0) {
1431             if (Res.Size == 0)
1432                Res.Size = File->Size();
1433
1434             struct utimbuf UBuf;
1435             time(&UBuf.actime);
1436             UBuf.actime = Res.LastModified;
1437             UBuf.modtime = Res.LastModified;
1438             utime(Queue->DestFile.c_str(), &UBuf);
1439
1440             Res.TakeHashes(hash);
1441             URIDone(Res);
1442          } else {
1443             hash.Add(data, rd);
1444
1445             uint8_t *dt = data;
1446             while (rd != 0) {
1447                int sz = write(File->Fd(), dt, rd);
1448
1449                if (sz == -1) {
1450                    delete File;
1451                    File = NULL;
1452                    Fail();
1453                    goto done;
1454                }
1455
1456                dt += sz;
1457                rd -= sz;
1458             }
1459
1460             rd = CFReadStreamRead(rs, data, sizeof(data));
1461             goto read;
1462          }
1463       }
1464
1465      goto done;
1466     done_:
1467       CFRelease(hm);
1468     done:
1469       CFReadStreamClose(rs);
1470       CFRelease(rs);
1471       free(url);
1472
1473       FailCounter = 0;
1474    }
1475
1476    return 0;
1477 }
1478                                                                         /*}}}*/
1479
1480 int main()
1481 {
1482    setlocale(LC_ALL, "");
1483
1484    HttpMethod Mth;
1485
1486     size_t size;
1487     sysctlbyname("hw.machine", NULL, &size, NULL, 0);
1488     char *machine = new char[size];
1489     sysctlbyname("hw.machine", machine, &size, NULL, 0);
1490     Machine_ = machine;
1491
1492     const char *path = "/System/Library/CoreServices/SystemVersion.plist";
1493     CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false);
1494
1495     CFPropertyListRef plist; {
1496         CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url);
1497         CFReadStreamOpen(stream);
1498         plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL);
1499         CFReadStreamClose(stream);
1500     }
1501
1502     CFRelease(url);
1503
1504     if (plist != NULL) {
1505         Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion")));
1506         CFRelease(plist);
1507     }
1508
1509     if (void *lockdown = lockdown_connect()) {
1510         UniqueID_ = lockdown_copy_value(lockdown, NULL, kLockdownUniqueDeviceIDKey);
1511         lockdown_disconnect(lockdown);
1512     }
1513
1514    return Mth.Loop();
1515 }
1516
1517