methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/sysctl.h>
  34 #include <sys/stat.h>
  35 #include <sys/time.h>
  36 #include <utime.h>
  37 #include <unistd.h>
  38 #include <signal.h>
  39 #include <stdio.h>
  40 #include <errno.h>
  41 #include <string.h>
  42 #include <iostream>
  43 #include <apti18n.h>
  44 #include <set>
  45
  46 // Internet stuff
  47 #include <netdb.h>
  48 #include <arpa/inet.h>
  49
  50 #include <lockdown.h>
  51 #include <CoreFoundation/CoreFoundation.h>
  52 #include <CoreServices/CoreServices.h>
  53 #include <SystemConfiguration/SystemConfiguration.h>
  54
  55 #include "config.h"
  56 #include "connect.h"
  57 #include "rfc2553emu.h"
  58 #include "http.h"
  59
  60                                                                         /*}}}*/
  61 using namespace std;
  62
  63 CFStringRef Firmware_;
  64 const char *Machine_;
  65 CFStringRef UniqueID_;
  66
  67 void CfrsError(const char *name, CFReadStreamRef rs) {
  68     CFStreamError se = CFReadStreamGetError(rs);
  69
  70     if (se.domain == kCFStreamErrorDomainCustom) {
  71     } else if (se.domain == kCFStreamErrorDomainPOSIX) {
  72         _error->Error("POSIX: %s", strerror(se.error));
  73     } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
  74         _error->Error("MacOSStatus: %ld", se.error);
  75     } else if (se.domain == kCFStreamErrorDomainNetDB) {
  76         _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
  77     } else if (se.domain == kCFStreamErrorDomainMach) {
  78         _error->Error("Mach: %ld", se.error);
  79     } else if (se.domain == kCFStreamErrorDomainHTTP) {
  80         switch (se.error) {
  81             case kCFStreamErrorHTTPParseFailure:
  82                 _error->Error("Parse failure");
  83             break;
  84
  85             case kCFStreamErrorHTTPRedirectionLoop:
  86                 _error->Error("Redirection loop");
  87             break;
  88
  89             case kCFStreamErrorHTTPBadURL:
  90                 _error->Error("Bad URL");
  91             break;
  92
  93             default:
  94                 _error->Error("Unknown HTTP error: %ld", se.error);
  95             break;
  96         }
  97     } else if (se.domain == kCFStreamErrorDomainSOCKS) {
  98         _error->Error("SOCKS: %ld", se.error);
  99     } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
 100         _error->Error("SystemConfiguration: %ld", se.error);
 101     } else if (se.domain == kCFStreamErrorDomainSSL) {
 102         _error->Error("SSL: %ld", se.error);
 103     } else {
 104         _error->Error("Domain #%ld: %ld", se.domain, se.error);
 105     }
 106 }
 107
 108 string HttpMethod::FailFile;
 109 int HttpMethod::FailFd = -1;
 110 time_t HttpMethod::FailTime = 0;
 111 unsigned long PipelineDepth = 10;
 112 unsigned long TimeOut = 120;
 113 bool Debug = false;
 114 URI Proxy;
 115
 116 unsigned long CircleBuf::BwReadLimit=0;
 117 unsigned long CircleBuf::BwTickReadData=0;
 118 struct timeval CircleBuf::BwReadTick={0,0};
 119 const unsigned int CircleBuf::BW_HZ=10;
 120
 121 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
 122 // ---------------------------------------------------------------------
 123 /* */
 124 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
 125 {
 126    Buf = new unsigned char[Size];
 127    Reset();
 128
 129    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
 130 }
 131                                                                         /*}}}*/
 132 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
 133 // ---------------------------------------------------------------------
 134 /* */
 135 void CircleBuf::Reset()
 136 {
 137    InP = 0;
 138    OutP = 0;
 139    StrPos = 0;
 140    MaxGet = (unsigned int)-1;
 141    OutQueue = string();
 142    if (Hash != 0)
 143    {
 144       delete Hash;
 145       Hash = new Hashes;
 146    }
 147 };
 148                                                                         /*}}}*/
 149 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
 150 // ---------------------------------------------------------------------
 151 /* This fills up the buffer with as much data as is in the FD, assuming it
 152    is non-blocking.. */
 153 bool CircleBuf::Read(int Fd)
 154 {
 155    unsigned long BwReadMax;
 156
 157    while (1)
 158    {
 159       // Woops, buffer is full
 160       if (InP - OutP == Size)
 161          return true;
 162
 163       // what's left to read in this tick
 164       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 165
 166       if(CircleBuf::BwReadLimit) {
 167          struct timeval now;
 168          gettimeofday(&now,0);
 169
 170          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 171             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 172          if(d > 1000000/BW_HZ) {
 173             CircleBuf::BwReadTick = now;
 174             CircleBuf::BwTickReadData = 0;
 175          }
 176
 177          if(CircleBuf::BwTickReadData >= BwReadMax) {
 178             usleep(1000000/BW_HZ);
 179             return true;
 180          }
 181       }
 182
 183       // Write the buffer segment
 184       int Res;
 185       if(CircleBuf::BwReadLimit) {
 186          Res = read(Fd,Buf + (InP%Size),
 187                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 188       } else
 189          Res = read(Fd,Buf + (InP%Size),LeftRead());
 190
 191       if(Res > 0 && BwReadLimit > 0)
 192          CircleBuf::BwTickReadData += Res;
 193
 194       if (Res == 0)
 195          return false;
 196       if (Res < 0)
 197       {
 198          if (errno == EAGAIN)
 199             return true;
 200          return false;
 201       }
 202
 203       if (InP == 0)
 204          gettimeofday(&Start,0);
 205       InP += Res;
 206    }
 207 }
 208                                                                         /*}}}*/
 209 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 210 // ---------------------------------------------------------------------
 211 /* This will hold the string in and fill the buffer with it as it empties */
 212 bool CircleBuf::Read(string Data)
 213 {
 214    OutQueue += Data;
 215    FillOut();
 216    return true;
 217 }
 218                                                                         /*}}}*/
 219 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 220 // ---------------------------------------------------------------------
 221 /* */
 222 void CircleBuf::FillOut()
 223 {
 224    if (OutQueue.empty() == true)
 225       return;
 226    while (1)
 227    {
 228       // Woops, buffer is full
 229       if (InP - OutP == Size)
 230          return;
 231
 232       // Write the buffer segment
 233       unsigned long Sz = LeftRead();
 234       if (OutQueue.length() - StrPos < Sz)
 235          Sz = OutQueue.length() - StrPos;
 236       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 237
 238       // Advance
 239       StrPos += Sz;
 240       InP += Sz;
 241       if (OutQueue.length() == StrPos)
 242       {
 243          StrPos = 0;
 244          OutQueue = "";
 245          return;
 246       }
 247    }
 248 }
 249                                                                         /*}}}*/
 250 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 251 // ---------------------------------------------------------------------
 252 /* This empties the buffer into the FD. */
 253 bool CircleBuf::Write(int Fd)
 254 {
 255    while (1)
 256    {
 257       FillOut();
 258
 259       // Woops, buffer is empty
 260       if (OutP == InP)
 261          return true;
 262
 263       if (OutP == MaxGet)
 264          return true;
 265
 266       // Write the buffer segment
 267       int Res;
 268       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 269
 270       if (Res == 0)
 271          return false;
 272       if (Res < 0)
 273       {
 274          if (errno == EAGAIN)
 275             return true;
 276
 277          return false;
 278       }
 279
 280       if (Hash != 0)
 281          Hash->Add(Buf + (OutP%Size),Res);
 282
 283       OutP += Res;
 284    }
 285 }
 286                                                                         /*}}}*/
 287 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 288 // ---------------------------------------------------------------------
 289 /* This copies till the first empty line */
 290 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 291 {
 292    // We cheat and assume it is unneeded to have more than one buffer load
 293    for (unsigned long I = OutP; I < InP; I++)
 294    {
 295       if (Buf[I%Size] != '\n')
 296          continue;
 297       ++I;
 298
 299       if (Single == false)
 300       {
 301          if (I < InP  && Buf[I%Size] == '\r')
 302             ++I;
 303          if (I >= InP || Buf[I%Size] != '\n')
 304             continue;
 305          ++I;
 306       }
 307
 308       Data = "";
 309       while (OutP < I)
 310       {
 311          unsigned long Sz = LeftWrite();
 312          if (Sz == 0)
 313             return false;
 314          if (I - OutP < Sz)
 315             Sz = I - OutP;
 316          Data += string((char *)(Buf + (OutP%Size)),Sz);
 317          OutP += Sz;
 318       }
 319       return true;
 320    }
 321    return false;
 322 }
 323                                                                         /*}}}*/
 324 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* */
 327 void CircleBuf::Stats()
 328 {
 329    if (InP == 0)
 330       return;
 331
 332    struct timeval Stop;
 333    gettimeofday(&Stop,0);
 334 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 335              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 336    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 337 }
 338                                                                         /*}}}*/
 339
 340 // ServerState::ServerState - Constructor                               /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* */
 343 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 344                         In(64*1024), Out(4*1024),
 345                         ServerName(Srv)
 346 {
 347    Reset();
 348 }
 349                                                                         /*}}}*/
 350 // ServerState::Open - Open a connection to the server                  /*{{{*/
 351 // ---------------------------------------------------------------------
 352 /* This opens a connection to the server. */
 353 bool ServerState::Open()
 354 {
 355    // Use the already open connection if possible.
 356    if (ServerFd != -1)
 357       return true;
 358
 359    Close();
 360    In.Reset();
 361    Out.Reset();
 362    Persistent = true;
 363
 364    // Determine the proxy setting
 365    if (getenv("http_proxy") == 0)
 366    {
 367       string DefProxy = _config->Find("Acquire::http::Proxy");
 368       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 369       if (SpecificProxy.empty() == false)
 370       {
 371          if (SpecificProxy == "DIRECT")
 372             Proxy = "";
 373          else
 374             Proxy = SpecificProxy;
 375       }
 376       else
 377          Proxy = DefProxy;
 378    }
 379    else
 380       Proxy = getenv("http_proxy");
 381
 382    // Parse no_proxy, a , separated list of domains
 383    if (getenv("no_proxy") != 0)
 384    {
 385       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 386          Proxy = "";
 387    }
 388
 389    // Determine what host and port to use based on the proxy settings
 390    int Port = 0;
 391    string Host;
 392    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 393    {
 394       if (ServerName.Port != 0)
 395          Port = ServerName.Port;
 396       Host = ServerName.Host;
 397    }
 398    else
 399    {
 400       if (Proxy.Port != 0)
 401          Port = Proxy.Port;
 402       Host = Proxy.Host;
 403    }
 404
 405    // Connect to the remote server
 406    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 407       return false;
 408
 409    return true;
 410 }
 411                                                                         /*}}}*/
 412 // ServerState::Close - Close a connection to the server                /*{{{*/
 413 // ---------------------------------------------------------------------
 414 /* */
 415 bool ServerState::Close()
 416 {
 417    close(ServerFd);
 418    ServerFd = -1;
 419    return true;
 420 }
 421                                                                         /*}}}*/
 422 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 423 // ---------------------------------------------------------------------
 424 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
 425    parse error occurred */
 426 int ServerState::RunHeaders()
 427 {
 428    State = Header;
 429
 430    Owner->Status(_("Waiting for headers"));
 431
 432    Major = 0;
 433    Minor = 0;
 434    Result = 0;
 435    Size = 0;
 436    StartPos = 0;
 437    Encoding = Closes;
 438    HaveContent = false;
 439    time(&Date);
 440
 441    do
 442    {
 443       string Data;
 444       if (In.WriteTillEl(Data) == false)
 445          continue;
 446
 447       if (Debug == true)
 448          clog << Data;
 449
 450       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 451       {
 452          string::const_iterator J = I;
 453          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 454          if (HeaderLine(string(I,J)) == false)
 455             return 2;
 456          I = J;
 457       }
 458
 459       // 100 Continue is a Nop...
 460       if (Result == 100)
 461          continue;
 462
 463       // Tidy up the connection persistance state.
 464       if (Encoding == Closes && HaveContent == true)
 465          Persistent = false;
 466
 467       return 0;
 468    }
 469    while (Owner->Go(false,this) == true);
 470
 471    return 1;
 472 }
 473                                                                         /*}}}*/
 474 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 475 // ---------------------------------------------------------------------
 476 /* */
 477 bool ServerState::RunData()
 478 {
 479    State = Data;
 480
 481    // Chunked transfer encoding is fun..
 482    if (Encoding == Chunked)
 483    {
 484       while (1)
 485       {
 486          // Grab the block size
 487          bool Last = true;
 488          string Data;
 489          In.Limit(-1);
 490          do
 491          {
 492             if (In.WriteTillEl(Data,true) == true)
 493                break;
 494          }
 495          while ((Last = Owner->Go(false,this)) == true);
 496
 497          if (Last == false)
 498             return false;
 499
 500          // See if we are done
 501          unsigned long Len = strtol(Data.c_str(),0,16);
 502          if (Len == 0)
 503          {
 504             In.Limit(-1);
 505
 506             // We have to remove the entity trailer
 507             Last = true;
 508             do
 509             {
 510                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 511                   break;
 512             }
 513             while ((Last = Owner->Go(false,this)) == true);
 514             if (Last == false)
 515                return false;
 516             return !_error->PendingError();
 517          }
 518
 519          // Transfer the block
 520          In.Limit(Len);
 521          while (Owner->Go(true,this) == true)
 522             if (In.IsLimit() == true)
 523                break;
 524
 525          // Error
 526          if (In.IsLimit() == false)
 527             return false;
 528
 529          // The server sends an extra new line before the next block specifier..
 530          In.Limit(-1);
 531          Last = true;
 532          do
 533          {
 534             if (In.WriteTillEl(Data,true) == true)
 535                break;
 536          }
 537          while ((Last = Owner->Go(false,this)) == true);
 538          if (Last == false)
 539             return false;
 540       }
 541    }
 542    else
 543    {
 544       /* Closes encoding is used when the server did not specify a size, the
 545          loss of the connection means we are done */
 546       if (Encoding == Closes)
 547          In.Limit(-1);
 548       else
 549          In.Limit(Size - StartPos);
 550
 551       // Just transfer the whole block.
 552       do
 553       {
 554          if (In.IsLimit() == false)
 555             continue;
 556
 557          In.Limit(-1);
 558          return !_error->PendingError();
 559       }
 560       while (Owner->Go(true,this) == true);
 561    }
 562
 563    return Owner->Flush(this) && !_error->PendingError();
 564 }
 565                                                                         /*}}}*/
 566 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 567 // ---------------------------------------------------------------------
 568 /* */
 569 bool ServerState::HeaderLine(string Line)
 570 {
 571    if (Line.empty() == true)
 572       return true;
 573
 574    // The http server might be trying to do something evil.
 575    if (Line.length() >= MAXLEN)
 576       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 577
 578    string::size_type Pos = Line.find(' ');
 579    if (Pos == string::npos || Pos+1 > Line.length())
 580    {
 581       // Blah, some servers use "connection:closes", evil.
 582       Pos = Line.find(':');
 583       if (Pos == string::npos || Pos + 2 > Line.length())
 584          return _error->Error(_("Bad header line"));
 585       Pos++;
 586    }
 587
 588    // Parse off any trailing spaces between the : and the next word.
 589    string::size_type Pos2 = Pos;
 590    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 591       Pos2++;
 592
 593    string Tag = string(Line,0,Pos);
 594    string Val = string(Line,Pos2);
 595
 596    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 597    {
 598       // Evil servers return no version
 599       if (Line[4] == '/')
 600       {
 601          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 602                     &Result,Code) != 4)
 603             return _error->Error(_("The HTTP server sent an invalid reply header"));
 604       }
 605       else
 606       {
 607          Major = 0;
 608          Minor = 9;
 609          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 610             return _error->Error(_("The HTTP server sent an invalid reply header"));
 611       }
 612
 613       /* Check the HTTP response header to get the default persistance
 614          state. */
 615       if (Major < 1)
 616          Persistent = false;
 617       else
 618       {
 619          if (Major == 1 && Minor <= 0)
 620             Persistent = false;
 621          else
 622             Persistent = true;
 623       }
 624
 625       return true;
 626    }
 627
 628    if (stringcasecmp(Tag,"Content-Length:") == 0)
 629    {
 630       if (Encoding == Closes)
 631          Encoding = Stream;
 632       HaveContent = true;
 633
 634       // The length is already set from the Content-Range header
 635       if (StartPos != 0)
 636          return true;
 637
 638       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 639          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 640       return true;
 641    }
 642
 643    if (stringcasecmp(Tag,"Content-Type:") == 0)
 644    {
 645       HaveContent = true;
 646       return true;
 647    }
 648
 649    if (stringcasecmp(Tag,"Content-Range:") == 0)
 650    {
 651       HaveContent = true;
 652
 653       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 654          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 655       if ((unsigned)StartPos > Size)
 656          return _error->Error(_("This HTTP server has broken range support"));
 657       return true;
 658    }
 659
 660    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 661    {
 662       HaveContent = true;
 663       if (stringcasecmp(Val,"chunked") == 0)
 664          Encoding = Chunked;
 665       return true;
 666    }
 667
 668    if (stringcasecmp(Tag,"Connection:") == 0)
 669    {
 670       if (stringcasecmp(Val,"close") == 0)
 671          Persistent = false;
 672       if (stringcasecmp(Val,"keep-alive") == 0)
 673          Persistent = true;
 674       return true;
 675    }
 676
 677    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 678    {
 679       if (StrToTime(Val,Date) == false)
 680          return _error->Error(_("Unknown date format"));
 681       return true;
 682    }
 683
 684    return true;
 685 }
 686                                                                         /*}}}*/
 687
 688 static const CFOptionFlags kNetworkEvents =
 689     kCFStreamEventOpenCompleted |
 690     kCFStreamEventHasBytesAvailable |
 691     kCFStreamEventEndEncountered |
 692     kCFStreamEventErrorOccurred |
 693 0;
 694
 695 static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
 696     switch (event) {
 697         case kCFStreamEventOpenCompleted:
 698         break;
 699
 700         case kCFStreamEventHasBytesAvailable:
 701         case kCFStreamEventEndEncountered:
 702             *reinterpret_cast<int *>(arg) = 1;
 703             CFRunLoopStop(CFRunLoopGetCurrent());
 704         break;
 705
 706         case kCFStreamEventErrorOccurred:
 707             *reinterpret_cast<int *>(arg) = -1;
 708             CFRunLoopStop(CFRunLoopGetCurrent());
 709         break;
 710     }
 711 }
 712
 713 /* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
 714 int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
 715     CFStreamClientContext context;
 716     int value(0);
 717
 718     memset(&context, 0, sizeof(context));
 719     context.info = &value;
 720
 721     if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
 722         CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
 723         if (CFReadStreamOpen(stream))
 724             CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
 725         else
 726             value = -1;
 727         CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
 728     }
 729
 730     return value;
 731 }
 732
 733 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 734 // ---------------------------------------------------------------------
 735 /* This places the http request in the outbound buffer */
 736 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 737 {
 738    URI Uri = Itm->Uri;
 739
 740    // The HTTP server expects a hostname with a trailing :port
 741    char Buf[1000];
 742    string ProperHost = Uri.Host;
 743    if (Uri.Port != 0)
 744    {
 745       sprintf(Buf,":%u",Uri.Port);
 746       ProperHost += Buf;
 747    }
 748
 749    // Just in case.
 750    if (Itm->Uri.length() >= sizeof(Buf))
 751        abort();
 752
 753    /* Build the request. We include a keep-alive header only for non-proxy
 754       requests. This is to tweak old http/1.0 servers that do support keep-alive
 755       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 756       will glitch HTTP/1.0 proxies because they do not filter it out and
 757       pass it on, HTTP/1.1 says the connection should default to keep alive
 758       and we expect the proxy to do this */
 759    if (Proxy.empty() == true || Proxy.Host.empty())
 760       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 761               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 762    else
 763    {
 764       /* Generate a cache control header if necessary. We place a max
 765          cache age on index files, optionally set a no-cache directive
 766          and a no-store directive for archives. */
 767       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 768               Itm->Uri.c_str(),ProperHost.c_str());
 769       // only generate a cache control header if we actually want to
 770       // use a cache
 771       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 772       {
 773          if (Itm->IndexFile == true)
 774             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 775                     _config->FindI("Acquire::http::Max-Age",0));
 776          else
 777          {
 778             if (_config->FindB("Acquire::http::No-Store",false) == true)
 779                strcat(Buf,"Cache-Control: no-store\r\n");
 780          }
 781       }
 782    }
 783    // generate a no-cache header if needed
 784    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 785       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 786
 787
 788    string Req = Buf;
 789
 790    // Check for a partial file
 791    struct stat SBuf;
 792    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 793    {
 794       // In this case we send an if-range query with a range header
 795       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 796               TimeRFC1123(SBuf.st_mtime).c_str());
 797       Req += Buf;
 798    }
 799    else
 800    {
 801       if (Itm->LastModified != 0)
 802       {
 803          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 804          Req += Buf;
 805       }
 806    }
 807
 808    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 809       Req += string("Proxy-Authorization: Basic ") +
 810           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 811
 812    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 813       Req += string("Authorization: Basic ") +
 814           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 815
 816    Req += "User-Agent: Debian APT-HTTP/1.3 ("VERSION")\r\n\r\n";
 817
 818    if (Debug == true)
 819       cerr << Req << endl;
 820
 821    Out.Read(Req);
 822 }
 823                                                                         /*}}}*/
 824 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 825 // ---------------------------------------------------------------------
 826 /* This runs the select loop over the server FDs, Output file FDs and
 827    stdin. */
 828 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 829 {
 830    // Server has closed the connection
 831    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 832                                ToFile == false))
 833       return false;
 834
 835    fd_set rfds,wfds;
 836    FD_ZERO(&rfds);
 837    FD_ZERO(&wfds);
 838
 839    /* Add the server. We only send more requests if the connection will
 840       be persisting */
 841    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 842        && Srv->Persistent == true)
 843       FD_SET(Srv->ServerFd,&wfds);
 844    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 845       FD_SET(Srv->ServerFd,&rfds);
 846
 847    // Add the file
 848    int FileFD = -1;
 849    if (File != 0)
 850       FileFD = File->Fd();
 851
 852    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 853       FD_SET(FileFD,&wfds);
 854
 855    // Add stdin
 856    FD_SET(STDIN_FILENO,&rfds);
 857
 858    // Figure out the max fd
 859    int MaxFd = FileFD;
 860    if (MaxFd < Srv->ServerFd)
 861       MaxFd = Srv->ServerFd;
 862
 863    // Select
 864    struct timeval tv;
 865    tv.tv_sec = TimeOut;
 866    tv.tv_usec = 0;
 867    int Res = 0;
 868    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 869    {
 870       if (errno == EINTR)
 871          return true;
 872       return _error->Errno("select",_("Select failed"));
 873    }
 874
 875    if (Res == 0)
 876    {
 877       _error->Error(_("Connection timed out"));
 878       return ServerDie(Srv);
 879    }
 880
 881    // Handle server IO
 882    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 883    {
 884       errno = 0;
 885       if (Srv->In.Read(Srv->ServerFd) == false)
 886          return ServerDie(Srv);
 887    }
 888
 889    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 890    {
 891       errno = 0;
 892       if (Srv->Out.Write(Srv->ServerFd) == false)
 893          return ServerDie(Srv);
 894    }
 895
 896    // Send data to the file
 897    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 898    {
 899       if (Srv->In.Write(FileFD) == false)
 900          return _error->Errno("write",_("Error writing to output file"));
 901    }
 902
 903    // Handle commands from APT
 904    if (FD_ISSET(STDIN_FILENO,&rfds))
 905    {
 906       if (Run(true) != -1)
 907          exit(100);
 908    }
 909
 910    return true;
 911 }
 912                                                                         /*}}}*/
 913 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 914 // ---------------------------------------------------------------------
 915 /* This takes the current input buffer from the Server FD and writes it
 916    into the file */
 917 bool HttpMethod::Flush(ServerState *Srv)
 918 {
 919    if (File != 0)
 920    {
 921       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 922       // can't be set
 923       if (File->Name() != "/dev/null")
 924          SetNonBlock(File->Fd(),false);
 925       if (Srv->In.WriteSpace() == false)
 926          return true;
 927
 928       while (Srv->In.WriteSpace() == true)
 929       {
 930          if (Srv->In.Write(File->Fd()) == false)
 931             return _error->Errno("write",_("Error writing to file"));
 932          if (Srv->In.IsLimit() == true)
 933             return true;
 934       }
 935
 936       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 937          return true;
 938    }
 939    return false;
 940 }
 941                                                                         /*}}}*/
 942 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 943 // ---------------------------------------------------------------------
 944 /* */
 945 bool HttpMethod::ServerDie(ServerState *Srv)
 946 {
 947    unsigned int LErrno = errno;
 948
 949    // Dump the buffer to the file
 950    if (Srv->State == ServerState::Data)
 951    {
 952       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 953       // can't be set
 954       if (File->Name() != "/dev/null")
 955          SetNonBlock(File->Fd(),false);
 956       while (Srv->In.WriteSpace() == true)
 957       {
 958          if (Srv->In.Write(File->Fd()) == false)
 959             return _error->Errno("write",_("Error writing to the file"));
 960
 961          // Done
 962          if (Srv->In.IsLimit() == true)
 963             return true;
 964       }
 965    }
 966
 967    // See if this is because the server finished the data stream
 968    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 969        Srv->Encoding != ServerState::Closes)
 970    {
 971       Srv->Close();
 972       if (LErrno == 0)
 973          return _error->Error(_("Error reading from server. Remote end closed connection"));
 974       errno = LErrno;
 975       return _error->Errno("read",_("Error reading from server"));
 976    }
 977    else
 978    {
 979       Srv->In.Limit(-1);
 980
 981       // Nothing left in the buffer
 982       if (Srv->In.WriteSpace() == false)
 983          return false;
 984
 985       // We may have got multiple responses back in one packet..
 986       Srv->Close();
 987       return true;
 988    }
 989
 990    return false;
 991 }
 992                                                                         /*}}}*/
 993 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 994 // ---------------------------------------------------------------------
 995 /* We look at the header data we got back from the server and decide what
 996    to do. Returns
 997      0 - File is open,
 998      1 - IMS hit
 999      3 - Unrecoverable error
1000      4 - Error with error content page
1001      5 - Unrecoverable non-server error (close the connection) */
1002 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
1003 {
1004    // Not Modified
1005    if (Srv->Result == 304)
1006    {
1007       unlink(Queue->DestFile.c_str());
1008       Res.IMSHit = true;
1009       Res.LastModified = Queue->LastModified;
1010       return 1;
1011    }
1012
1013    /* We have a reply we dont handle. This should indicate a perm server
1014       failure */
1015    if (Srv->Result < 200 || Srv->Result >= 300)
1016    {
1017       _error->Error("%u %s",Srv->Result,Srv->Code);
1018       if (Srv->HaveContent == true)
1019          return 4;
1020       return 3;
1021    }
1022
1023    // This is some sort of 2xx 'data follows' reply
1024    Res.LastModified = Srv->Date;
1025    Res.Size = Srv->Size;
1026
1027    // Open the file
1028    delete File;
1029    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
1030    if (_error->PendingError() == true)
1031       return 5;
1032
1033    FailFile = Queue->DestFile;
1034    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1035    FailFd = File->Fd();
1036    FailTime = Srv->Date;
1037
1038    // Set the expected size
1039    if (Srv->StartPos >= 0)
1040    {
1041       Res.ResumePoint = Srv->StartPos;
1042       if (ftruncate(File->Fd(),Srv->StartPos) < 0)
1043          _error->Errno("ftruncate", _("Failed to truncate file"));
1044    }
1045
1046    // Set the start point
1047    lseek(File->Fd(),0,SEEK_END);
1048
1049    delete Srv->In.Hash;
1050    Srv->In.Hash = new Hashes;
1051
1052    // Fill the Hash if the file is non-empty (resume)
1053    if (Srv->StartPos > 0)
1054    {
1055       lseek(File->Fd(),0,SEEK_SET);
1056       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1057       {
1058          _error->Errno("read",_("Problem hashing file"));
1059          return 5;
1060       }
1061       lseek(File->Fd(),0,SEEK_END);
1062    }
1063
1064    SetNonBlock(File->Fd(),true);
1065    return 0;
1066 }
1067                                                                         /*}}}*/
1068 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
1069 // ---------------------------------------------------------------------
1070 /* This closes and timestamps the open file. This is neccessary to get
1071    resume behavoir on user abort */
1072 void HttpMethod::SigTerm(int)
1073 {
1074    if (FailFd == -1)
1075       _exit(100);
1076    close(FailFd);
1077
1078    // Timestamp
1079    struct utimbuf UBuf;
1080    UBuf.actime = FailTime;
1081    UBuf.modtime = FailTime;
1082    utime(FailFile.c_str(),&UBuf);
1083
1084    _exit(100);
1085 }
1086                                                                         /*}}}*/
1087 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
1088 // ---------------------------------------------------------------------
1089 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1090    depth. */
1091 bool HttpMethod::Fetch(FetchItem *)
1092 {
1093    if (Server == 0)
1094       return true;
1095
1096    // Queue the requests
1097    int Depth = -1;
1098    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1099         I = I->Next, Depth++)
1100    {
1101       // If pipelining is disabled, we only queue 1 request
1102       if (Server->Pipeline == false && Depth >= 0)
1103          break;
1104
1105       // Make sure we stick with the same server
1106       if (Server->Comp(I->Uri) == false)
1107          break;
1108       if (QueueBack == I)
1109       {
1110          QueueBack = I->Next;
1111          SendReq(I,Server->Out);
1112          continue;
1113       }
1114    }
1115
1116    return true;
1117 };
1118                                                                         /*}}}*/
1119 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1120 // ---------------------------------------------------------------------
1121 /* We stash the desired pipeline depth */
1122 bool HttpMethod::Configuration(string Message)
1123 {
1124    if (pkgAcqMethod::Configuration(Message) == false)
1125       return false;
1126
1127    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1128    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1129                                   PipelineDepth);
1130    Debug = _config->FindB("Debug::Acquire::http",false);
1131
1132    return true;
1133 }
1134                                                                         /*}}}*/
1135 // HttpMethod::Loop - Main loop                                         /*{{{*/
1136 // ---------------------------------------------------------------------
1137 /* */
1138 int HttpMethod::Loop()
1139 {
1140    signal(SIGTERM,SigTerm);
1141    signal(SIGINT,SigTerm);
1142
1143    Server = 0;
1144
1145    std::set<std::string> cached;
1146
1147    int FailCounter = 0;
1148    while (1)
1149    {
1150       // We have no commands, wait for some to arrive
1151       if (Queue == 0)
1152       {
1153          if (WaitFd(STDIN_FILENO) == false)
1154             return 0;
1155       }
1156
1157       /* Run messages, we can accept 0 (no message) if we didn't
1158          do a WaitFd above.. Otherwise the FD is closed. */
1159       int Result = Run(true);
1160       if (Result != -1 && (Result != 0 || Queue == 0))
1161          return 100;
1162
1163       if (Queue == 0)
1164          continue;
1165
1166       CFStringEncoding se = kCFStringEncodingUTF8;
1167
1168       char *url = strdup(Queue->Uri.c_str());
1169     url:
1170       URI uri = std::string(url);
1171       std::string hs = uri.Host;
1172
1173       if (cached.find(hs) != cached.end()) {
1174          _error->Error("Cached Failure");
1175          Fail(true);
1176          free(url);
1177          FailCounter = 0;
1178          continue;
1179       }
1180
1181       std::string urs = uri;
1182
1183       for (;;) {
1184          size_t bad = urs.find_first_of("+");
1185          if (bad == std::string::npos)
1186             break;
1187          // XXX: generalize
1188          urs = urs.substr(0, bad) + "%2b" + urs.substr(bad + 1);
1189       }
1190
1191       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
1192       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1193       CFRelease(sr);
1194       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1195       CFRelease(ur);
1196
1197       struct stat SBuf;
1198       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1199          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1200          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1201          CFRelease(sr);
1202
1203          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1204          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1205          CFRelease(sr);
1206
1207          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
1208       } else if (Queue->LastModified != 0) {
1209          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified).c_str(), se);
1210          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1211          CFRelease(sr);
1212
1213          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
1214       } else
1215          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("max-age=0"));
1216
1217       if (Firmware_ != NULL)
1218          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_);
1219
1220       sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
1221       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
1222       CFRelease(sr);
1223
1224       if (UniqueID_ != NULL)
1225          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
1226
1227       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.592"));
1228
1229       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1230       CFRelease(hm);
1231
1232 #define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
1233 #define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
1234 #define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
1235
1236       /*SInt32 to(TimeOut);
1237       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/
1238       double to(TimeOut);
1239       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
1240
1241       CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
1242       CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
1243       CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
1244       CFRelease(nm);
1245
1246       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1247       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1248       CFRelease(dr);
1249
1250       //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1251       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1252
1253       FetchResult Res;
1254       CFIndex rd;
1255       UInt32 sc;
1256
1257       uint8_t data[10240];
1258       size_t offset = 0;
1259
1260       Status("Connecting to %s", hs.c_str());
1261
1262       switch (CFReadStreamOpen(rs, to)) {
1263          case -1:
1264             CfrsError("Open", rs);
1265          goto fail;
1266
1267          case 0:
1268             _error->Error("Host Unreachable");
1269             cached.insert(hs);
1270          goto fail;
1271
1272          case 1:
1273             /* success */
1274          break;
1275
1276          fail:
1277             Fail(true);
1278          goto done;
1279       }
1280
1281       rd = CFReadStreamRead(rs, data, sizeof(data));
1282
1283       if (rd == -1) {
1284          CfrsError(uri.Host.c_str(), rs);
1285          cached.insert(hs);
1286          Fail(true);
1287          goto done;
1288       }
1289
1290       Res.Filename = Queue->DestFile;
1291
1292       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1293       sc = CFHTTPMessageGetResponseStatusCode(hm);
1294
1295       if (sc == 301 || sc == 302) {
1296          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1297          if (sr == NULL) {
1298             Fail();
1299             goto done_;
1300          } else {
1301             size_t ln = CFStringGetLength(sr) + 1;
1302             free(url);
1303             url = static_cast<char *>(malloc(ln));
1304
1305             if (!CFStringGetCString(sr, url, ln, se)) {
1306                Fail();
1307                goto done_;
1308             }
1309
1310             CFRelease(sr);
1311             goto url;
1312          }
1313       }
1314
1315       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1316       if (sr != NULL) {
1317          size_t ln = CFStringGetLength(sr) + 1;
1318          char cr[ln];
1319
1320          if (!CFStringGetCString(sr, cr, ln, se)) {
1321             Fail();
1322             goto done_;
1323          }
1324
1325          CFRelease(sr);
1326
1327          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1328             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1329             Fail();
1330             goto done_;
1331          }
1332
1333          if (offset > Res.Size) {
1334             _error->Error(_("This HTTP server has broken range support"));
1335             Fail();
1336             goto done_;
1337          }
1338       } else {
1339          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1340          if (sr != NULL) {
1341             Res.Size = CFStringGetIntValue(sr);
1342             CFRelease(sr);
1343          }
1344       }
1345
1346       time(&Res.LastModified);
1347
1348       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1349       if (sr != NULL) {
1350          size_t ln = CFStringGetLength(sr) + 1;
1351          char cr[ln];
1352
1353          if (!CFStringGetCString(sr, cr, ln, se)) {
1354             Fail();
1355             goto done_;
1356          }
1357
1358          CFRelease(sr);
1359
1360          if (!StrToTime(cr, Res.LastModified)) {
1361             _error->Error(_("Unknown date format"));
1362             Fail();
1363             goto done_;
1364          }
1365       }
1366
1367       if (sc < 200 || sc >= 300 && sc != 304) {
1368          sr = CFHTTPMessageCopyResponseStatusLine(hm);
1369
1370          size_t ln = CFStringGetLength(sr) + 1;
1371          char cr[ln];
1372
1373          if (!CFStringGetCString(sr, cr, ln, se)) {
1374             Fail();
1375             goto done;
1376          }
1377
1378          CFRelease(sr);
1379
1380          _error->Error("%s", cr);
1381
1382          Fail();
1383          goto done_;
1384       }
1385
1386       CFRelease(hm);
1387
1388       if (sc == 304) {
1389          unlink(Queue->DestFile.c_str());
1390          Res.IMSHit = true;
1391          Res.LastModified = Queue->LastModified;
1392          URIDone(Res);
1393       } else {
1394          Hashes hash;
1395
1396          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1397          if (_error->PendingError() == true) {
1398             delete File;
1399             File = NULL;
1400             Fail();
1401             goto done;
1402          }
1403
1404          FailFile = Queue->DestFile;
1405          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1406          FailFd = File->Fd();
1407          FailTime = Res.LastModified;
1408
1409          Res.ResumePoint = offset;
1410          ftruncate(File->Fd(), offset);
1411
1412          if (offset != 0) {
1413             lseek(File->Fd(), 0, SEEK_SET);
1414             if (!hash.AddFD(File->Fd(), offset)) {
1415                _error->Errno("read", _("Problem hashing file"));
1416                delete File;
1417                File = NULL;
1418                Fail();
1419                goto done;
1420             }
1421          }
1422
1423          lseek(File->Fd(), 0, SEEK_END);
1424
1425          URIStart(Res);
1426
1427          read: if (rd == -1) {
1428             CfrsError("rd", rs);
1429             Fail(true);
1430          } else if (rd == 0) {
1431             if (Res.Size == 0)
1432                Res.Size = File->Size();
1433
1434             struct utimbuf UBuf;
1435             time(&UBuf.actime);
1436             UBuf.actime = Res.LastModified;
1437             UBuf.modtime = Res.LastModified;
1438             utime(Queue->DestFile.c_str(), &UBuf);
1439
1440             Res.TakeHashes(hash);
1441             URIDone(Res);
1442          } else {
1443             hash.Add(data, rd);
1444
1445             uint8_t *dt = data;
1446             while (rd != 0) {
1447                int sz = write(File->Fd(), dt, rd);
1448
1449                if (sz == -1) {
1450                    delete File;
1451                    File = NULL;
1452                    Fail();
1453                    goto done;
1454                }
1455
1456                dt += sz;
1457                rd -= sz;
1458             }
1459
1460             rd = CFReadStreamRead(rs, data, sizeof(data));
1461             goto read;
1462          }
1463       }
1464
1465      goto done;
1466     done_:
1467       CFRelease(hm);
1468     done:
1469       CFReadStreamClose(rs);
1470       CFRelease(rs);
1471       free(url);
1472
1473       FailCounter = 0;
1474    }
1475
1476    return 0;
1477 }
1478                                                                         /*}}}*/
1479
1480 int main()
1481 {
1482    setlocale(LC_ALL, "");
1483
1484    HttpMethod Mth;
1485
1486     size_t size;
1487     sysctlbyname("hw.machine", NULL, &size, NULL, 0);
1488     char *machine = new char[size];
1489     sysctlbyname("hw.machine", machine, &size, NULL, 0);
1490     Machine_ = machine;
1491
1492     const char *path = "/System/Library/CoreServices/SystemVersion.plist";
1493     CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false);
1494
1495     CFPropertyListRef plist; {
1496         CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url);
1497         CFReadStreamOpen(stream);
1498         plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL);
1499         CFReadStreamClose(stream);
1500     }
1501
1502     CFRelease(url);
1503
1504     if (plist != NULL) {
1505         Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion")));
1506         CFRelease(plist);
1507     }
1508
1509     if (void *lockdown = lockdown_connect()) {
1510         UniqueID_ = lockdown_copy_value(lockdown, NULL, kLockdownUniqueDeviceIDKey);
1511         lockdown_disconnect(lockdown);
1512     }
1513
1514    return Mth.Loop();
1515 }
1516
1517