methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32 #include <apt-pkg/netrc.h>
  33
  34 #include <sys/sysctl.h>
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42 #include <string.h>
  43 #include <iostream>
  44 #include <map>
  45 #include <set>
  46 #include <apti18n.h>
  47
  48
  49 // Internet stuff
  50 #include <netdb.h>
  51 #include <arpa/inet.h>
  52
  53 #include <lockdown.h>
  54 #include <CoreFoundation/CoreFoundation.h>
  55 #include <CoreServices/CoreServices.h>
  56 #include <SystemConfiguration/SystemConfiguration.h>
  57
  58 #include "config.h"
  59 #include "connect.h"
  60 #include "rfc2553emu.h"
  61 #include "http.h"
  62                                                                         /*}}}*/
  63 using namespace std;
  64
  65 CFStringRef Firmware_;
  66 const char *Machine_;
  67 CFStringRef UniqueID_;
  68
  69 void CfrsError(const char *name, CFReadStreamRef rs) {
  70     CFStreamError se = CFReadStreamGetError(rs);
  71
  72     if (se.domain == kCFStreamErrorDomainCustom) {
  73     } else if (se.domain == kCFStreamErrorDomainPOSIX) {
  74         _error->Error("POSIX: %s", strerror(se.error));
  75     } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
  76         _error->Error("MacOSStatus: %ld", se.error);
  77     } else if (se.domain == kCFStreamErrorDomainNetDB) {
  78         _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
  79     } else if (se.domain == kCFStreamErrorDomainMach) {
  80         _error->Error("Mach: %ld", se.error);
  81     } else if (se.domain == kCFStreamErrorDomainHTTP) {
  82         switch (se.error) {
  83             case kCFStreamErrorHTTPParseFailure:
  84                 _error->Error("Parse failure");
  85             break;
  86
  87             case kCFStreamErrorHTTPRedirectionLoop:
  88                 _error->Error("Redirection loop");
  89             break;
  90
  91             case kCFStreamErrorHTTPBadURL:
  92                 _error->Error("Bad URL");
  93             break;
  94
  95             default:
  96                 _error->Error("Unknown HTTP error: %ld", se.error);
  97             break;
  98         }
  99     } else if (se.domain == kCFStreamErrorDomainSOCKS) {
 100         _error->Error("SOCKS: %ld", se.error);
 101     } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
 102         _error->Error("SystemConfiguration: %ld", se.error);
 103     } else if (se.domain == kCFStreamErrorDomainSSL) {
 104         _error->Error("SSL: %ld", se.error);
 105     } else {
 106         _error->Error("Domain #%ld: %ld", se.domain, se.error);
 107     }
 108 }
 109
 110 string HttpMethod::FailFile;
 111 int HttpMethod::FailFd = -1;
 112 time_t HttpMethod::FailTime = 0;
 113 unsigned long PipelineDepth = 10;
 114 unsigned long TimeOut = 120;
 115 bool AllowRedirect = false;
 116 bool Debug = false;
 117 URI Proxy;
 118
 119 unsigned long CircleBuf::BwReadLimit=0;
 120 unsigned long CircleBuf::BwTickReadData=0;
 121 struct timeval CircleBuf::BwReadTick={0,0};
 122 const unsigned int CircleBuf::BW_HZ=10;
 123
 124 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
 128 {
 129    Buf = new unsigned char[Size];
 130    Reset();
 131
 132    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
 133 }
 134                                                                         /*}}}*/
 135 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* */
 138 void CircleBuf::Reset()
 139 {
 140    InP = 0;
 141    OutP = 0;
 142    StrPos = 0;
 143    MaxGet = (unsigned int)-1;
 144    OutQueue = string();
 145    if (Hash != 0)
 146    {
 147       delete Hash;
 148       Hash = new Hashes;
 149    }
 150 };
 151                                                                         /*}}}*/
 152 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
 153 // ---------------------------------------------------------------------
 154 /* This fills up the buffer with as much data as is in the FD, assuming it
 155    is non-blocking.. */
 156 bool CircleBuf::Read(int Fd)
 157 {
 158    unsigned long BwReadMax;
 159
 160    while (1)
 161    {
 162       // Woops, buffer is full
 163       if (InP - OutP == Size)
 164          return true;
 165
 166       // what's left to read in this tick
 167       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 168
 169       if(CircleBuf::BwReadLimit) {
 170          struct timeval now;
 171          gettimeofday(&now,0);
 172
 173          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 174             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 175          if(d > 1000000/BW_HZ) {
 176             CircleBuf::BwReadTick = now;
 177             CircleBuf::BwTickReadData = 0;
 178          }
 179
 180          if(CircleBuf::BwTickReadData >= BwReadMax) {
 181             usleep(1000000/BW_HZ);
 182             return true;
 183          }
 184       }
 185
 186       // Write the buffer segment
 187       int Res;
 188       if(CircleBuf::BwReadLimit) {
 189          Res = read(Fd,Buf + (InP%Size),
 190                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 191       } else
 192          Res = read(Fd,Buf + (InP%Size),LeftRead());
 193
 194       if(Res > 0 && BwReadLimit > 0)
 195          CircleBuf::BwTickReadData += Res;
 196
 197       if (Res == 0)
 198          return false;
 199       if (Res < 0)
 200       {
 201          if (errno == EAGAIN)
 202             return true;
 203          return false;
 204       }
 205
 206       if (InP == 0)
 207          gettimeofday(&Start,0);
 208       InP += Res;
 209    }
 210 }
 211                                                                         /*}}}*/
 212 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 213 // ---------------------------------------------------------------------
 214 /* This will hold the string in and fill the buffer with it as it empties */
 215 bool CircleBuf::Read(string Data)
 216 {
 217    OutQueue += Data;
 218    FillOut();
 219    return true;
 220 }
 221                                                                         /*}}}*/
 222 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 223 // ---------------------------------------------------------------------
 224 /* */
 225 void CircleBuf::FillOut()
 226 {
 227    if (OutQueue.empty() == true)
 228       return;
 229    while (1)
 230    {
 231       // Woops, buffer is full
 232       if (InP - OutP == Size)
 233          return;
 234
 235       // Write the buffer segment
 236       unsigned long Sz = LeftRead();
 237       if (OutQueue.length() - StrPos < Sz)
 238          Sz = OutQueue.length() - StrPos;
 239       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 240
 241       // Advance
 242       StrPos += Sz;
 243       InP += Sz;
 244       if (OutQueue.length() == StrPos)
 245       {
 246          StrPos = 0;
 247          OutQueue = "";
 248          return;
 249       }
 250    }
 251 }
 252                                                                         /*}}}*/
 253 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 254 // ---------------------------------------------------------------------
 255 /* This empties the buffer into the FD. */
 256 bool CircleBuf::Write(int Fd)
 257 {
 258    while (1)
 259    {
 260       FillOut();
 261
 262       // Woops, buffer is empty
 263       if (OutP == InP)
 264          return true;
 265
 266       if (OutP == MaxGet)
 267          return true;
 268
 269       // Write the buffer segment
 270       int Res;
 271       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 272
 273       if (Res == 0)
 274          return false;
 275       if (Res < 0)
 276       {
 277          if (errno == EAGAIN)
 278             return true;
 279
 280          return false;
 281       }
 282
 283       if (Hash != 0)
 284          Hash->Add(Buf + (OutP%Size),Res);
 285
 286       OutP += Res;
 287    }
 288 }
 289                                                                         /*}}}*/
 290 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 291 // ---------------------------------------------------------------------
 292 /* This copies till the first empty line */
 293 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 294 {
 295    // We cheat and assume it is unneeded to have more than one buffer load
 296    for (unsigned long I = OutP; I < InP; I++)
 297    {
 298       if (Buf[I%Size] != '\n')
 299          continue;
 300       ++I;
 301
 302       if (Single == false)
 303       {
 304          if (I < InP  && Buf[I%Size] == '\r')
 305             ++I;
 306          if (I >= InP || Buf[I%Size] != '\n')
 307             continue;
 308          ++I;
 309       }
 310
 311       Data = "";
 312       while (OutP < I)
 313       {
 314          unsigned long Sz = LeftWrite();
 315          if (Sz == 0)
 316             return false;
 317          if (I - OutP < Sz)
 318             Sz = I - OutP;
 319          Data += string((char *)(Buf + (OutP%Size)),Sz);
 320          OutP += Sz;
 321       }
 322       return true;
 323    }
 324    return false;
 325 }
 326                                                                         /*}}}*/
 327 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 328 // ---------------------------------------------------------------------
 329 /* */
 330 void CircleBuf::Stats()
 331 {
 332    if (InP == 0)
 333       return;
 334
 335    struct timeval Stop;
 336    gettimeofday(&Stop,0);
 337 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 338              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 339    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 340 }
 341                                                                         /*}}}*/
 342
 343 // ServerState::ServerState - Constructor                               /*{{{*/
 344 // ---------------------------------------------------------------------
 345 /* */
 346 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 347                         In(64*1024), Out(4*1024),
 348                         ServerName(Srv)
 349 {
 350    Reset();
 351 }
 352                                                                         /*}}}*/
 353 // ServerState::Open - Open a connection to the server                  /*{{{*/
 354 // ---------------------------------------------------------------------
 355 /* This opens a connection to the server. */
 356 bool ServerState::Open()
 357 {
 358    // Use the already open connection if possible.
 359    if (ServerFd != -1)
 360       return true;
 361
 362    Close();
 363    In.Reset();
 364    Out.Reset();
 365    Persistent = true;
 366
 367    // Determine the proxy setting
 368    string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 369    if (!SpecificProxy.empty())
 370    {
 371            if (SpecificProxy == "DIRECT")
 372                    Proxy = "";
 373            else
 374                    Proxy = SpecificProxy;
 375    }
 376    else
 377    {
 378            string DefProxy = _config->Find("Acquire::http::Proxy");
 379            if (!DefProxy.empty())
 380            {
 381                    Proxy = DefProxy;
 382            }
 383            else
 384            {
 385                    char* result = getenv("http_proxy");
 386                    Proxy = result ? result : "";
 387            }
 388    }
 389
 390    // Parse no_proxy, a , separated list of domains
 391    if (getenv("no_proxy") != 0)
 392    {
 393       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 394          Proxy = "";
 395    }
 396
 397    // Determine what host and port to use based on the proxy settings
 398    int Port = 0;
 399    string Host;
 400    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 401    {
 402       if (ServerName.Port != 0)
 403          Port = ServerName.Port;
 404       Host = ServerName.Host;
 405    }
 406    else
 407    {
 408       if (Proxy.Port != 0)
 409          Port = Proxy.Port;
 410       Host = Proxy.Host;
 411    }
 412
 413    // Connect to the remote server
 414    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 415       return false;
 416
 417    return true;
 418 }
 419                                                                         /*}}}*/
 420 // ServerState::Close - Close a connection to the server                /*{{{*/
 421 // ---------------------------------------------------------------------
 422 /* */
 423 bool ServerState::Close()
 424 {
 425    close(ServerFd);
 426    ServerFd = -1;
 427    return true;
 428 }
 429                                                                         /*}}}*/
 430 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 431 // ---------------------------------------------------------------------
 432 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
 433    parse error occurred */
 434 int ServerState::RunHeaders()
 435 {
 436    State = Header;
 437
 438    Owner->Status(_("Waiting for headers"));
 439
 440    Major = 0;
 441    Minor = 0;
 442    Result = 0;
 443    Size = 0;
 444    StartPos = 0;
 445    Encoding = Closes;
 446    HaveContent = false;
 447    time(&Date);
 448
 449    do
 450    {
 451       string Data;
 452       if (In.WriteTillEl(Data) == false)
 453          continue;
 454
 455       if (Debug == true)
 456          clog << Data;
 457
 458       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 459       {
 460          string::const_iterator J = I;
 461          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 462          if (HeaderLine(string(I,J)) == false)
 463             return 2;
 464          I = J;
 465       }
 466
 467       // 100 Continue is a Nop...
 468       if (Result == 100)
 469          continue;
 470
 471       // Tidy up the connection persistance state.
 472       if (Encoding == Closes && HaveContent == true)
 473          Persistent = false;
 474
 475       return 0;
 476    }
 477    while (Owner->Go(false,this) == true);
 478
 479    return 1;
 480 }
 481                                                                         /*}}}*/
 482 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 483 // ---------------------------------------------------------------------
 484 /* */
 485 bool ServerState::RunData()
 486 {
 487    State = Data;
 488
 489    // Chunked transfer encoding is fun..
 490    if (Encoding == Chunked)
 491    {
 492       while (1)
 493       {
 494          // Grab the block size
 495          bool Last = true;
 496          string Data;
 497          In.Limit(-1);
 498          do
 499          {
 500             if (In.WriteTillEl(Data,true) == true)
 501                break;
 502          }
 503          while ((Last = Owner->Go(false,this)) == true);
 504
 505          if (Last == false)
 506             return false;
 507
 508          // See if we are done
 509          unsigned long Len = strtol(Data.c_str(),0,16);
 510          if (Len == 0)
 511          {
 512             In.Limit(-1);
 513
 514             // We have to remove the entity trailer
 515             Last = true;
 516             do
 517             {
 518                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 519                   break;
 520             }
 521             while ((Last = Owner->Go(false,this)) == true);
 522             if (Last == false)
 523                return false;
 524             return !_error->PendingError();
 525          }
 526
 527          // Transfer the block
 528          In.Limit(Len);
 529          while (Owner->Go(true,this) == true)
 530             if (In.IsLimit() == true)
 531                break;
 532
 533          // Error
 534          if (In.IsLimit() == false)
 535             return false;
 536
 537          // The server sends an extra new line before the next block specifier..
 538          In.Limit(-1);
 539          Last = true;
 540          do
 541          {
 542             if (In.WriteTillEl(Data,true) == true)
 543                break;
 544          }
 545          while ((Last = Owner->Go(false,this)) == true);
 546          if (Last == false)
 547             return false;
 548       }
 549    }
 550    else
 551    {
 552       /* Closes encoding is used when the server did not specify a size, the
 553          loss of the connection means we are done */
 554       if (Encoding == Closes)
 555          In.Limit(-1);
 556       else
 557          In.Limit(Size - StartPos);
 558
 559       // Just transfer the whole block.
 560       do
 561       {
 562          if (In.IsLimit() == false)
 563             continue;
 564
 565          In.Limit(-1);
 566          return !_error->PendingError();
 567       }
 568       while (Owner->Go(true,this) == true);
 569    }
 570
 571    return Owner->Flush(this) && !_error->PendingError();
 572 }
 573                                                                         /*}}}*/
 574 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 575 // ---------------------------------------------------------------------
 576 /* */
 577 bool ServerState::HeaderLine(string Line)
 578 {
 579    if (Line.empty() == true)
 580       return true;
 581
 582    // The http server might be trying to do something evil.
 583    if (Line.length() >= MAXLEN)
 584       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 585
 586    string::size_type Pos = Line.find(' ');
 587    if (Pos == string::npos || Pos+1 > Line.length())
 588    {
 589       // Blah, some servers use "connection:closes", evil.
 590       Pos = Line.find(':');
 591       if (Pos == string::npos || Pos + 2 > Line.length())
 592          return _error->Error(_("Bad header line"));
 593       Pos++;
 594    }
 595
 596    // Parse off any trailing spaces between the : and the next word.
 597    string::size_type Pos2 = Pos;
 598    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 599       Pos2++;
 600
 601    string Tag = string(Line,0,Pos);
 602    string Val = string(Line,Pos2);
 603
 604    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 605    {
 606       // Evil servers return no version
 607       if (Line[4] == '/')
 608       {
 609          if (sscanf(Line.c_str(),"HTTP/%u.%u %u%[^\n]",&Major,&Minor,
 610                     &Result,Code) != 4)
 611             return _error->Error(_("The HTTP server sent an invalid reply header"));
 612       }
 613       else
 614       {
 615          Major = 0;
 616          Minor = 9;
 617          if (sscanf(Line.c_str(),"HTTP %u%[^\n]",&Result,Code) != 2)
 618             return _error->Error(_("The HTTP server sent an invalid reply header"));
 619       }
 620
 621       /* Check the HTTP response header to get the default persistance
 622          state. */
 623       if (Major < 1)
 624          Persistent = false;
 625       else
 626       {
 627          if (Major == 1 && Minor <= 0)
 628             Persistent = false;
 629          else
 630             Persistent = true;
 631       }
 632
 633       return true;
 634    }
 635
 636    if (stringcasecmp(Tag,"Content-Length:") == 0)
 637    {
 638       if (Encoding == Closes)
 639          Encoding = Stream;
 640       HaveContent = true;
 641
 642       // The length is already set from the Content-Range header
 643       if (StartPos != 0)
 644          return true;
 645
 646       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 647          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 648       return true;
 649    }
 650
 651    if (stringcasecmp(Tag,"Content-Type:") == 0)
 652    {
 653       HaveContent = true;
 654       return true;
 655    }
 656
 657    if (stringcasecmp(Tag,"Content-Range:") == 0)
 658    {
 659       HaveContent = true;
 660
 661       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 662          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 663       if ((unsigned)StartPos > Size)
 664          return _error->Error(_("This HTTP server has broken range support"));
 665       return true;
 666    }
 667
 668    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 669    {
 670       HaveContent = true;
 671       if (stringcasecmp(Val,"chunked") == 0)
 672          Encoding = Chunked;
 673       return true;
 674    }
 675
 676    if (stringcasecmp(Tag,"Connection:") == 0)
 677    {
 678       if (stringcasecmp(Val,"close") == 0)
 679          Persistent = false;
 680       if (stringcasecmp(Val,"keep-alive") == 0)
 681          Persistent = true;
 682       return true;
 683    }
 684
 685    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 686    {
 687       if (StrToTime(Val,Date) == false)
 688          return _error->Error(_("Unknown date format"));
 689       return true;
 690    }
 691
 692    if (stringcasecmp(Tag,"Location:") == 0)
 693    {
 694       Location = Val;
 695       return true;
 696    }
 697
 698    return true;
 699 }
 700                                                                         /*}}}*/
 701
 702 static const CFOptionFlags kNetworkEvents =
 703     kCFStreamEventOpenCompleted |
 704     kCFStreamEventHasBytesAvailable |
 705     kCFStreamEventEndEncountered |
 706     kCFStreamEventErrorOccurred |
 707 0;
 708
 709 static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
 710     switch (event) {
 711         case kCFStreamEventOpenCompleted:
 712         break;
 713
 714         case kCFStreamEventHasBytesAvailable:
 715         case kCFStreamEventEndEncountered:
 716             *reinterpret_cast<int *>(arg) = 1;
 717             CFRunLoopStop(CFRunLoopGetCurrent());
 718         break;
 719
 720         case kCFStreamEventErrorOccurred:
 721             *reinterpret_cast<int *>(arg) = -1;
 722             CFRunLoopStop(CFRunLoopGetCurrent());
 723         break;
 724     }
 725 }
 726
 727 /* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
 728 int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
 729     CFStreamClientContext context;
 730     int value(0);
 731
 732     memset(&context, 0, sizeof(context));
 733     context.info = &value;
 734
 735     if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
 736         CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
 737         if (CFReadStreamOpen(stream))
 738             CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
 739         else
 740             value = -1;
 741         CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
 742     }
 743
 744     return value;
 745 }
 746
 747 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 748 // ---------------------------------------------------------------------
 749 /* This places the http request in the outbound buffer */
 750 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 751 {
 752    URI Uri = Itm->Uri;
 753
 754    // The HTTP server expects a hostname with a trailing :port
 755    char Buf[1000];
 756    string ProperHost = Uri.Host;
 757    if (Uri.Port != 0)
 758    {
 759       sprintf(Buf,":%u",Uri.Port);
 760       ProperHost += Buf;
 761    }
 762
 763    // Just in case.
 764    if (Itm->Uri.length() >= sizeof(Buf))
 765        abort();
 766
 767    /* Build the request. We include a keep-alive header only for non-proxy
 768       requests. This is to tweak old http/1.0 servers that do support keep-alive
 769       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 770       will glitch HTTP/1.0 proxies because they do not filter it out and
 771       pass it on, HTTP/1.1 says the connection should default to keep alive
 772       and we expect the proxy to do this */
 773    if (Proxy.empty() == true || Proxy.Host.empty())
 774       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 775               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 776    else
 777    {
 778       /* Generate a cache control header if necessary. We place a max
 779          cache age on index files, optionally set a no-cache directive
 780          and a no-store directive for archives. */
 781       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 782               Itm->Uri.c_str(),ProperHost.c_str());
 783       // only generate a cache control header if we actually want to
 784       // use a cache
 785       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 786       {
 787          if (Itm->IndexFile == true)
 788             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 789                     _config->FindI("Acquire::http::Max-Age",0));
 790          else
 791          {
 792             if (_config->FindB("Acquire::http::No-Store",false) == true)
 793                strcat(Buf,"Cache-Control: no-store\r\n");
 794          }
 795       }
 796    }
 797    // generate a no-cache header if needed
 798    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 799       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 800
 801
 802    string Req = Buf;
 803
 804    // Check for a partial file
 805    struct stat SBuf;
 806    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 807    {
 808       // In this case we send an if-range query with a range header
 809       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 810               TimeRFC1123(SBuf.st_mtime).c_str());
 811       Req += Buf;
 812    }
 813    else
 814    {
 815       if (Itm->LastModified != 0)
 816       {
 817          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 818          Req += Buf;
 819       }
 820    }
 821
 822    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 823       Req += string("Proxy-Authorization: Basic ") +
 824           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 825
 826    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 827    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 828    {
 829       Req += string("Authorization: Basic ") +
 830           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 831    }
 832    Req += "User-Agent: " + _config->Find("Acquire::http::User-Agent",
 833                 "Debian APT-HTTP/1.3 ("VERSION")") + "\r\n\r\n";
 834
 835    if (Debug == true)
 836       cerr << Req << endl;
 837
 838    Out.Read(Req);
 839 }
 840                                                                         /*}}}*/
 841 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* This runs the select loop over the server FDs, Output file FDs and
 844    stdin. */
 845 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 846 {
 847    // Server has closed the connection
 848    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 849                                ToFile == false))
 850       return false;
 851
 852    fd_set rfds,wfds;
 853    FD_ZERO(&rfds);
 854    FD_ZERO(&wfds);
 855
 856    /* Add the server. We only send more requests if the connection will
 857       be persisting */
 858    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 859        && Srv->Persistent == true)
 860       FD_SET(Srv->ServerFd,&wfds);
 861    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 862       FD_SET(Srv->ServerFd,&rfds);
 863
 864    // Add the file
 865    int FileFD = -1;
 866    if (File != 0)
 867       FileFD = File->Fd();
 868
 869    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 870       FD_SET(FileFD,&wfds);
 871
 872    // Add stdin
 873    FD_SET(STDIN_FILENO,&rfds);
 874
 875    // Figure out the max fd
 876    int MaxFd = FileFD;
 877    if (MaxFd < Srv->ServerFd)
 878       MaxFd = Srv->ServerFd;
 879
 880    // Select
 881    struct timeval tv;
 882    tv.tv_sec = TimeOut;
 883    tv.tv_usec = 0;
 884    int Res = 0;
 885    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 886    {
 887       if (errno == EINTR)
 888          return true;
 889       return _error->Errno("select",_("Select failed"));
 890    }
 891
 892    if (Res == 0)
 893    {
 894       _error->Error(_("Connection timed out"));
 895       return ServerDie(Srv);
 896    }
 897
 898    // Handle server IO
 899    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 900    {
 901       errno = 0;
 902       if (Srv->In.Read(Srv->ServerFd) == false)
 903          return ServerDie(Srv);
 904    }
 905
 906    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 907    {
 908       errno = 0;
 909       if (Srv->Out.Write(Srv->ServerFd) == false)
 910          return ServerDie(Srv);
 911    }
 912
 913    // Send data to the file
 914    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 915    {
 916       if (Srv->In.Write(FileFD) == false)
 917          return _error->Errno("write",_("Error writing to output file"));
 918    }
 919
 920    // Handle commands from APT
 921    if (FD_ISSET(STDIN_FILENO,&rfds))
 922    {
 923       if (Run(true) != -1)
 924          exit(100);
 925    }
 926
 927    return true;
 928 }
 929                                                                         /*}}}*/
 930 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 931 // ---------------------------------------------------------------------
 932 /* This takes the current input buffer from the Server FD and writes it
 933    into the file */
 934 bool HttpMethod::Flush(ServerState *Srv)
 935 {
 936    if (File != 0)
 937    {
 938       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 939       // can't be set
 940       if (File->Name() != "/dev/null")
 941          SetNonBlock(File->Fd(),false);
 942       if (Srv->In.WriteSpace() == false)
 943          return true;
 944
 945       while (Srv->In.WriteSpace() == true)
 946       {
 947          if (Srv->In.Write(File->Fd()) == false)
 948             return _error->Errno("write",_("Error writing to file"));
 949          if (Srv->In.IsLimit() == true)
 950             return true;
 951       }
 952
 953       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 954          return true;
 955    }
 956    return false;
 957 }
 958                                                                         /*}}}*/
 959 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 960 // ---------------------------------------------------------------------
 961 /* */
 962 bool HttpMethod::ServerDie(ServerState *Srv)
 963 {
 964    unsigned int LErrno = errno;
 965
 966    // Dump the buffer to the file
 967    if (Srv->State == ServerState::Data)
 968    {
 969       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 970       // can't be set
 971       if (File->Name() != "/dev/null")
 972          SetNonBlock(File->Fd(),false);
 973       while (Srv->In.WriteSpace() == true)
 974       {
 975          if (Srv->In.Write(File->Fd()) == false)
 976             return _error->Errno("write",_("Error writing to the file"));
 977
 978          // Done
 979          if (Srv->In.IsLimit() == true)
 980             return true;
 981       }
 982    }
 983
 984    // See if this is because the server finished the data stream
 985    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 986        Srv->Encoding != ServerState::Closes)
 987    {
 988       Srv->Close();
 989       if (LErrno == 0)
 990          return _error->Error(_("Error reading from server. Remote end closed connection"));
 991       errno = LErrno;
 992       return _error->Errno("read",_("Error reading from server"));
 993    }
 994    else
 995    {
 996       Srv->In.Limit(-1);
 997
 998       // Nothing left in the buffer
 999       if (Srv->In.WriteSpace() == false)
1000          return false;
1001
1002       // We may have got multiple responses back in one packet..
1003       Srv->Close();
1004       return true;
1005    }
1006
1007    return false;
1008 }
1009                                                                         /*}}}*/
1010 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
1011 // ---------------------------------------------------------------------
1012 /* We look at the header data we got back from the server and decide what
1013    to do. Returns
1014      0 - File is open,
1015      1 - IMS hit
1016      3 - Unrecoverable error
1017      4 - Error with error content page
1018      5 - Unrecoverable non-server error (close the connection)
1019      6 - Try again with a new or changed URI
1020  */
1021 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
1022 {
1023    // Not Modified
1024    if (Srv->Result == 304)
1025    {
1026       unlink(Queue->DestFile.c_str());
1027       Res.IMSHit = true;
1028       Res.LastModified = Queue->LastModified;
1029       return 1;
1030    }
1031
1032    /* Redirect
1033     *
1034     * Note that it is only OK for us to treat all redirection the same
1035     * because we *always* use GET, not other HTTP methods.  There are
1036     * three redirection codes for which it is not appropriate that we
1037     * redirect.  Pass on those codes so the error handling kicks in.
1038     */
1039    if (AllowRedirect
1040        && (Srv->Result > 300 && Srv->Result < 400)
1041        && (Srv->Result != 300       // Multiple Choices
1042            && Srv->Result != 304    // Not Modified
1043            && Srv->Result != 306))  // (Not part of HTTP/1.1, reserved)
1044    {
1045       if (!Srv->Location.empty())
1046       {
1047          NextURI = Srv->Location;
1048          return 6;
1049       }
1050       /* else pass through for error message */
1051    }
1052
1053    /* We have a reply we dont handle. This should indicate a perm server
1054       failure */
1055    if (Srv->Result < 200 || Srv->Result >= 300)
1056    {
1057       _error->Error("%u %s",Srv->Result,Srv->Code);
1058       if (Srv->HaveContent == true)
1059          return 4;
1060       return 3;
1061    }
1062
1063    // This is some sort of 2xx 'data follows' reply
1064    Res.LastModified = Srv->Date;
1065    Res.Size = Srv->Size;
1066
1067    // Open the file
1068    delete File;
1069    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
1070    if (_error->PendingError() == true)
1071       return 5;
1072
1073    FailFile = Queue->DestFile;
1074    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1075    FailFd = File->Fd();
1076    FailTime = Srv->Date;
1077
1078    // Set the expected size
1079    if (Srv->StartPos >= 0)
1080    {
1081       Res.ResumePoint = Srv->StartPos;
1082       if (ftruncate(File->Fd(),Srv->StartPos) < 0)
1083          _error->Errno("ftruncate", _("Failed to truncate file"));
1084    }
1085
1086    // Set the start point
1087    lseek(File->Fd(),0,SEEK_END);
1088
1089    delete Srv->In.Hash;
1090    Srv->In.Hash = new Hashes;
1091
1092    // Fill the Hash if the file is non-empty (resume)
1093    if (Srv->StartPos > 0)
1094    {
1095       lseek(File->Fd(),0,SEEK_SET);
1096       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1097       {
1098          _error->Errno("read",_("Problem hashing file"));
1099          return 5;
1100       }
1101       lseek(File->Fd(),0,SEEK_END);
1102    }
1103
1104    SetNonBlock(File->Fd(),true);
1105    return 0;
1106 }
1107                                                                         /*}}}*/
1108 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
1109 // ---------------------------------------------------------------------
1110 /* This closes and timestamps the open file. This is neccessary to get
1111    resume behavoir on user abort */
1112 void HttpMethod::SigTerm(int)
1113 {
1114    if (FailFd == -1)
1115       _exit(100);
1116    close(FailFd);
1117
1118    // Timestamp
1119    struct utimbuf UBuf;
1120    UBuf.actime = FailTime;
1121    UBuf.modtime = FailTime;
1122    utime(FailFile.c_str(),&UBuf);
1123
1124    _exit(100);
1125 }
1126                                                                         /*}}}*/
1127 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
1128 // ---------------------------------------------------------------------
1129 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1130    depth. */
1131 bool HttpMethod::Fetch(FetchItem *)
1132 {
1133    if (Server == 0)
1134       return true;
1135
1136    // Queue the requests
1137    int Depth = -1;
1138    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1139         I = I->Next, Depth++)
1140    {
1141       // If pipelining is disabled, we only queue 1 request
1142       if (Server->Pipeline == false && Depth >= 0)
1143          break;
1144
1145       // Make sure we stick with the same server
1146       if (Server->Comp(I->Uri) == false)
1147          break;
1148       if (QueueBack == I)
1149       {
1150          QueueBack = I->Next;
1151          SendReq(I,Server->Out);
1152          continue;
1153       }
1154    }
1155
1156    return true;
1157 };
1158                                                                         /*}}}*/
1159 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1160 // ---------------------------------------------------------------------
1161 /* We stash the desired pipeline depth */
1162 bool HttpMethod::Configuration(string Message)
1163 {
1164    if (pkgAcqMethod::Configuration(Message) == false)
1165       return false;
1166
1167    AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
1168    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1169    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1170                                   PipelineDepth);
1171    Debug = _config->FindB("Debug::Acquire::http",false);
1172
1173    return true;
1174 }
1175                                                                         /*}}}*/
1176 // HttpMethod::Loop - Main loop                                         /*{{{*/
1177 // ---------------------------------------------------------------------
1178 /* */
1179 int HttpMethod::Loop()
1180 {
1181    typedef vector<string> StringVector;
1182    typedef vector<string>::iterator StringVectorIterator;
1183    map<string, StringVector> Redirected;
1184
1185    signal(SIGTERM,SigTerm);
1186    signal(SIGINT,SigTerm);
1187
1188    Server = 0;
1189
1190    std::set<std::string> cached;
1191
1192    int FailCounter = 0;
1193    while (1)
1194    {
1195       // We have no commands, wait for some to arrive
1196       if (Queue == 0)
1197       {
1198          if (WaitFd(STDIN_FILENO) == false)
1199             return 0;
1200       }
1201
1202       /* Run messages, we can accept 0 (no message) if we didn't
1203          do a WaitFd above.. Otherwise the FD is closed. */
1204       int Result = Run(true);
1205       if (Result != -1 && (Result != 0 || Queue == 0))
1206          return 100;
1207
1208       if (Queue == 0)
1209          continue;
1210
1211       CFStringEncoding se = kCFStringEncodingUTF8;
1212
1213       char *url = strdup(Queue->Uri.c_str());
1214     url:
1215       URI uri = std::string(url);
1216       std::string hs = uri.Host;
1217
1218       if (cached.find(hs) != cached.end()) {
1219          _error->Error("Cached Failure");
1220          Fail(true);
1221          free(url);
1222          FailCounter = 0;
1223          continue;
1224       }
1225
1226       std::string urs = uri;
1227
1228       for (;;) {
1229          size_t bad = urs.find_first_of("+");
1230          if (bad == std::string::npos)
1231             break;
1232          // XXX: generalize
1233          urs = urs.substr(0, bad) + "%2b" + urs.substr(bad + 1);
1234       }
1235
1236       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
1237       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1238       CFRelease(sr);
1239       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1240       CFRelease(ur);
1241
1242       struct stat SBuf;
1243       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1244          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1245          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1246          CFRelease(sr);
1247
1248          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1249          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1250          CFRelease(sr);
1251
1252          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
1253       } else if (Queue->LastModified != 0) {
1254          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified).c_str(), se);
1255          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1256          CFRelease(sr);
1257
1258          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
1259       } else
1260          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("max-age=0"));
1261
1262       if (Firmware_ != NULL)
1263          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_);
1264
1265       sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
1266       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
1267       CFRelease(sr);
1268
1269       if (UniqueID_ != NULL)
1270          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
1271
1272       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.592"));
1273
1274       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1275       CFRelease(hm);
1276
1277 #define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
1278 #define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
1279 #define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
1280
1281       /*SInt32 to(TimeOut);
1282       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));*/
1283       double to(TimeOut);
1284       CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
1285
1286       CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
1287       CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
1288       CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
1289       CFRelease(nm);
1290
1291       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1292       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1293       CFRelease(dr);
1294
1295       //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1296       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1297
1298       FetchResult Res;
1299       CFIndex rd;
1300       UInt32 sc;
1301
1302       uint8_t data[10240];
1303       size_t offset = 0;
1304
1305       Status("Connecting to %s", hs.c_str());
1306
1307       switch (CFReadStreamOpen(rs, to)) {
1308          case -1:
1309             CfrsError("Open", rs);
1310          goto fail;
1311
1312          case 0:
1313             _error->Error("Host Unreachable");
1314             cached.insert(hs);
1315          goto fail;
1316
1317          case 1:
1318             /* success */
1319          break;
1320
1321          fail:
1322             Fail(true);
1323          goto done;
1324       }
1325
1326       rd = CFReadStreamRead(rs, data, sizeof(data));
1327
1328       if (rd == -1) {
1329          CfrsError(uri.Host.c_str(), rs);
1330          cached.insert(hs);
1331          Fail(true);
1332          goto done;
1333       }
1334
1335       Res.Filename = Queue->DestFile;
1336
1337       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1338       sc = CFHTTPMessageGetResponseStatusCode(hm);
1339
1340       if (sc == 301 || sc == 302) {
1341          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1342          if (sr == NULL) {
1343             Fail();
1344             goto done_;
1345          } else {
1346             size_t ln = CFStringGetLength(sr) + 1;
1347             free(url);
1348             url = static_cast<char *>(malloc(ln));
1349
1350             if (!CFStringGetCString(sr, url, ln, se)) {
1351                Fail();
1352                goto done_;
1353             }
1354
1355             CFRelease(sr);
1356             goto url;
1357          }
1358       }
1359
1360       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1361       if (sr != NULL) {
1362          size_t ln = CFStringGetLength(sr) + 1;
1363          char cr[ln];
1364
1365          if (!CFStringGetCString(sr, cr, ln, se)) {
1366             Fail();
1367             goto done_;
1368          }
1369
1370          CFRelease(sr);
1371
1372          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1373             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1374             Fail();
1375             goto done_;
1376          }
1377
1378          if (offset > Res.Size) {
1379             _error->Error(_("This HTTP server has broken range support"));
1380             Fail();
1381             goto done_;
1382          }
1383       } else {
1384          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1385          if (sr != NULL) {
1386             Res.Size = CFStringGetIntValue(sr);
1387             CFRelease(sr);
1388          }
1389       }
1390
1391       time(&Res.LastModified);
1392
1393       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1394       if (sr != NULL) {
1395          size_t ln = CFStringGetLength(sr) + 1;
1396          char cr[ln];
1397
1398          if (!CFStringGetCString(sr, cr, ln, se)) {
1399             Fail();
1400             goto done_;
1401          }
1402
1403          CFRelease(sr);
1404
1405          if (!StrToTime(cr, Res.LastModified)) {
1406             _error->Error(_("Unknown date format"));
1407             Fail();
1408             goto done_;
1409          }
1410       }
1411
1412       if (sc < 200 || sc >= 300 && sc != 304) {
1413          sr = CFHTTPMessageCopyResponseStatusLine(hm);
1414
1415          size_t ln = CFStringGetLength(sr) + 1;
1416          char cr[ln];
1417
1418          if (!CFStringGetCString(sr, cr, ln, se)) {
1419             Fail();
1420             goto done;
1421          }
1422
1423          CFRelease(sr);
1424
1425          _error->Error("%s", cr);
1426
1427          Fail();
1428          goto done_;
1429       }
1430
1431       CFRelease(hm);
1432
1433       if (sc == 304) {
1434          unlink(Queue->DestFile.c_str());
1435          Res.IMSHit = true;
1436          Res.LastModified = Queue->LastModified;
1437          URIDone(Res);
1438       } else {
1439          Hashes hash;
1440
1441          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1442          if (_error->PendingError() == true) {
1443             delete File;
1444             File = NULL;
1445             Fail();
1446             goto done;
1447          }
1448
1449          FailFile = Queue->DestFile;
1450          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1451          FailFd = File->Fd();
1452          FailTime = Res.LastModified;
1453
1454          Res.ResumePoint = offset;
1455          ftruncate(File->Fd(), offset);
1456
1457          if (offset != 0) {
1458             lseek(File->Fd(), 0, SEEK_SET);
1459             if (!hash.AddFD(File->Fd(), offset)) {
1460                _error->Errno("read", _("Problem hashing file"));
1461                delete File;
1462                File = NULL;
1463                Fail();
1464                goto done;
1465             }
1466          }
1467
1468          lseek(File->Fd(), 0, SEEK_END);
1469
1470          URIStart(Res);
1471
1472          read: if (rd == -1) {
1473             CfrsError("rd", rs);
1474             Fail(true);
1475          } else if (rd == 0) {
1476             if (Res.Size == 0)
1477                Res.Size = File->Size();
1478
1479             struct utimbuf UBuf;
1480             time(&UBuf.actime);
1481             UBuf.actime = Res.LastModified;
1482             UBuf.modtime = Res.LastModified;
1483             utime(Queue->DestFile.c_str(), &UBuf);
1484
1485             Res.TakeHashes(hash);
1486             URIDone(Res);
1487          } else {
1488             hash.Add(data, rd);
1489
1490             uint8_t *dt = data;
1491             while (rd != 0) {
1492                int sz = write(File->Fd(), dt, rd);
1493
1494                if (sz == -1) {
1495                    delete File;
1496                    File = NULL;
1497                    Fail();
1498                    goto done;
1499                }
1500
1501                dt += sz;
1502                rd -= sz;
1503             }
1504
1505             rd = CFReadStreamRead(rs, data, sizeof(data));
1506             goto read;
1507          }
1508       }
1509
1510      goto done;
1511     done_:
1512       CFRelease(hm);
1513     done:
1514       CFReadStreamClose(rs);
1515       CFRelease(rs);
1516       free(url);
1517
1518       FailCounter = 0;
1519    }
1520
1521    return 0;
1522 }
1523                                                                         /*}}}*/
1524
1525 int main()
1526 {
1527    setlocale(LC_ALL, "");
1528    // ignore SIGPIPE, this can happen on write() if the socket
1529    // closes the connection (this is dealt with via ServerDie())
1530    signal(SIGPIPE, SIG_IGN);
1531
1532    HttpMethod Mth;
1533
1534     size_t size;
1535     sysctlbyname("hw.machine", NULL, &size, NULL, 0);
1536     char *machine = new char[size];
1537     sysctlbyname("hw.machine", machine, &size, NULL, 0);
1538     Machine_ = machine;
1539
1540     const char *path = "/System/Library/CoreServices/SystemVersion.plist";
1541     CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false);
1542
1543     CFPropertyListRef plist; {
1544         CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url);
1545         CFReadStreamOpen(stream);
1546         plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL);
1547         CFReadStreamClose(stream);
1548     }
1549
1550     CFRelease(url);
1551
1552     if (plist != NULL) {
1553         Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion")));
1554         CFRelease(plist);
1555     }
1556
1557     if (void *lockdown = lockdown_connect()) {
1558         UniqueID_ = lockdown_copy_value(lockdown, NULL, kLockdownUniqueDeviceIDKey);
1559         lockdown_disconnect(lockdown);
1560     }
1561
1562    return Mth.Loop();
1563 }
1564
1565