apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by alot of item methods as thier final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 147 // ---------------------------------------------------------------------
 148 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 149 {
 150    // we only act if a mirror was used at all
 151    if(UsedMirror.empty())
 152       return;
 153 #if 0
 154    std::cerr << "\nReportMirrorFailure: "
 155              << UsedMirror
 156              << " Uri: " << DescURI()
 157              << " FailCode: "
 158              << FailCode << std::endl;
 159 #endif
 160    const char *Args[40];
 161    unsigned int i = 0;
 162    string report = _config->Find("Methods::Mirror::ProblemReporting",
 163                                  "/usr/lib/apt/apt-report-mirror-failure");
 164    if(!FileExists(report))
 165       return;
 166    Args[i++] = report.c_str();
 167    Args[i++] = UsedMirror.c_str();
 168    Args[i++] = DescURI().c_str();
 169    Args[i++] = FailCode.c_str();
 170    Args[i++] = NULL;
 171    pid_t pid = ExecFork();
 172    if(pid < 0)
 173    {
 174       _error->Error("ReportMirrorFailure Fork failed");
 175       return;
 176    }
 177    else if(pid == 0)
 178    {
 179       execvp(Args[0], (char**)Args);
 180       std::cerr << "Could not exec " << Args[0] << std::endl;
 181       _exit(100);
 182    }
 183    if(!ExecWait(pid, "report-mirror-failure"))
 184    {
 185       _error->Warning("Couldn't report problem to '%s'",
 186                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 187    }
 188 }
 189                                                                         /*}}}*/
 190 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* Get the Index file first and see if there are languages available
 193  * If so, create a pkgAcqIndexTrans for the found language(s).
 194  */
 195 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 196                                  string const &URIDesc, string const &ShortDesc,
 197                                  HashString const &ExpectedHash)
 198    : Item(Owner), ExpectedHash(ExpectedHash)
 199 {
 200    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 201
 202    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 203    DestFile += URItoFileName(URI);
 204
 205    Desc.URI = URI;
 206    Desc.Description = URIDesc;
 207    Desc.Owner = this;
 208    Desc.ShortDesc = ShortDesc;
 209
 210    QueueURI(Desc);
 211
 212    if(Debug)
 213       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* The only header we use is the last-modified header. */
 219 string pkgAcqSubIndex::Custom600Headers()
 220 {
 221    string Final = _config->FindDir("Dir::State::lists");
 222    Final += URItoFileName(Desc.URI);
 223
 224    struct stat Buf;
 225    if (stat(Final.c_str(),&Buf) != 0)
 226       return "\nIndex-File: true\nFail-Ignore: true\n";
 227    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 228 }
 229                                                                         /*}}}*/
 230 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 231 {
 232    if(Debug)
 233       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 234
 235    Complete = false;
 236    Status = StatDone;
 237    Dequeue();
 238
 239    // No good Index is provided, so try guessing
 240    std::vector<std::string> langs = APT::Configuration::getLanguages(true);
 241    for (std::vector<std::string>::const_iterator l = langs.begin();
 242         l != langs.end(); ++l)
 243    {
 244       if (*l == "none") continue;
 245       string const file = "Translation-" + *l;
 246       new pkgAcqIndexTrans(Owner, Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file),
 247                 Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file),
 248                 file);
 249    }
 250 }
 251                                                                         /*}}}*/
 252 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 253                            pkgAcquire::MethodConfig *Cnf)
 254 {
 255    if(Debug)
 256       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 257
 258    string FileName = LookupTag(Message,"Filename");
 259    if (FileName.empty() == true)
 260    {
 261       Status = StatError;
 262       ErrorText = "Method gave a blank filename";
 263       return;
 264    }
 265
 266    if (FileName != DestFile)
 267    {
 268       Local = true;
 269       Desc.URI = "copy:" + FileName;
 270       QueueURI(Desc);
 271       return;
 272    }
 273
 274    Item::Done(Message,Size,Md5Hash,Cnf);
 275
 276    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 277
 278    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 279    indexRecords SubIndexParser;
 280    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 281       Status = StatError;
 282       ErrorText = SubIndexParser.ErrorText;
 283       return;
 284    }
 285
 286    // sucess in downloading the index
 287    // rename the index
 288    if(Debug)
 289       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 290    Rename(DestFile,FinalFile);
 291    chmod(FinalFile.c_str(),0644);
 292    DestFile = FinalFile;
 293
 294    if(ParseIndex(DestFile) == false)
 295       return Failed("", NULL);
 296
 297    Complete = true;
 298    Status = StatDone;
 299    Dequeue();
 300    return;
 301 }
 302                                                                         /*}}}*/
 303 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 304 {
 305    indexRecords SubIndexParser;
 306    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 307       return false;
 308
 309    std::vector<std::string> lang = APT::Configuration::getLanguages(true);
 310    for (std::vector<std::string>::const_iterator l = lang.begin();
 311         l != lang.end(); ++l)
 312    {
 313       if (*l == "none")
 314          continue;
 315
 316       string file = "Translation-" + *l;
 317       indexRecords::checkSum const *Record = SubIndexParser.Lookup(file);
 318       HashString expected;
 319       if (Record == NULL)
 320       {
 321          // FIXME: the Index file provided by debian currently only includes bz2 records
 322          Record = SubIndexParser.Lookup(file + ".bz2");
 323          if (Record == NULL)
 324             continue;
 325       }
 326       else
 327       {
 328          expected = Record->Hash;
 329          if (expected.empty() == true)
 330             continue;
 331       }
 332
 333       IndexTarget target;
 334       target.Description = Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file);
 335       target.MetaKey = file;
 336       target.ShortDesc = file;
 337       target.URI = Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file);
 338       new pkgAcqIndexTrans(Owner, &target, expected, &SubIndexParser);
 339    }
 340    return true;
 341 }
 342                                                                         /*}}}*/
 343 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 344 // ---------------------------------------------------------------------
 345 /* Get the DiffIndex file first and see if there are patches availabe
 346  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 347  * patches. If anything goes wrong in that process, it will fall back to
 348  * the original packages file
 349  */
 350 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 351                                  string URI,string URIDesc,string ShortDesc,
 352                                  HashString ExpectedHash)
 353    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 354      Description(URIDesc)
 355 {
 356
 357    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 358
 359    Desc.Description = URIDesc + "/DiffIndex";
 360    Desc.Owner = this;
 361    Desc.ShortDesc = ShortDesc;
 362    Desc.URI = URI + ".diff/Index";
 363
 364    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 365    DestFile += URItoFileName(URI) + string(".DiffIndex");
 366
 367    if(Debug)
 368       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 369
 370    // look for the current package file
 371    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 372    CurrentPackagesFile += URItoFileName(RealURI);
 373
 374    // FIXME: this file:/ check is a hack to prevent fetching
 375    //        from local sources. this is really silly, and
 376    //        should be fixed cleanly as soon as possible
 377    if(!FileExists(CurrentPackagesFile) ||
 378       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 379    {
 380       // we don't have a pkg file or we don't want to queue
 381       if(Debug)
 382          std::clog << "No index file, local or canceld by user" << std::endl;
 383       Failed("", NULL);
 384       return;
 385    }
 386
 387    if(Debug)
 388       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 389                 << CurrentPackagesFile << std::endl;
 390
 391    QueueURI(Desc);
 392
 393 }
 394                                                                         /*}}}*/
 395 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 396 // ---------------------------------------------------------------------
 397 /* The only header we use is the last-modified header. */
 398 string pkgAcqDiffIndex::Custom600Headers()
 399 {
 400    string Final = _config->FindDir("Dir::State::lists");
 401    Final += URItoFileName(RealURI) + string(".IndexDiff");
 402
 403    if(Debug)
 404       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 405
 406    struct stat Buf;
 407    if (stat(Final.c_str(),&Buf) != 0)
 408       return "\nIndex-File: true";
 409
 410    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 411 }
 412                                                                         /*}}}*/
 413 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 414 {
 415    if(Debug)
 416       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 417                 << std::endl;
 418
 419    pkgTagSection Tags;
 420    string ServerSha1;
 421    vector<DiffInfo> available_patches;
 422
 423    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 424    pkgTagFile TF(&Fd);
 425    if (_error->PendingError() == true)
 426       return false;
 427
 428    if(TF.Step(Tags) == true)
 429    {
 430       bool found = false;
 431       DiffInfo d;
 432       string size;
 433
 434       string const tmp = Tags.FindS("SHA1-Current");
 435       std::stringstream ss(tmp);
 436       ss >> ServerSha1 >> size;
 437       unsigned long const ServerSize = atol(size.c_str());
 438
 439       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 440       SHA1Summation SHA1;
 441       SHA1.AddFD(fd);
 442       string const local_sha1 = SHA1.Result();
 443
 444       if(local_sha1 == ServerSha1)
 445       {
 446          // we have the same sha1 as the server
 447          if(Debug)
 448             std::clog << "Package file is up-to-date" << std::endl;
 449          // set found to true, this will queue a pkgAcqIndexDiffs with
 450          // a empty availabe_patches
 451          found = true;
 452       }
 453       else
 454       {
 455          if(Debug)
 456             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 457
 458          // check the historie and see what patches we need
 459          string const history = Tags.FindS("SHA1-History");
 460          std::stringstream hist(history);
 461          while(hist >> d.sha1 >> size >> d.file)
 462          {
 463             // read until the first match is found
 464             // from that point on, we probably need all diffs
 465             if(d.sha1 == local_sha1)
 466                found=true;
 467             else if (found == false)
 468                continue;
 469
 470             if(Debug)
 471                std::clog << "Need to get diff: " << d.file << std::endl;
 472             available_patches.push_back(d);
 473          }
 474
 475          if (available_patches.empty() == false)
 476          {
 477             // patching with too many files is rather slow compared to a fast download
 478             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 479             if (fileLimit != 0 && fileLimit < available_patches.size())
 480             {
 481                if (Debug)
 482                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 483                         << ") so fallback to complete download" << std::endl;
 484                return false;
 485             }
 486
 487             // see if the patches are too big
 488             found = false; // it was true and it will be true again at the end
 489             d = *available_patches.begin();
 490             string const firstPatch = d.file;
 491             unsigned long patchesSize = 0;
 492             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 493             while(patches >> d.sha1 >> size >> d.file)
 494             {
 495                if (firstPatch == d.file)
 496                   found = true;
 497                else if (found == false)
 498                   continue;
 499
 500                patchesSize += atol(size.c_str());
 501             }
 502             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 503             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 504             {
 505                if (Debug)
 506                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 507                         << ") so fallback to complete download" << std::endl;
 508                return false;
 509             }
 510          }
 511       }
 512
 513       // we have something, queue the next diff
 514       if(found)
 515       {
 516          // queue the diffs
 517          string::size_type const last_space = Description.rfind(" ");
 518          if(last_space != string::npos)
 519             Description.erase(last_space, Description.size()-last_space);
 520          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 521                               ExpectedHash, ServerSha1, available_patches);
 522          Complete = false;
 523          Status = StatDone;
 524          Dequeue();
 525          return true;
 526       }
 527    }
 528
 529    // Nothing found, report and return false
 530    // Failing here is ok, if we return false later, the full
 531    // IndexFile is queued
 532    if(Debug)
 533       std::clog << "Can't find a patch in the index file" << std::endl;
 534    return false;
 535 }
 536                                                                         /*}}}*/
 537 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 538 {
 539    if(Debug)
 540       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 541                 << "Falling back to normal index file aquire" << std::endl;
 542
 543    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 544                    ExpectedHash);
 545
 546    Complete = false;
 547    Status = StatDone;
 548    Dequeue();
 549 }
 550                                                                         /*}}}*/
 551 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 552                            pkgAcquire::MethodConfig *Cnf)
 553 {
 554    if(Debug)
 555       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 556
 557    Item::Done(Message,Size,Md5Hash,Cnf);
 558
 559    string FinalFile;
 560    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 561
 562    // sucess in downloading the index
 563    // rename the index
 564    FinalFile += string(".IndexDiff");
 565    if(Debug)
 566       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 567                 << std::endl;
 568    Rename(DestFile,FinalFile);
 569    chmod(FinalFile.c_str(),0644);
 570    DestFile = FinalFile;
 571
 572    if(!ParseDiffIndex(DestFile))
 573       return Failed("", NULL);
 574
 575    Complete = true;
 576    Status = StatDone;
 577    Dequeue();
 578    return;
 579 }
 580                                                                         /*}}}*/
 581 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* The package diff is added to the queue. one object is constructed
 584  * for each diff and the index
 585  */
 586 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 587                                    string URI,string URIDesc,string ShortDesc,
 588                                    HashString ExpectedHash,
 589                                    string ServerSha1,
 590                                    vector<DiffInfo> diffs)
 591    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 592      available_patches(diffs), ServerSha1(ServerSha1)
 593 {
 594
 595    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 596    DestFile += URItoFileName(URI);
 597
 598    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 599
 600    Description = URIDesc;
 601    Desc.Owner = this;
 602    Desc.ShortDesc = ShortDesc;
 603
 604    if(available_patches.size() == 0)
 605    {
 606       // we are done (yeah!)
 607       Finish(true);
 608    }
 609    else
 610    {
 611       // get the next diff
 612       State = StateFetchDiff;
 613       QueueNextDiff();
 614    }
 615 }
 616                                                                         /*}}}*/
 617 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 618 {
 619    if(Debug)
 620       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 621                 << "Falling back to normal index file aquire" << std::endl;
 622    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 623                    ExpectedHash);
 624    Finish();
 625 }
 626                                                                         /*}}}*/
 627 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 628 void pkgAcqIndexDiffs::Finish(bool allDone)
 629 {
 630    // we restore the original name, this is required, otherwise
 631    // the file will be cleaned
 632    if(allDone)
 633    {
 634       DestFile = _config->FindDir("Dir::State::lists");
 635       DestFile += URItoFileName(RealURI);
 636
 637       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 638       {
 639          Status = StatAuthError;
 640          ErrorText = _("MD5Sum mismatch");
 641          Rename(DestFile,DestFile + ".FAILED");
 642          Dequeue();
 643          return;
 644       }
 645
 646       // this is for the "real" finish
 647       Complete = true;
 648       Status = StatDone;
 649       Dequeue();
 650       if(Debug)
 651          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 652       return;
 653    }
 654
 655    if(Debug)
 656       std::clog << "Finishing: " << Desc.URI << std::endl;
 657    Complete = false;
 658    Status = StatDone;
 659    Dequeue();
 660    return;
 661 }
 662                                                                         /*}}}*/
 663 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 664 {
 665
 666    // calc sha1 of the just patched file
 667    string FinalFile = _config->FindDir("Dir::State::lists");
 668    FinalFile += URItoFileName(RealURI);
 669
 670    FileFd fd(FinalFile, FileFd::ReadOnly);
 671    SHA1Summation SHA1;
 672    SHA1.AddFD(fd);
 673    string local_sha1 = string(SHA1.Result());
 674    if(Debug)
 675       std::clog << "QueueNextDiff: "
 676                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 677
 678    // final file reached before all patches are applied
 679    if(local_sha1 == ServerSha1)
 680    {
 681       Finish(true);
 682       return true;
 683    }
 684
 685    // remove all patches until the next matching patch is found
 686    // this requires the Index file to be ordered
 687    for(vector<DiffInfo>::iterator I=available_patches.begin();
 688        available_patches.empty() == false &&
 689           I != available_patches.end() &&
 690           I->sha1 != local_sha1;
 691        ++I)
 692    {
 693       available_patches.erase(I);
 694    }
 695
 696    // error checking and falling back if no patch was found
 697    if(available_patches.empty() == true)
 698    {
 699       Failed("", NULL);
 700       return false;
 701    }
 702
 703    // queue the right diff
 704    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 705    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 706    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 707    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 708
 709    if(Debug)
 710       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 711
 712    QueueURI(Desc);
 713
 714    return true;
 715 }
 716                                                                         /*}}}*/
 717 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 718                             pkgAcquire::MethodConfig *Cnf)
 719 {
 720    if(Debug)
 721       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 722
 723    Item::Done(Message,Size,Md5Hash,Cnf);
 724
 725    string FinalFile;
 726    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 727
 728    // sucess in downloading a diff, enter ApplyDiff state
 729    if(State == StateFetchDiff)
 730    {
 731
 732       // rred excepts the patch as $FinalFile.ed
 733       Rename(DestFile,FinalFile+".ed");
 734
 735       if(Debug)
 736          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 737
 738       State = StateApplyDiff;
 739       Local = true;
 740       Desc.URI = "rred:" + FinalFile;
 741       QueueURI(Desc);
 742       Mode = "rred";
 743       return;
 744    }
 745
 746
 747    // success in download/apply a diff, queue next (if needed)
 748    if(State == StateApplyDiff)
 749    {
 750       // remove the just applied patch
 751       available_patches.erase(available_patches.begin());
 752
 753       // move into place
 754       if(Debug)
 755       {
 756          std::clog << "Moving patched file in place: " << std::endl
 757                    << DestFile << " -> " << FinalFile << std::endl;
 758       }
 759       Rename(DestFile,FinalFile);
 760       chmod(FinalFile.c_str(),0644);
 761
 762       // see if there is more to download
 763       if(available_patches.empty() == false) {
 764          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 765                               ExpectedHash, ServerSha1, available_patches);
 766          return Finish();
 767       } else
 768          return Finish(true);
 769    }
 770 }
 771                                                                         /*}}}*/
 772 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 773 // ---------------------------------------------------------------------
 774 /* The package file is added to the queue and a second class is
 775    instantiated to fetch the revision file */
 776 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 777                          string URI,string URIDesc,string ShortDesc,
 778                          HashString ExpectedHash, string comprExt)
 779    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 780 {
 781    if(comprExt.empty() == true)
 782    {
 783       // autoselect the compression method
 784       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 785       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 786          comprExt.append(*t).append(" ");
 787       if (comprExt.empty() == false)
 788          comprExt.erase(comprExt.end()-1);
 789    }
 790    CompressionExtension = comprExt;
 791
 792    Init(URI, URIDesc, ShortDesc);
 793 }
 794 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 795                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 796    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 797 {
 798    // autoselect the compression method
 799    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 800    CompressionExtension = "";
 801    if (ExpectedHash.empty() == false)
 802    {
 803       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 804          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 805             CompressionExtension.append(*t).append(" ");
 806    }
 807    else
 808    {
 809       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 810          CompressionExtension.append(*t).append(" ");
 811    }
 812    if (CompressionExtension.empty() == false)
 813       CompressionExtension.erase(CompressionExtension.end()-1);
 814
 815    // only verify non-optional targets, see acquire-item.h for a FIXME
 816    // to make this more flexible
 817    if (Target->IsOptional())
 818      Verify = false;
 819    else
 820      Verify = true;
 821
 822    Init(Target->URI, Target->Description, Target->ShortDesc);
 823 }
 824                                                                         /*}}}*/
 825 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 826 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 827    Decompression = false;
 828    Erase = false;
 829
 830    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 831    DestFile += URItoFileName(URI);
 832
 833    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 834    if (comprExt == "uncompressed")
 835       Desc.URI = URI;
 836    else
 837       Desc.URI = URI + '.' + comprExt;
 838
 839    Desc.Description = URIDesc;
 840    Desc.Owner = this;
 841    Desc.ShortDesc = ShortDesc;
 842
 843    QueueURI(Desc);
 844 }
 845                                                                         /*}}}*/
 846 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* The only header we use is the last-modified header. */
 849 string pkgAcqIndex::Custom600Headers()
 850 {
 851    string Final = _config->FindDir("Dir::State::lists");
 852    Final += URItoFileName(RealURI);
 853    if (_config->FindB("Acquire::GzipIndexes",false))
 854       Final += ".gz";
 855
 856    string msg = "\nIndex-File: true";
 857    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 858    //        seems to be difficult without breaking ABI
 859    if (ShortDesc().find("Translation") != 0)
 860       msg += "\nFail-Ignore: true";
 861    struct stat Buf;
 862    if (stat(Final.c_str(),&Buf) == 0)
 863       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 864
 865    return msg;
 866 }
 867                                                                         /*}}}*/
 868 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 869 {
 870    size_t const nextExt = CompressionExtension.find(' ');
 871    if (nextExt != std::string::npos)
 872    {
 873       CompressionExtension = CompressionExtension.substr(nextExt+1);
 874       Init(RealURI, Desc.Description, Desc.ShortDesc);
 875       return;
 876    }
 877
 878    // on decompression failure, remove bad versions in partial/
 879    if (Decompression && Erase) {
 880       string s = _config->FindDir("Dir::State::lists") + "partial/";
 881       s.append(URItoFileName(RealURI));
 882       unlink(s.c_str());
 883    }
 884
 885    Item::Failed(Message,Cnf);
 886 }
 887                                                                         /*}}}*/
 888 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 889 // ---------------------------------------------------------------------
 890 /* This goes through a number of states.. On the initial fetch the
 891    method could possibly return an alternate filename which points
 892    to the uncompressed version of the file. If this is so the file
 893    is copied into the partial directory. In all other cases the file
 894    is decompressed with a gzip uri. */
 895 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
 896                        pkgAcquire::MethodConfig *Cfg)
 897 {
 898    Item::Done(Message,Size,Hash,Cfg);
 899
 900    if (Decompression == true)
 901    {
 902       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 903       {
 904          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 905          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 906       }
 907
 908       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 909       {
 910          Status = StatAuthError;
 911          ErrorText = _("Hash Sum mismatch");
 912          Rename(DestFile,DestFile + ".FAILED");
 913          ReportMirrorFailure("HashChecksumFailure");
 914          return;
 915       }
 916
 917       /* Verify the index file for correctness (all indexes must
 918        * have a Package field) (LP: #346386) (Closes: #627642) */
 919       if (Verify == true)
 920       {
 921          FileFd fd(DestFile, FileFd::ReadOnly);
 922          pkgTagSection sec;
 923          pkgTagFile tag(&fd);
 924
 925          // Only test for correctness if the file is not empty (empty is ok)
 926          if (fd.Size() > 0) {
 927             if (_error->PendingError() || !tag.Step(sec)) {
 928                Status = StatError;
 929                _error->DumpErrors();
 930                Rename(DestFile,DestFile + ".FAILED");
 931                return;
 932             } else if (!sec.Exists("Package")) {
 933                Status = StatError;
 934                ErrorText = ("Encountered a section with no Package: header");
 935                Rename(DestFile,DestFile + ".FAILED");
 936                return;
 937             }
 938          }
 939       }
 940
 941       // Done, move it into position
 942       string FinalFile = _config->FindDir("Dir::State::lists");
 943       FinalFile += URItoFileName(RealURI);
 944       Rename(DestFile,FinalFile);
 945       chmod(FinalFile.c_str(),0644);
 946
 947       /* We restore the original name to DestFile so that the clean operation
 948          will work OK */
 949       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 950       DestFile += URItoFileName(RealURI);
 951
 952       // Remove the compressed version.
 953       if (Erase == true)
 954          unlink(DestFile.c_str());
 955       return;
 956    }
 957
 958    Erase = false;
 959    Complete = true;
 960
 961    // Handle the unzipd case
 962    string FileName = LookupTag(Message,"Alt-Filename");
 963    if (FileName.empty() == false)
 964    {
 965       // The files timestamp matches
 966       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 967          return;
 968       Decompression = true;
 969       Local = true;
 970       DestFile += ".decomp";
 971       Desc.URI = "copy:" + FileName;
 972       QueueURI(Desc);
 973       Mode = "copy";
 974       return;
 975    }
 976
 977    FileName = LookupTag(Message,"Filename");
 978    if (FileName.empty() == true)
 979    {
 980       Status = StatError;
 981       ErrorText = "Method gave a blank filename";
 982    }
 983
 984    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 985
 986    // The files timestamp matches
 987    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 988        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 989           // Update DestFile for .gz suffix so that the clean operation keeps it
 990           DestFile += ".gz";
 991       return;
 992     }
 993
 994    if (FileName == DestFile)
 995       Erase = true;
 996    else
 997       Local = true;
 998
 999    string decompProg;
1000
1001    // If we enable compressed indexes and already have gzip, keep it
1002    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1003       string FinalFile = _config->FindDir("Dir::State::lists");
1004       FinalFile += URItoFileName(RealURI) + ".gz";
1005       Rename(DestFile,FinalFile);
1006       chmod(FinalFile.c_str(),0644);
1007
1008       // Update DestFile for .gz suffix so that the clean operation keeps it
1009       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1010       DestFile += URItoFileName(RealURI) + ".gz";
1011       return;
1012     }
1013
1014    // get the binary name for your used compression type
1015    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1016    if(decompProg.empty() == false);
1017    else if(compExt == "uncompressed")
1018       decompProg = "copy";
1019    else {
1020       _error->Error("Unsupported extension: %s", compExt.c_str());
1021       return;
1022    }
1023
1024    Decompression = true;
1025    DestFile += ".decomp";
1026    Desc.URI = decompProg + ":" + FileName;
1027    QueueURI(Desc);
1028    Mode = decompProg.c_str();
1029 }
1030                                                                         /*}}}*/
1031 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1032 // ---------------------------------------------------------------------
1033 /* The Translation file is added to the queue */
1034 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1035                             string URI,string URIDesc,string ShortDesc)
1036   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1037 {
1038 }
1039 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1040                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1041   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1042 {
1043 }
1044                                                                         /*}}}*/
1045 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1046 // ---------------------------------------------------------------------
1047 string pkgAcqIndexTrans::Custom600Headers()
1048 {
1049    string Final = _config->FindDir("Dir::State::lists");
1050    Final += URItoFileName(RealURI);
1051
1052    struct stat Buf;
1053    if (stat(Final.c_str(),&Buf) != 0)
1054       return "\nFail-Ignore: true\nIndex-File: true";
1055    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1056 }
1057                                                                         /*}}}*/
1058 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1059 // ---------------------------------------------------------------------
1060 /* */
1061 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1062 {
1063    size_t const nextExt = CompressionExtension.find(' ');
1064    if (nextExt != std::string::npos)
1065    {
1066       CompressionExtension = CompressionExtension.substr(nextExt+1);
1067       Init(RealURI, Desc.Description, Desc.ShortDesc);
1068       Status = StatIdle;
1069       return;
1070    }
1071
1072    if (Cnf->LocalOnly == true ||
1073        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1074    {
1075       // Ignore this
1076       Status = StatDone;
1077       Complete = false;
1078       Dequeue();
1079       return;
1080    }
1081
1082    Item::Failed(Message,Cnf);
1083 }
1084                                                                         /*}}}*/
1085 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1086                              string URI,string URIDesc,string ShortDesc,
1087                              string MetaIndexURI, string MetaIndexURIDesc,
1088                              string MetaIndexShortDesc,
1089                              const vector<IndexTarget*>* IndexTargets,
1090                              indexRecords* MetaIndexParser) :
1091    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1092    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1093    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1094 {
1095    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1096    DestFile += URItoFileName(URI);
1097
1098    // remove any partial downloaded sig-file in partial/.
1099    // it may confuse proxies and is too small to warrant a
1100    // partial download anyway
1101    unlink(DestFile.c_str());
1102
1103    // Create the item
1104    Desc.Description = URIDesc;
1105    Desc.Owner = this;
1106    Desc.ShortDesc = ShortDesc;
1107    Desc.URI = URI;
1108
1109    string Final = _config->FindDir("Dir::State::lists");
1110    Final += URItoFileName(RealURI);
1111    struct stat Buf;
1112    if (stat(Final.c_str(),&Buf) == 0)
1113    {
1114       // File was already in place.  It needs to be re-downloaded/verified
1115       // because Release might have changed, we do give it a differnt
1116       // name than DestFile because otherwise the http method will
1117       // send If-Range requests and there are too many broken servers
1118       // out there that do not understand them
1119       LastGoodSig = DestFile+".reverify";
1120       Rename(Final,LastGoodSig);
1121    }
1122
1123    QueueURI(Desc);
1124 }
1125                                                                         /*}}}*/
1126 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1127 // ---------------------------------------------------------------------
1128 /* The only header we use is the last-modified header. */
1129 string pkgAcqMetaSig::Custom600Headers()
1130 {
1131    struct stat Buf;
1132    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1133       return "\nIndex-File: true";
1134
1135    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1136 }
1137
1138 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1139                          pkgAcquire::MethodConfig *Cfg)
1140 {
1141    Item::Done(Message,Size,MD5,Cfg);
1142
1143    string FileName = LookupTag(Message,"Filename");
1144    if (FileName.empty() == true)
1145    {
1146       Status = StatError;
1147       ErrorText = "Method gave a blank filename";
1148       return;
1149    }
1150
1151    if (FileName != DestFile)
1152    {
1153       // We have to copy it into place
1154       Local = true;
1155       Desc.URI = "copy:" + FileName;
1156       QueueURI(Desc);
1157       return;
1158    }
1159
1160    Complete = true;
1161
1162    // put the last known good file back on i-m-s hit (it will
1163    // be re-verified again)
1164    // Else do nothing, we have the new file in DestFile then
1165    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1166       Rename(LastGoodSig, DestFile);
1167
1168    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1169    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1170                        MetaIndexShortDesc,  DestFile, IndexTargets,
1171                        MetaIndexParser);
1172
1173 }
1174                                                                         /*}}}*/
1175 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1176 {
1177    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1178
1179    // if we get a network error we fail gracefully
1180    if(Status == StatTransientNetworkError)
1181    {
1182       Item::Failed(Message,Cnf);
1183       // move the sigfile back on transient network failures
1184       if(FileExists(LastGoodSig))
1185          Rename(LastGoodSig,Final);
1186
1187       // set the status back to , Item::Failed likes to reset it
1188       Status = pkgAcquire::Item::StatTransientNetworkError;
1189       return;
1190    }
1191
1192    // Delete any existing sigfile when the acquire failed
1193    unlink(Final.c_str());
1194
1195    // queue a pkgAcqMetaIndex with no sigfile
1196    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1197                        "", IndexTargets, MetaIndexParser);
1198
1199    if (Cnf->LocalOnly == true ||
1200        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1201    {
1202       // Ignore this
1203       Status = StatDone;
1204       Complete = false;
1205       Dequeue();
1206       return;
1207    }
1208
1209    Item::Failed(Message,Cnf);
1210 }
1211                                                                         /*}}}*/
1212 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1213                                  string URI,string URIDesc,string ShortDesc,
1214                                  string SigFile,
1215                                  const vector<struct IndexTarget*>* IndexTargets,
1216                                  indexRecords* MetaIndexParser) :
1217    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1218    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1219 {
1220    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1221    DestFile += URItoFileName(URI);
1222
1223    // Create the item
1224    Desc.Description = URIDesc;
1225    Desc.Owner = this;
1226    Desc.ShortDesc = ShortDesc;
1227    Desc.URI = URI;
1228
1229    QueueURI(Desc);
1230 }
1231                                                                         /*}}}*/
1232 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1233 // ---------------------------------------------------------------------
1234 /* The only header we use is the last-modified header. */
1235 string pkgAcqMetaIndex::Custom600Headers()
1236 {
1237    string Final = _config->FindDir("Dir::State::lists");
1238    Final += URItoFileName(RealURI);
1239
1240    struct stat Buf;
1241    if (stat(Final.c_str(),&Buf) != 0)
1242       return "\nIndex-File: true";
1243
1244    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1245 }
1246                                                                         /*}}}*/
1247 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1248                            pkgAcquire::MethodConfig *Cfg)
1249 {
1250    Item::Done(Message,Size,Hash,Cfg);
1251
1252    // MetaIndexes are done in two passes: one to download the
1253    // metaindex with an appropriate method, and a second to verify it
1254    // with the gpgv method
1255
1256    if (AuthPass == true)
1257    {
1258       AuthDone(Message);
1259
1260       // all cool, move Release file into place
1261       Complete = true;
1262    }
1263    else
1264    {
1265       RetrievalDone(Message);
1266       if (!Complete)
1267          // Still more retrieving to do
1268          return;
1269
1270       if (SigFile == "")
1271       {
1272          // There was no signature file, so we are finished.  Download
1273          // the indexes and do only hashsum verification if possible
1274          MetaIndexParser->Load(DestFile);
1275          QueueIndexes(false);
1276       }
1277       else
1278       {
1279          // There was a signature file, so pass it to gpgv for
1280          // verification
1281
1282          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1283             std::cerr << "Metaindex acquired, queueing gpg verification ("
1284                       << SigFile << "," << DestFile << ")\n";
1285          AuthPass = true;
1286          Desc.URI = "gpgv:" + SigFile;
1287          QueueURI(Desc);
1288          Mode = "gpgv";
1289          return;
1290       }
1291    }
1292
1293    if (Complete == true)
1294    {
1295       string FinalFile = _config->FindDir("Dir::State::lists");
1296       FinalFile += URItoFileName(RealURI);
1297       if (SigFile == DestFile)
1298          SigFile = FinalFile;
1299       Rename(DestFile,FinalFile);
1300       chmod(FinalFile.c_str(),0644);
1301       DestFile = FinalFile;
1302    }
1303 }
1304                                                                         /*}}}*/
1305 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1306 {
1307    // We have just finished downloading a Release file (it is not
1308    // verified yet)
1309
1310    string FileName = LookupTag(Message,"Filename");
1311    if (FileName.empty() == true)
1312    {
1313       Status = StatError;
1314       ErrorText = "Method gave a blank filename";
1315       return;
1316    }
1317
1318    if (FileName != DestFile)
1319    {
1320       Local = true;
1321       Desc.URI = "copy:" + FileName;
1322       QueueURI(Desc);
1323       return;
1324    }
1325
1326    // make sure to verify against the right file on I-M-S hit
1327    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1328    if(IMSHit)
1329    {
1330       string FinalFile = _config->FindDir("Dir::State::lists");
1331       FinalFile += URItoFileName(RealURI);
1332       if (SigFile == DestFile)
1333          SigFile = FinalFile;
1334       DestFile = FinalFile;
1335    }
1336    Complete = true;
1337 }
1338                                                                         /*}}}*/
1339 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1340 {
1341    // At this point, the gpgv method has succeeded, so there is a
1342    // valid signature from a key in the trusted keyring.  We
1343    // perform additional verification of its contents, and use them
1344    // to verify the indexes we are about to download
1345
1346    if (!MetaIndexParser->Load(DestFile))
1347    {
1348       Status = StatAuthError;
1349       ErrorText = MetaIndexParser->ErrorText;
1350       return;
1351    }
1352
1353    if (!VerifyVendor(Message))
1354    {
1355       return;
1356    }
1357
1358    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1359       std::cerr << "Signature verification succeeded: "
1360                 << DestFile << std::endl;
1361
1362    // Download further indexes with verification
1363    QueueIndexes(true);
1364
1365    // is it a clearsigned MetaIndex file?
1366    if (DestFile == SigFile)
1367       return;
1368
1369    // Done, move signature file into position
1370    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1371       URItoFileName(RealURI) + ".gpg";
1372    Rename(SigFile,VerifiedSigFile);
1373    chmod(VerifiedSigFile.c_str(),0644);
1374 }
1375                                                                         /*}}}*/
1376 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1377 {
1378 #if 0
1379    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1380     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1381    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1382    {
1383       Status = StatError;
1384       ErrorText = MetaIndexParser->ErrorText;
1385       return;
1386    }
1387 #endif
1388    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1389         Target != IndexTargets->end();
1390         ++Target)
1391    {
1392       HashString ExpectedIndexHash;
1393       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1394       if (Record == NULL)
1395       {
1396          if (verify == true && (*Target)->IsOptional() == false)
1397          {
1398             Status = StatAuthError;
1399             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1400             return;
1401          }
1402       }
1403       else
1404       {
1405          ExpectedIndexHash = Record->Hash;
1406          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1407          {
1408             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1409             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1410             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1411          }
1412          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1413          {
1414             Status = StatAuthError;
1415             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1416             return;
1417          }
1418       }
1419
1420       if ((*Target)->IsOptional() == true)
1421       {
1422          if ((*Target)->IsSubIndex() == true)
1423             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1424                                 (*Target)->ShortDesc, ExpectedIndexHash);
1425          else
1426             new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1427          continue;
1428       }
1429
1430       /* Queue Packages file (either diff or full packages files, depending
1431          on the users option) - we also check if the PDiff Index file is listed
1432          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1433          instead, but passing the required info to it is to much hassle */
1434       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1435           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1436          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1437                              (*Target)->ShortDesc, ExpectedIndexHash);
1438       else
1439          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1440    }
1441 }
1442                                                                         /*}}}*/
1443 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1444 {
1445    string::size_type pos;
1446
1447    // check for missing sigs (that where not fatal because otherwise we had
1448    // bombed earlier)
1449    string missingkeys;
1450    string msg = _("There is no public key available for the "
1451                   "following key IDs:\n");
1452    pos = Message.find("NO_PUBKEY ");
1453    if (pos != std::string::npos)
1454    {
1455       string::size_type start = pos+strlen("NO_PUBKEY ");
1456       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1457       missingkeys += (Fingerprint);
1458    }
1459    if(!missingkeys.empty())
1460       _error->Warning("%s", string(msg+missingkeys).c_str());
1461
1462    string Transformed = MetaIndexParser->GetExpectedDist();
1463
1464    if (Transformed == "../project/experimental")
1465    {
1466       Transformed = "experimental";
1467    }
1468
1469    pos = Transformed.rfind('/');
1470    if (pos != string::npos)
1471    {
1472       Transformed = Transformed.substr(0, pos);
1473    }
1474
1475    if (Transformed == ".")
1476    {
1477       Transformed = "";
1478    }
1479
1480    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1481        MetaIndexParser->GetValidUntil() > 0) {
1482       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1483       if (invalid_since > 0)
1484          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1485          // the time since then the file is invalid - formated in the same way as in
1486          // the download progress display (e.g. 7d 3h 42min 1s)
1487          return _error->Error(
1488             _("Release file for %s is expired (invalid since %s). "
1489               "Updates for this repository will not be applied."),
1490             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1491    }
1492
1493    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1494    {
1495       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1496       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1497       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1498    }
1499
1500    if (MetaIndexParser->CheckDist(Transformed) == false)
1501    {
1502       // This might become fatal one day
1503 //       Status = StatAuthError;
1504 //       ErrorText = "Conflicting distribution; expected "
1505 //          + MetaIndexParser->GetExpectedDist() + " but got "
1506 //          + MetaIndexParser->GetDist();
1507 //       return false;
1508       if (!Transformed.empty())
1509       {
1510          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1511                          Desc.Description.c_str(),
1512                          Transformed.c_str(),
1513                          MetaIndexParser->GetDist().c_str());
1514       }
1515    }
1516
1517    return true;
1518 }
1519                                                                         /*}}}*/
1520 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1521 // ---------------------------------------------------------------------
1522 /* */
1523 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1524 {
1525    if (AuthPass == true)
1526    {
1527       // gpgv method failed, if we have a good signature
1528       string LastGoodSigFile = _config->FindDir("Dir::State::lists");
1529       if (DestFile == SigFile)
1530          LastGoodSigFile.append(URItoFileName(RealURI));
1531       else
1532          LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify");
1533
1534       if(FileExists(LastGoodSigFile))
1535       {
1536          if (DestFile != SigFile)
1537          {
1538             string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1539                                         URItoFileName(RealURI) + ".gpg";
1540             Rename(LastGoodSigFile,VerifiedSigFile);
1541          }
1542          Status = StatTransientNetworkError;
1543          _error->Warning(_("A error occurred during the signature "
1544                            "verification. The repository is not updated "
1545                            "and the previous index files will be used. "
1546                            "GPG error: %s: %s\n"),
1547                          Desc.Description.c_str(),
1548                          LookupTag(Message,"Message").c_str());
1549          RunScripts("APT::Update::Auth-Failure");
1550          return;
1551       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1552          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1553          _error->Error(_("GPG error: %s: %s"),
1554                          Desc.Description.c_str(),
1555                          LookupTag(Message,"Message").c_str());
1556          return;
1557       } else {
1558          _error->Warning(_("GPG error: %s: %s"),
1559                          Desc.Description.c_str(),
1560                          LookupTag(Message,"Message").c_str());
1561       }
1562       // gpgv method failed
1563       ReportMirrorFailure("GPGFailure");
1564    }
1565
1566    /* Always move the meta index, even if gpgv failed. This ensures
1567     * that PackageFile objects are correctly filled in */
1568    if (FileExists(DestFile)) {
1569       string FinalFile = _config->FindDir("Dir::State::lists");
1570       FinalFile += URItoFileName(RealURI);
1571       /* InRelease files become Release files, otherwise
1572        * they would be considered as trusted later on */
1573       if (SigFile == DestFile) {
1574          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1575                                        "Release");
1576          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1577                                        "Release");
1578          SigFile = FinalFile;
1579       }
1580       Rename(DestFile,FinalFile);
1581       chmod(FinalFile.c_str(),0644);
1582
1583       DestFile = FinalFile;
1584    }
1585
1586    // No Release file was present, or verification failed, so fall
1587    // back to queueing Packages files without verification
1588    QueueIndexes(false);
1589 }
1590                                                                         /*}}}*/
1591 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1592                 string const &URI, string const &URIDesc, string const &ShortDesc,
1593                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1594                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1595                 const vector<struct IndexTarget*>* IndexTargets,
1596                 indexRecords* MetaIndexParser) :
1597         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1598         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1599         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1600 {
1601    SigFile = DestFile;
1602 }
1603                                                                         /*}}}*/
1604 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1605 // ---------------------------------------------------------------------
1606 // FIXME: this can go away once the InRelease file is used widely
1607 string pkgAcqMetaClearSig::Custom600Headers()
1608 {
1609    string Final = _config->FindDir("Dir::State::lists");
1610    Final += URItoFileName(RealURI);
1611
1612    struct stat Buf;
1613    if (stat(Final.c_str(),&Buf) != 0)
1614       return "\nIndex-File: true\nFail-Ignore: true\n";
1615
1616    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1617 }
1618                                                                         /*}}}*/
1619 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1620 {
1621    if (AuthPass == false)
1622    {
1623       new pkgAcqMetaSig(Owner,
1624                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1625                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1626                         IndexTargets, MetaIndexParser);
1627       if (Cnf->LocalOnly == true ||
1628           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1629          Dequeue();
1630    }
1631    else
1632       pkgAcqMetaIndex::Failed(Message, Cnf);
1633 }
1634                                                                         /*}}}*/
1635 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1636 // ---------------------------------------------------------------------
1637 /* This just sets up the initial fetch environment and queues the first
1638    possibilitiy */
1639 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1640                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1641                              string &StoreFilename) :
1642                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1643                StoreFilename(StoreFilename), Vf(Version.FileList()),
1644                Trusted(false)
1645 {
1646    Retries = _config->FindI("Acquire::Retries",0);
1647
1648    if (Version.Arch() == 0)
1649    {
1650       _error->Error(_("I wasn't able to locate a file for the %s package. "
1651                       "This might mean you need to manually fix this package. "
1652                       "(due to missing arch)"),
1653                     Version.ParentPkg().Name());
1654       return;
1655    }
1656
1657    /* We need to find a filename to determine the extension. We make the
1658       assumption here that all the available sources for this version share
1659       the same extension.. */
1660    // Skip not source sources, they do not have file fields.
1661    for (; Vf.end() == false; Vf++)
1662    {
1663       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1664          continue;
1665       break;
1666    }
1667
1668    // Does not really matter here.. we are going to fail out below
1669    if (Vf.end() != true)
1670    {
1671       // If this fails to get a file name we will bomb out below.
1672       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1673       if (_error->PendingError() == true)
1674          return;
1675
1676       // Generate the final file name as: package_version_arch.foo
1677       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1678                       QuoteString(Version.VerStr(),"_:") + '_' +
1679                       QuoteString(Version.Arch(),"_:.") +
1680                       "." + flExtension(Parse.FileName());
1681    }
1682
1683    // check if we have one trusted source for the package. if so, switch
1684    // to "TrustedOnly" mode
1685    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1686    {
1687       pkgIndexFile *Index;
1688       if (Sources->FindIndex(i.File(),Index) == false)
1689          continue;
1690       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1691       {
1692          std::cerr << "Checking index: " << Index->Describe()
1693                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1694       }
1695       if (Index->IsTrusted()) {
1696          Trusted = true;
1697          break;
1698       }
1699    }
1700
1701    // "allow-unauthenticated" restores apts old fetching behaviour
1702    // that means that e.g. unauthenticated file:// uris are higher
1703    // priority than authenticated http:// uris
1704    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1705       Trusted = false;
1706
1707    // Select a source
1708    if (QueueNext() == false && _error->PendingError() == false)
1709       _error->Error(_("I wasn't able to locate a file for the %s package. "
1710                     "This might mean you need to manually fix this package."),
1711                     Version.ParentPkg().Name());
1712 }
1713                                                                         /*}}}*/
1714 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1715 // ---------------------------------------------------------------------
1716 /* This queues the next available file version for download. It checks if
1717    the archive is already available in the cache and stashs the MD5 for
1718    checking later. */
1719 bool pkgAcqArchive::QueueNext()
1720 {
1721    string const ForceHash = _config->Find("Acquire::ForceHash");
1722    for (; Vf.end() == false; ++Vf)
1723    {
1724       // Ignore not source sources
1725       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1726          continue;
1727
1728       // Try to cross match against the source list
1729       pkgIndexFile *Index;
1730       if (Sources->FindIndex(Vf.File(),Index) == false)
1731             continue;
1732
1733       // only try to get a trusted package from another source if that source
1734       // is also trusted
1735       if(Trusted && !Index->IsTrusted())
1736          continue;
1737
1738       // Grab the text package record
1739       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1740       if (_error->PendingError() == true)
1741          return false;
1742
1743       string PkgFile = Parse.FileName();
1744       if (ForceHash.empty() == false)
1745       {
1746          if(stringcasecmp(ForceHash, "sha512") == 0)
1747             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1748          if(stringcasecmp(ForceHash, "sha256") == 0)
1749             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1750          else if (stringcasecmp(ForceHash, "sha1") == 0)
1751             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1752          else
1753             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1754       }
1755       else
1756       {
1757          string Hash;
1758          if ((Hash = Parse.SHA512Hash()).empty() == false)
1759             ExpectedHash = HashString("SHA512", Hash);
1760          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1761             ExpectedHash = HashString("SHA256", Hash);
1762          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1763             ExpectedHash = HashString("SHA1", Hash);
1764          else
1765             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1766       }
1767       if (PkgFile.empty() == true)
1768          return _error->Error(_("The package index files are corrupted. No Filename: "
1769                               "field for package %s."),
1770                               Version.ParentPkg().Name());
1771
1772       Desc.URI = Index->ArchiveURI(PkgFile);
1773       Desc.Description = Index->ArchiveInfo(Version);
1774       Desc.Owner = this;
1775       Desc.ShortDesc = Version.ParentPkg().Name();
1776
1777       // See if we already have the file. (Legacy filenames)
1778       FileSize = Version->Size;
1779       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1780       struct stat Buf;
1781       if (stat(FinalFile.c_str(),&Buf) == 0)
1782       {
1783          // Make sure the size matches
1784          if ((unsigned long long)Buf.st_size == Version->Size)
1785          {
1786             Complete = true;
1787             Local = true;
1788             Status = StatDone;
1789             StoreFilename = DestFile = FinalFile;
1790             return true;
1791          }
1792
1793          /* Hmm, we have a file and its size does not match, this means it is
1794             an old style mismatched arch */
1795          unlink(FinalFile.c_str());
1796       }
1797
1798       // Check it again using the new style output filenames
1799       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1800       if (stat(FinalFile.c_str(),&Buf) == 0)
1801       {
1802          // Make sure the size matches
1803          if ((unsigned long long)Buf.st_size == Version->Size)
1804          {
1805             Complete = true;
1806             Local = true;
1807             Status = StatDone;
1808             StoreFilename = DestFile = FinalFile;
1809             return true;
1810          }
1811
1812          /* Hmm, we have a file and its size does not match, this shouldnt
1813             happen.. */
1814          unlink(FinalFile.c_str());
1815       }
1816
1817       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1818
1819       // Check the destination file
1820       if (stat(DestFile.c_str(),&Buf) == 0)
1821       {
1822          // Hmm, the partial file is too big, erase it
1823          if ((unsigned long long)Buf.st_size > Version->Size)
1824             unlink(DestFile.c_str());
1825          else
1826             PartialSize = Buf.st_size;
1827       }
1828
1829       // Create the item
1830       Local = false;
1831       Desc.URI = Index->ArchiveURI(PkgFile);
1832       Desc.Description = Index->ArchiveInfo(Version);
1833       Desc.Owner = this;
1834       Desc.ShortDesc = Version.ParentPkg().Name();
1835       QueueURI(Desc);
1836
1837       ++Vf;
1838       return true;
1839    }
1840    return false;
1841 }
1842                                                                         /*}}}*/
1843 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1844 // ---------------------------------------------------------------------
1845 /* */
1846 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
1847                          pkgAcquire::MethodConfig *Cfg)
1848 {
1849    Item::Done(Message,Size,CalcHash,Cfg);
1850
1851    // Check the size
1852    if (Size != Version->Size)
1853    {
1854       Status = StatError;
1855       ErrorText = _("Size mismatch");
1856       return;
1857    }
1858
1859    // Check the hash
1860    if(ExpectedHash.toStr() != CalcHash)
1861    {
1862       Status = StatError;
1863       ErrorText = _("Hash Sum mismatch");
1864       if(FileExists(DestFile))
1865          Rename(DestFile,DestFile + ".FAILED");
1866       return;
1867    }
1868
1869    // Grab the output filename
1870    string FileName = LookupTag(Message,"Filename");
1871    if (FileName.empty() == true)
1872    {
1873       Status = StatError;
1874       ErrorText = "Method gave a blank filename";
1875       return;
1876    }
1877
1878    Complete = true;
1879
1880    // Reference filename
1881    if (FileName != DestFile)
1882    {
1883       StoreFilename = DestFile = FileName;
1884       Local = true;
1885       return;
1886    }
1887
1888    // Done, move it into position
1889    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1890    FinalFile += flNotDir(StoreFilename);
1891    Rename(DestFile,FinalFile);
1892
1893    StoreFilename = DestFile = FinalFile;
1894    Complete = true;
1895 }
1896                                                                         /*}}}*/
1897 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1898 // ---------------------------------------------------------------------
1899 /* Here we try other sources */
1900 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1901 {
1902    ErrorText = LookupTag(Message,"Message");
1903
1904    /* We don't really want to retry on failed media swaps, this prevents
1905       that. An interesting observation is that permanent failures are not
1906       recorded. */
1907    if (Cnf->Removable == true &&
1908        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1909    {
1910       // Vf = Version.FileList();
1911       while (Vf.end() == false) ++Vf;
1912       StoreFilename = string();
1913       Item::Failed(Message,Cnf);
1914       return;
1915    }
1916
1917    if (QueueNext() == false)
1918    {
1919       // This is the retry counter
1920       if (Retries != 0 &&
1921           Cnf->LocalOnly == false &&
1922           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1923       {
1924          Retries--;
1925          Vf = Version.FileList();
1926          if (QueueNext() == true)
1927             return;
1928       }
1929
1930       StoreFilename = string();
1931       Item::Failed(Message,Cnf);
1932    }
1933 }
1934                                                                         /*}}}*/
1935 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1936 // ---------------------------------------------------------------------
1937 bool pkgAcqArchive::IsTrusted()
1938 {
1939    return Trusted;
1940 }
1941                                                                         /*}}}*/
1942 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1943 // ---------------------------------------------------------------------
1944 /* */
1945 void pkgAcqArchive::Finished()
1946 {
1947    if (Status == pkgAcquire::Item::StatDone &&
1948        Complete == true)
1949       return;
1950    StoreFilename = string();
1951 }
1952                                                                         /*}}}*/
1953 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1954 // ---------------------------------------------------------------------
1955 /* The file is added to the queue */
1956 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1957                        unsigned long long Size,string Dsc,string ShortDesc,
1958                        const string &DestDir, const string &DestFilename,
1959                        bool IsIndexFile) :
1960                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1961 {
1962    Retries = _config->FindI("Acquire::Retries",0);
1963
1964    if(!DestFilename.empty())
1965       DestFile = DestFilename;
1966    else if(!DestDir.empty())
1967       DestFile = DestDir + "/" + flNotDir(URI);
1968    else
1969       DestFile = flNotDir(URI);
1970
1971    // Create the item
1972    Desc.URI = URI;
1973    Desc.Description = Dsc;
1974    Desc.Owner = this;
1975
1976    // Set the short description to the archive component
1977    Desc.ShortDesc = ShortDesc;
1978
1979    // Get the transfer sizes
1980    FileSize = Size;
1981    struct stat Buf;
1982    if (stat(DestFile.c_str(),&Buf) == 0)
1983    {
1984       // Hmm, the partial file is too big, erase it
1985       if ((unsigned long long)Buf.st_size > Size)
1986          unlink(DestFile.c_str());
1987       else
1988          PartialSize = Buf.st_size;
1989    }
1990
1991    QueueURI(Desc);
1992 }
1993                                                                         /*}}}*/
1994 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1995 // ---------------------------------------------------------------------
1996 /* */
1997 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
1998                       pkgAcquire::MethodConfig *Cnf)
1999 {
2000    Item::Done(Message,Size,CalcHash,Cnf);
2001
2002    // Check the hash
2003    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2004    {
2005       Status = StatError;
2006       ErrorText = _("Hash Sum mismatch");
2007       Rename(DestFile,DestFile + ".FAILED");
2008       return;
2009    }
2010
2011    string FileName = LookupTag(Message,"Filename");
2012    if (FileName.empty() == true)
2013    {
2014       Status = StatError;
2015       ErrorText = "Method gave a blank filename";
2016       return;
2017    }
2018
2019    Complete = true;
2020
2021    // The files timestamp matches
2022    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2023       return;
2024
2025    // We have to copy it into place
2026    if (FileName != DestFile)
2027    {
2028       Local = true;
2029       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2030           Cnf->Removable == true)
2031       {
2032          Desc.URI = "copy:" + FileName;
2033          QueueURI(Desc);
2034          return;
2035       }
2036
2037       // Erase the file if it is a symlink so we can overwrite it
2038       struct stat St;
2039       if (lstat(DestFile.c_str(),&St) == 0)
2040       {
2041          if (S_ISLNK(St.st_mode) != 0)
2042             unlink(DestFile.c_str());
2043       }
2044
2045       // Symlink the file
2046       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2047       {
2048          ErrorText = "Link to " + DestFile + " failure ";
2049          Status = StatError;
2050          Complete = false;
2051       }
2052    }
2053 }
2054                                                                         /*}}}*/
2055 // AcqFile::Failed - Failure handler                                    /*{{{*/
2056 // ---------------------------------------------------------------------
2057 /* Here we try other sources */
2058 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2059 {
2060    ErrorText = LookupTag(Message,"Message");
2061
2062    // This is the retry counter
2063    if (Retries != 0 &&
2064        Cnf->LocalOnly == false &&
2065        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2066    {
2067       Retries--;
2068       QueueURI(Desc);
2069       return;
2070    }
2071
2072    Item::Failed(Message,Cnf);
2073 }
2074                                                                         /*}}}*/
2075 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2076 // ---------------------------------------------------------------------
2077 /* The only header we use is the last-modified header. */
2078 string pkgAcqFile::Custom600Headers()
2079 {
2080    if (IsIndexFile)
2081       return "\nIndex-File: true";
2082    return "";
2083 }
2084                                                                         /*}}}*/