apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by alot of item methods as thier final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 147 {
 148    if(FileExists(DestFile))
 149       Rename(DestFile, DestFile + ".FAILED");
 150
 151    switch (error)
 152    {
 153       case HashSumMismatch:
 154          ErrorText = _("Hash Sum mismatch");
 155          Status = StatAuthError;
 156          ReportMirrorFailure("HashChecksumFailure");
 157          break;
 158       case SizeMismatch:
 159          ErrorText = _("Size mismatch");
 160          Status = StatAuthError;
 161          ReportMirrorFailure("SizeFailure");
 162          break;
 163       case InvalidFormat:
 164          ErrorText = _("Invalid file format");
 165          Status = StatError;
 166          // do not report as usually its not the mirrors fault, but Portal/Proxy
 167          break;
 168    }
 169    return false;
 170 }
 171                                                                         /*}}}*/
 172 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 173 // ---------------------------------------------------------------------
 174 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 175 {
 176    // we only act if a mirror was used at all
 177    if(UsedMirror.empty())
 178       return;
 179 #if 0
 180    std::cerr << "\nReportMirrorFailure: "
 181              << UsedMirror
 182              << " Uri: " << DescURI()
 183              << " FailCode: "
 184              << FailCode << std::endl;
 185 #endif
 186    const char *Args[40];
 187    unsigned int i = 0;
 188    string report = _config->Find("Methods::Mirror::ProblemReporting",
 189                                  "/usr/lib/apt/apt-report-mirror-failure");
 190    if(!FileExists(report))
 191       return;
 192    Args[i++] = report.c_str();
 193    Args[i++] = UsedMirror.c_str();
 194    Args[i++] = DescURI().c_str();
 195    Args[i++] = FailCode.c_str();
 196    Args[i++] = NULL;
 197    pid_t pid = ExecFork();
 198    if(pid < 0)
 199    {
 200       _error->Error("ReportMirrorFailure Fork failed");
 201       return;
 202    }
 203    else if(pid == 0)
 204    {
 205       execvp(Args[0], (char**)Args);
 206       std::cerr << "Could not exec " << Args[0] << std::endl;
 207       _exit(100);
 208    }
 209    if(!ExecWait(pid, "report-mirror-failure"))
 210    {
 211       _error->Warning("Couldn't report problem to '%s'",
 212                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 213    }
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* Get a sub-index file based on checksums from a 'master' file and
 219    possibly query additional files */
 220 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 221                                  string const &URIDesc, string const &ShortDesc,
 222                                  HashString const &ExpectedHash)
 223    : Item(Owner), ExpectedHash(ExpectedHash)
 224 {
 225    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 226    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 227
 228    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 229    DestFile += URItoFileName(URI);
 230
 231    Desc.URI = URI;
 232    Desc.Description = URIDesc;
 233    Desc.Owner = this;
 234    Desc.ShortDesc = ShortDesc;
 235
 236    QueueURI(Desc);
 237
 238    if(Debug)
 239       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 240 }
 241                                                                         /*}}}*/
 242 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 243 // ---------------------------------------------------------------------
 244 /* The only header we use is the last-modified header. */
 245 string pkgAcqSubIndex::Custom600Headers()
 246 {
 247    string Final = _config->FindDir("Dir::State::lists");
 248    Final += URItoFileName(Desc.URI);
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true\nFail-Ignore: true\n";
 253    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255                                                                         /*}}}*/
 256 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 257 {
 258    if(Debug)
 259       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 260
 261    Complete = false;
 262    Status = StatDone;
 263    Dequeue();
 264
 265    // No good Index is provided
 266 }
 267                                                                         /*}}}*/
 268 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 269                            pkgAcquire::MethodConfig *Cnf)
 270 {
 271    if(Debug)
 272       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 273
 274    string FileName = LookupTag(Message,"Filename");
 275    if (FileName.empty() == true)
 276    {
 277       Status = StatError;
 278       ErrorText = "Method gave a blank filename";
 279       return;
 280    }
 281
 282    if (FileName != DestFile)
 283    {
 284       Local = true;
 285       Desc.URI = "copy:" + FileName;
 286       QueueURI(Desc);
 287       return;
 288    }
 289
 290    Item::Done(Message,Size,Md5Hash,Cnf);
 291
 292    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 293
 294    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 295    indexRecords SubIndexParser;
 296    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 297       Status = StatError;
 298       ErrorText = SubIndexParser.ErrorText;
 299       return;
 300    }
 301
 302    // sucess in downloading the index
 303    // rename the index
 304    if(Debug)
 305       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 306    Rename(DestFile,FinalFile);
 307    chmod(FinalFile.c_str(),0644);
 308    DestFile = FinalFile;
 309
 310    if(ParseIndex(DestFile) == false)
 311       return Failed("", NULL);
 312
 313    Complete = true;
 314    Status = StatDone;
 315    Dequeue();
 316    return;
 317 }
 318                                                                         /*}}}*/
 319 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 320 {
 321    indexRecords SubIndexParser;
 322    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 323       return false;
 324    // so something with the downloaded index
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* Get the DiffIndex file first and see if there are patches availabe
 331  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 332  * patches. If anything goes wrong in that process, it will fall back to
 333  * the original packages file
 334  */
 335 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 336                                  string URI,string URIDesc,string ShortDesc,
 337                                  HashString ExpectedHash)
 338    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 339      Description(URIDesc)
 340 {
 341
 342    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 343
 344    Desc.Description = URIDesc + "/DiffIndex";
 345    Desc.Owner = this;
 346    Desc.ShortDesc = ShortDesc;
 347    Desc.URI = URI + ".diff/Index";
 348
 349    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 350    DestFile += URItoFileName(URI) + string(".DiffIndex");
 351
 352    if(Debug)
 353       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 354
 355    // look for the current package file
 356    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 357    CurrentPackagesFile += URItoFileName(RealURI);
 358
 359    // FIXME: this file:/ check is a hack to prevent fetching
 360    //        from local sources. this is really silly, and
 361    //        should be fixed cleanly as soon as possible
 362    if(!FileExists(CurrentPackagesFile) ||
 363       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 364    {
 365       // we don't have a pkg file or we don't want to queue
 366       if(Debug)
 367          std::clog << "No index file, local or canceld by user" << std::endl;
 368       Failed("", NULL);
 369       return;
 370    }
 371
 372    if(Debug)
 373       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 374          << CurrentPackagesFile << std::endl;
 375
 376    QueueURI(Desc);
 377
 378 }
 379                                                                         /*}}}*/
 380 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* The only header we use is the last-modified header. */
 383 string pkgAcqDiffIndex::Custom600Headers()
 384 {
 385    string Final = _config->FindDir("Dir::State::lists");
 386    Final += URItoFileName(RealURI) + string(".IndexDiff");
 387
 388    if(Debug)
 389       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 390
 391    struct stat Buf;
 392    if (stat(Final.c_str(),&Buf) != 0)
 393       return "\nIndex-File: true";
 394
 395    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 396 }
 397                                                                         /*}}}*/
 398 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 399 {
 400    if(Debug)
 401       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 402          << std::endl;
 403
 404    pkgTagSection Tags;
 405    string ServerSha1;
 406    vector<DiffInfo> available_patches;
 407
 408    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 409    pkgTagFile TF(&Fd);
 410    if (_error->PendingError() == true)
 411       return false;
 412
 413    if(TF.Step(Tags) == true)
 414    {
 415       bool found = false;
 416       DiffInfo d;
 417       string size;
 418
 419       string const tmp = Tags.FindS("SHA1-Current");
 420       std::stringstream ss(tmp);
 421       ss >> ServerSha1 >> size;
 422       unsigned long const ServerSize = atol(size.c_str());
 423
 424       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 425       SHA1Summation SHA1;
 426       SHA1.AddFD(fd);
 427       string const local_sha1 = SHA1.Result();
 428
 429       if(local_sha1 == ServerSha1)
 430       {
 431          // we have the same sha1 as the server so we are done here
 432          if(Debug)
 433             std::clog << "Package file is up-to-date" << std::endl;
 434          // list cleanup needs to know that this file as well as the already
 435          // present index is ours, so we create an empty diff to save it for us
 436          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 437                ExpectedHash, ServerSha1, available_patches);
 438          return true;
 439       }
 440       else
 441       {
 442          if(Debug)
 443             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 444
 445          // check the historie and see what patches we need
 446          string const history = Tags.FindS("SHA1-History");
 447          std::stringstream hist(history);
 448          while(hist >> d.sha1 >> size >> d.file)
 449          {
 450             // read until the first match is found
 451             // from that point on, we probably need all diffs
 452             if(d.sha1 == local_sha1)
 453                found=true;
 454             else if (found == false)
 455                continue;
 456
 457             if(Debug)
 458                std::clog << "Need to get diff: " << d.file << std::endl;
 459             available_patches.push_back(d);
 460          }
 461
 462          if (available_patches.empty() == false)
 463          {
 464             // patching with too many files is rather slow compared to a fast download
 465             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 466             if (fileLimit != 0 && fileLimit < available_patches.size())
 467             {
 468                if (Debug)
 469                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 470                         << ") so fallback to complete download" << std::endl;
 471                return false;
 472             }
 473
 474             // see if the patches are too big
 475             found = false; // it was true and it will be true again at the end
 476             d = *available_patches.begin();
 477             string const firstPatch = d.file;
 478             unsigned long patchesSize = 0;
 479             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 480             while(patches >> d.sha1 >> size >> d.file)
 481             {
 482                if (firstPatch == d.file)
 483                   found = true;
 484                else if (found == false)
 485                   continue;
 486
 487                patchesSize += atol(size.c_str());
 488             }
 489             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 490             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 491             {
 492                if (Debug)
 493                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 494                         << ") so fallback to complete download" << std::endl;
 495                return false;
 496             }
 497          }
 498       }
 499
 500       // we have something, queue the next diff
 501       if(found)
 502       {
 503          // queue the diffs
 504          string::size_type const last_space = Description.rfind(" ");
 505          if(last_space != string::npos)
 506             Description.erase(last_space, Description.size()-last_space);
 507
 508          /* decide if we should download patches one by one or in one go:
 509             The first is good if the server merges patches, but many don't so client
 510             based merging can be attempt in which case the second is better.
 511             "bad things" will happen if patches are merged on the server,
 512             but client side merging is attempt as well */
 513          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 514          if (pdiff_merge == true)
 515          {
 516             // this perl script is provided by apt-file
 517             pdiff_merge = FileExists(_config->FindFile("Dir::Bin::rred", "/usr/bin/diffindex-rred"));
 518             if (pdiff_merge == true)
 519             {
 520                // reprepro adds this flag if it has merged patches on the server
 521                std::string const precedence = Tags.FindS("X-Patch-Precedence");
 522                pdiff_merge = (precedence != "merged");
 523             }
 524          }
 525
 526          if (pdiff_merge == false)
 527             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 528                   ExpectedHash, ServerSha1, available_patches);
 529          else
 530          {
 531             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 532             for(size_t i = 0; i < available_patches.size(); ++i)
 533                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 534                      available_patches[i], diffs);
 535          }
 536
 537          Complete = false;
 538          Status = StatDone;
 539          Dequeue();
 540          return true;
 541       }
 542    }
 543
 544    // Nothing found, report and return false
 545    // Failing here is ok, if we return false later, the full
 546    // IndexFile is queued
 547    if(Debug)
 548       std::clog << "Can't find a patch in the index file" << std::endl;
 549    return false;
 550 }
 551                                                                         /*}}}*/
 552 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 553 {
 554    if(Debug)
 555       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 556                 << "Falling back to normal index file aquire" << std::endl;
 557
 558    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 559                    ExpectedHash);
 560
 561    Complete = false;
 562    Status = StatDone;
 563    Dequeue();
 564 }
 565                                                                         /*}}}*/
 566 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 567                            pkgAcquire::MethodConfig *Cnf)
 568 {
 569    if(Debug)
 570       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 571
 572    Item::Done(Message,Size,Md5Hash,Cnf);
 573
 574    string FinalFile;
 575    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 576
 577    // sucess in downloading the index
 578    // rename the index
 579    FinalFile += string(".IndexDiff");
 580    if(Debug)
 581       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 582                 << std::endl;
 583    Rename(DestFile,FinalFile);
 584    chmod(FinalFile.c_str(),0644);
 585    DestFile = FinalFile;
 586
 587    if(!ParseDiffIndex(DestFile))
 588       return Failed("", NULL);
 589
 590    Complete = true;
 591    Status = StatDone;
 592    Dequeue();
 593    return;
 594 }
 595                                                                         /*}}}*/
 596 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 597 // ---------------------------------------------------------------------
 598 /* The package diff is added to the queue. one object is constructed
 599  * for each diff and the index
 600  */
 601 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 602                                    string URI,string URIDesc,string ShortDesc,
 603                                    HashString ExpectedHash,
 604                                    string ServerSha1,
 605                                    vector<DiffInfo> diffs)
 606    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 607      available_patches(diffs), ServerSha1(ServerSha1)
 608 {
 609
 610    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 611    DestFile += URItoFileName(URI);
 612
 613    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 614
 615    Description = URIDesc;
 616    Desc.Owner = this;
 617    Desc.ShortDesc = ShortDesc;
 618
 619    if(available_patches.empty() == true)
 620    {
 621       // we are done (yeah!)
 622       Finish(true);
 623    }
 624    else
 625    {
 626       // get the next diff
 627       State = StateFetchDiff;
 628       QueueNextDiff();
 629    }
 630 }
 631                                                                         /*}}}*/
 632 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 633 {
 634    if(Debug)
 635       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 636                 << "Falling back to normal index file aquire" << std::endl;
 637    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 638                    ExpectedHash);
 639    Finish();
 640 }
 641                                                                         /*}}}*/
 642 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 643 void pkgAcqIndexDiffs::Finish(bool allDone)
 644 {
 645    // we restore the original name, this is required, otherwise
 646    // the file will be cleaned
 647    if(allDone)
 648    {
 649       DestFile = _config->FindDir("Dir::State::lists");
 650       DestFile += URItoFileName(RealURI);
 651
 652       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 653       {
 654          RenameOnError(HashSumMismatch);
 655          Dequeue();
 656          return;
 657       }
 658
 659       // this is for the "real" finish
 660       Complete = true;
 661       Status = StatDone;
 662       Dequeue();
 663       if(Debug)
 664          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 665       return;
 666    }
 667
 668    if(Debug)
 669       std::clog << "Finishing: " << Desc.URI << std::endl;
 670    Complete = false;
 671    Status = StatDone;
 672    Dequeue();
 673    return;
 674 }
 675                                                                         /*}}}*/
 676 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 677 {
 678
 679    // calc sha1 of the just patched file
 680    string FinalFile = _config->FindDir("Dir::State::lists");
 681    FinalFile += URItoFileName(RealURI);
 682
 683    FileFd fd(FinalFile, FileFd::ReadOnly);
 684    SHA1Summation SHA1;
 685    SHA1.AddFD(fd);
 686    string local_sha1 = string(SHA1.Result());
 687    if(Debug)
 688       std::clog << "QueueNextDiff: "
 689                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 690
 691    // final file reached before all patches are applied
 692    if(local_sha1 == ServerSha1)
 693    {
 694       Finish(true);
 695       return true;
 696    }
 697
 698    // remove all patches until the next matching patch is found
 699    // this requires the Index file to be ordered
 700    for(vector<DiffInfo>::iterator I=available_patches.begin();
 701        available_patches.empty() == false &&
 702           I != available_patches.end() &&
 703           I->sha1 != local_sha1;
 704        ++I)
 705    {
 706       available_patches.erase(I);
 707    }
 708
 709    // error checking and falling back if no patch was found
 710    if(available_patches.empty() == true)
 711    {
 712       Failed("", NULL);
 713       return false;
 714    }
 715
 716    // queue the right diff
 717    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 718    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 719    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 720    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 721
 722    if(Debug)
 723       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 724
 725    QueueURI(Desc);
 726
 727    return true;
 728 }
 729                                                                         /*}}}*/
 730 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 731                             pkgAcquire::MethodConfig *Cnf)
 732 {
 733    if(Debug)
 734       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 735
 736    Item::Done(Message,Size,Md5Hash,Cnf);
 737
 738    string FinalFile;
 739    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 740
 741    // sucess in downloading a diff, enter ApplyDiff state
 742    if(State == StateFetchDiff)
 743    {
 744
 745       // rred excepts the patch as $FinalFile.ed
 746       Rename(DestFile,FinalFile+".ed");
 747
 748       if(Debug)
 749          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 750
 751       State = StateApplyDiff;
 752       Local = true;
 753       Desc.URI = "rred:" + FinalFile;
 754       QueueURI(Desc);
 755       Mode = "rred";
 756       return;
 757    }
 758
 759
 760    // success in download/apply a diff, queue next (if needed)
 761    if(State == StateApplyDiff)
 762    {
 763       // remove the just applied patch
 764       available_patches.erase(available_patches.begin());
 765
 766       // move into place
 767       if(Debug)
 768       {
 769          std::clog << "Moving patched file in place: " << std::endl
 770                    << DestFile << " -> " << FinalFile << std::endl;
 771       }
 772       Rename(DestFile,FinalFile);
 773       chmod(FinalFile.c_str(),0644);
 774
 775       // see if there is more to download
 776       if(available_patches.empty() == false) {
 777          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 778                               ExpectedHash, ServerSha1, available_patches);
 779          return Finish();
 780       } else
 781          return Finish(true);
 782    }
 783 }
 784                                                                         /*}}}*/
 785 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 786 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 787                                    string const &URI, string const &URIDesc,
 788                                    string const &ShortDesc, HashString const &ExpectedHash,
 789                                    DiffInfo const &patch,
 790                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 791    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 792      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 793 {
 794
 795    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 796    DestFile += URItoFileName(URI);
 797
 798    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 799
 800    Description = URIDesc;
 801    Desc.Owner = this;
 802    Desc.ShortDesc = ShortDesc;
 803
 804    Desc.URI = string(RealURI) + ".diff/" + patch.file + ".gz";
 805    Desc.Description = Description + " " + patch.file + string(".pdiff");
 806    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 807    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 808
 809    if(Debug)
 810       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 811
 812    QueueURI(Desc);
 813 }
 814                                                                         /*}}}*/
 815 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 816 {
 817    if(Debug)
 818       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 819    Complete = false;
 820    Status = StatDone;
 821    Dequeue();
 822
 823    // check if we are the first to fail, otherwise we are done here
 824    State = StateDoneDiff;
 825    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 826          I != allPatches->end(); ++I)
 827       if ((*I)->State == StateErrorDiff)
 828          return;
 829
 830    // first failure means we should fallback
 831    State = StateErrorDiff;
 832    std::clog << "Falling back to normal index file aquire" << std::endl;
 833    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 834                    ExpectedHash);
 835 }
 836                                                                         /*}}}*/
 837 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 838                             pkgAcquire::MethodConfig *Cnf)
 839 {
 840    if(Debug)
 841       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 842
 843    Item::Done(Message,Size,Md5Hash,Cnf);
 844
 845    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 846
 847    if (State == StateFetchDiff)
 848    {
 849       // rred expects the patch as $FinalFile.ed.$patchname.gz
 850       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 851
 852       // check if this is the last completed diff
 853       State = StateDoneDiff;
 854       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 855             I != allPatches->end(); ++I)
 856          if ((*I)->State != StateDoneDiff)
 857          {
 858             if(Debug)
 859                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 860             return;
 861          }
 862
 863       // this is the last completed diff, so we are ready to apply now
 864       State = StateApplyDiff;
 865
 866       if(Debug)
 867          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 868
 869       Local = true;
 870       Desc.URI = "rred:" + FinalFile;
 871       QueueURI(Desc);
 872       Mode = "rred";
 873       return;
 874    }
 875    // success in download/apply all diffs, clean up
 876    else if (State == StateApplyDiff)
 877    {
 878       // see if we really got the expected file
 879       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 880       {
 881          RenameOnError(HashSumMismatch);
 882          return;
 883       }
 884
 885       // move the result into place
 886       if(Debug)
 887          std::clog << "Moving patched file in place: " << std::endl
 888                    << DestFile << " -> " << FinalFile << std::endl;
 889       Rename(DestFile, FinalFile);
 890       chmod(FinalFile.c_str(), 0644);
 891
 892       // otherwise lists cleanup will eat the file
 893       DestFile = FinalFile;
 894
 895       // all set and done
 896       Complete = true;
 897       if(Debug)
 898          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 899    }
 900 }
 901                                                                         /*}}}*/
 902 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 903 // ---------------------------------------------------------------------
 904 /* The package file is added to the queue and a second class is
 905    instantiated to fetch the revision file */
 906 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 907                          string URI,string URIDesc,string ShortDesc,
 908                          HashString ExpectedHash, string comprExt)
 909    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 910 {
 911    if(comprExt.empty() == true)
 912    {
 913       // autoselect the compression method
 914       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 915       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 916          comprExt.append(*t).append(" ");
 917       if (comprExt.empty() == false)
 918          comprExt.erase(comprExt.end()-1);
 919    }
 920    CompressionExtension = comprExt;
 921
 922    Init(URI, URIDesc, ShortDesc);
 923 }
 924 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 925                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 926    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 927 {
 928    // autoselect the compression method
 929    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 930    CompressionExtension = "";
 931    if (ExpectedHash.empty() == false)
 932    {
 933       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 934          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 935             CompressionExtension.append(*t).append(" ");
 936    }
 937    else
 938    {
 939       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 940          CompressionExtension.append(*t).append(" ");
 941    }
 942    if (CompressionExtension.empty() == false)
 943       CompressionExtension.erase(CompressionExtension.end()-1);
 944
 945    // only verify non-optional targets, see acquire-item.h for a FIXME
 946    // to make this more flexible
 947    if (Target->IsOptional())
 948      Verify = false;
 949    else
 950      Verify = true;
 951
 952    Init(Target->URI, Target->Description, Target->ShortDesc);
 953 }
 954                                                                         /*}}}*/
 955 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 956 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 957    Decompression = false;
 958    Erase = false;
 959
 960    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 961    DestFile += URItoFileName(URI);
 962
 963    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 964    if (comprExt == "uncompressed")
 965       Desc.URI = URI;
 966    else
 967       Desc.URI = URI + '.' + comprExt;
 968
 969    Desc.Description = URIDesc;
 970    Desc.Owner = this;
 971    Desc.ShortDesc = ShortDesc;
 972
 973    QueueURI(Desc);
 974 }
 975                                                                         /*}}}*/
 976 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 977 // ---------------------------------------------------------------------
 978 /* The only header we use is the last-modified header. */
 979 string pkgAcqIndex::Custom600Headers()
 980 {
 981    string Final = _config->FindDir("Dir::State::lists");
 982    Final += URItoFileName(RealURI);
 983    if (_config->FindB("Acquire::GzipIndexes",false))
 984       Final += ".gz";
 985
 986    string msg = "\nIndex-File: true";
 987    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 988    //        seems to be difficult without breaking ABI
 989    if (ShortDesc().find("Translation") != 0)
 990       msg += "\nFail-Ignore: true";
 991    struct stat Buf;
 992    if (stat(Final.c_str(),&Buf) == 0)
 993       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 994
 995    return msg;
 996 }
 997                                                                         /*}}}*/
 998 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 999 {
1000    size_t const nextExt = CompressionExtension.find(' ');
1001    if (nextExt != std::string::npos)
1002    {
1003       CompressionExtension = CompressionExtension.substr(nextExt+1);
1004       Init(RealURI, Desc.Description, Desc.ShortDesc);
1005       return;
1006    }
1007
1008    // on decompression failure, remove bad versions in partial/
1009    if (Decompression && Erase) {
1010       string s = _config->FindDir("Dir::State::lists") + "partial/";
1011       s.append(URItoFileName(RealURI));
1012       unlink(s.c_str());
1013    }
1014
1015    Item::Failed(Message,Cnf);
1016 }
1017                                                                         /*}}}*/
1018 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1019 // ---------------------------------------------------------------------
1020 /* This goes through a number of states.. On the initial fetch the
1021    method could possibly return an alternate filename which points
1022    to the uncompressed version of the file. If this is so the file
1023    is copied into the partial directory. In all other cases the file
1024    is decompressed with a gzip uri. */
1025 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1026                        pkgAcquire::MethodConfig *Cfg)
1027 {
1028    Item::Done(Message,Size,Hash,Cfg);
1029
1030    if (Decompression == true)
1031    {
1032       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1033       {
1034          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1035          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1036       }
1037
1038       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1039       {
1040          RenameOnError(HashSumMismatch);
1041          return;
1042       }
1043
1044       /* Verify the index file for correctness (all indexes must
1045        * have a Package field) (LP: #346386) (Closes: #627642) */
1046       if (Verify == true)
1047       {
1048          FileFd fd(DestFile, FileFd::ReadOnly);
1049          // Only test for correctness if the file is not empty (empty is ok)
1050          if (fd.FileSize() > 0)
1051          {
1052             pkgTagSection sec;
1053             pkgTagFile tag(&fd);
1054
1055             // all our current indexes have a field 'Package' in each section
1056             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1057             {
1058                RenameOnError(InvalidFormat);
1059                return;
1060             }
1061          }
1062       }
1063
1064       // Done, move it into position
1065       string FinalFile = _config->FindDir("Dir::State::lists");
1066       FinalFile += URItoFileName(RealURI);
1067       Rename(DestFile,FinalFile);
1068       chmod(FinalFile.c_str(),0644);
1069
1070       /* We restore the original name to DestFile so that the clean operation
1071          will work OK */
1072       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1073       DestFile += URItoFileName(RealURI);
1074
1075       // Remove the compressed version.
1076       if (Erase == true)
1077          unlink(DestFile.c_str());
1078       return;
1079    }
1080
1081    Erase = false;
1082    Complete = true;
1083
1084    // Handle the unzipd case
1085    string FileName = LookupTag(Message,"Alt-Filename");
1086    if (FileName.empty() == false)
1087    {
1088       // The files timestamp matches
1089       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1090          return;
1091       Decompression = true;
1092       Local = true;
1093       DestFile += ".decomp";
1094       Desc.URI = "copy:" + FileName;
1095       QueueURI(Desc);
1096       Mode = "copy";
1097       return;
1098    }
1099
1100    FileName = LookupTag(Message,"Filename");
1101    if (FileName.empty() == true)
1102    {
1103       Status = StatError;
1104       ErrorText = "Method gave a blank filename";
1105    }
1106
1107    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1108
1109    // The files timestamp matches
1110    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1111        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1112           // Update DestFile for .gz suffix so that the clean operation keeps it
1113           DestFile += ".gz";
1114       return;
1115     }
1116
1117    if (FileName == DestFile)
1118       Erase = true;
1119    else
1120       Local = true;
1121
1122    string decompProg;
1123
1124    // If we enable compressed indexes and already have gzip, keep it
1125    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1126       string FinalFile = _config->FindDir("Dir::State::lists");
1127       FinalFile += URItoFileName(RealURI) + ".gz";
1128       Rename(DestFile,FinalFile);
1129       chmod(FinalFile.c_str(),0644);
1130
1131       // Update DestFile for .gz suffix so that the clean operation keeps it
1132       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1133       DestFile += URItoFileName(RealURI) + ".gz";
1134       return;
1135     }
1136
1137    // get the binary name for your used compression type
1138    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1139    if(decompProg.empty() == false);
1140    else if(compExt == "uncompressed")
1141       decompProg = "copy";
1142    else {
1143       _error->Error("Unsupported extension: %s", compExt.c_str());
1144       return;
1145    }
1146
1147    Decompression = true;
1148    DestFile += ".decomp";
1149    Desc.URI = decompProg + ":" + FileName;
1150    QueueURI(Desc);
1151
1152    // FIXME: this points to a c++ string that goes out of scope
1153    Mode = decompProg.c_str();
1154 }
1155                                                                         /*}}}*/
1156 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1157 // ---------------------------------------------------------------------
1158 /* The Translation file is added to the queue */
1159 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1160                             string URI,string URIDesc,string ShortDesc)
1161   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1162 {
1163 }
1164 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1165                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1166   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1167 {
1168 }
1169                                                                         /*}}}*/
1170 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1171 // ---------------------------------------------------------------------
1172 string pkgAcqIndexTrans::Custom600Headers()
1173 {
1174    string Final = _config->FindDir("Dir::State::lists");
1175    Final += URItoFileName(RealURI);
1176
1177    struct stat Buf;
1178    if (stat(Final.c_str(),&Buf) != 0)
1179       return "\nFail-Ignore: true\nIndex-File: true";
1180    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1181 }
1182                                                                         /*}}}*/
1183 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1184 // ---------------------------------------------------------------------
1185 /* */
1186 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1187 {
1188    size_t const nextExt = CompressionExtension.find(' ');
1189    if (nextExt != std::string::npos)
1190    {
1191       CompressionExtension = CompressionExtension.substr(nextExt+1);
1192       Init(RealURI, Desc.Description, Desc.ShortDesc);
1193       Status = StatIdle;
1194       return;
1195    }
1196
1197    if (Cnf->LocalOnly == true ||
1198        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1199    {
1200       // Ignore this
1201       Status = StatDone;
1202       Complete = false;
1203       Dequeue();
1204       return;
1205    }
1206
1207    Item::Failed(Message,Cnf);
1208 }
1209                                                                         /*}}}*/
1210 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1211                              string URI,string URIDesc,string ShortDesc,
1212                              string MetaIndexURI, string MetaIndexURIDesc,
1213                              string MetaIndexShortDesc,
1214                              const vector<IndexTarget*>* IndexTargets,
1215                              indexRecords* MetaIndexParser) :
1216    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1217    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1218    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1219 {
1220    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1221    DestFile += URItoFileName(URI);
1222
1223    // remove any partial downloaded sig-file in partial/.
1224    // it may confuse proxies and is too small to warrant a
1225    // partial download anyway
1226    unlink(DestFile.c_str());
1227
1228    // Create the item
1229    Desc.Description = URIDesc;
1230    Desc.Owner = this;
1231    Desc.ShortDesc = ShortDesc;
1232    Desc.URI = URI;
1233
1234    string Final = _config->FindDir("Dir::State::lists");
1235    Final += URItoFileName(RealURI);
1236    if (RealFileExists(Final) == true)
1237    {
1238       // File was already in place.  It needs to be re-downloaded/verified
1239       // because Release might have changed, we do give it a differnt
1240       // name than DestFile because otherwise the http method will
1241       // send If-Range requests and there are too many broken servers
1242       // out there that do not understand them
1243       LastGoodSig = DestFile+".reverify";
1244       Rename(Final,LastGoodSig);
1245    }
1246
1247    QueueURI(Desc);
1248 }
1249                                                                         /*}}}*/
1250 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1251 {
1252    // if the file was never queued undo file-changes done in the constructor
1253    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1254          LastGoodSig.empty() == false)
1255    {
1256       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1257       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1258          Rename(LastGoodSig, Final);
1259    }
1260
1261 }
1262                                                                         /*}}}*/
1263 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1264 // ---------------------------------------------------------------------
1265 /* The only header we use is the last-modified header. */
1266 string pkgAcqMetaSig::Custom600Headers()
1267 {
1268    struct stat Buf;
1269    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1270       return "\nIndex-File: true";
1271
1272    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1273 }
1274
1275 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1276                          pkgAcquire::MethodConfig *Cfg)
1277 {
1278    Item::Done(Message,Size,MD5,Cfg);
1279
1280    string FileName = LookupTag(Message,"Filename");
1281    if (FileName.empty() == true)
1282    {
1283       Status = StatError;
1284       ErrorText = "Method gave a blank filename";
1285       return;
1286    }
1287
1288    if (FileName != DestFile)
1289    {
1290       // We have to copy it into place
1291       Local = true;
1292       Desc.URI = "copy:" + FileName;
1293       QueueURI(Desc);
1294       return;
1295    }
1296
1297    Complete = true;
1298
1299    // put the last known good file back on i-m-s hit (it will
1300    // be re-verified again)
1301    // Else do nothing, we have the new file in DestFile then
1302    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1303       Rename(LastGoodSig, DestFile);
1304
1305    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1306    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1307                        MetaIndexShortDesc,  DestFile, IndexTargets,
1308                        MetaIndexParser);
1309
1310 }
1311                                                                         /*}}}*/
1312 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1313 {
1314    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1315
1316    // if we get a network error we fail gracefully
1317    if(Status == StatTransientNetworkError)
1318    {
1319       Item::Failed(Message,Cnf);
1320       // move the sigfile back on transient network failures
1321       if(FileExists(LastGoodSig))
1322          Rename(LastGoodSig,Final);
1323
1324       // set the status back to , Item::Failed likes to reset it
1325       Status = pkgAcquire::Item::StatTransientNetworkError;
1326       return;
1327    }
1328
1329    // Delete any existing sigfile when the acquire failed
1330    unlink(Final.c_str());
1331
1332    // queue a pkgAcqMetaIndex with no sigfile
1333    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1334                        "", IndexTargets, MetaIndexParser);
1335
1336    if (Cnf->LocalOnly == true ||
1337        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1338    {
1339       // Ignore this
1340       Status = StatDone;
1341       Complete = false;
1342       Dequeue();
1343       return;
1344    }
1345
1346    Item::Failed(Message,Cnf);
1347 }
1348                                                                         /*}}}*/
1349 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1350                                  string URI,string URIDesc,string ShortDesc,
1351                                  string SigFile,
1352                                  const vector<struct IndexTarget*>* IndexTargets,
1353                                  indexRecords* MetaIndexParser) :
1354    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1355    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1356 {
1357    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1358    DestFile += URItoFileName(URI);
1359
1360    // Create the item
1361    Desc.Description = URIDesc;
1362    Desc.Owner = this;
1363    Desc.ShortDesc = ShortDesc;
1364    Desc.URI = URI;
1365
1366    QueueURI(Desc);
1367 }
1368                                                                         /*}}}*/
1369 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1370 // ---------------------------------------------------------------------
1371 /* The only header we use is the last-modified header. */
1372 string pkgAcqMetaIndex::Custom600Headers()
1373 {
1374    string Final = _config->FindDir("Dir::State::lists");
1375    Final += URItoFileName(RealURI);
1376
1377    struct stat Buf;
1378    if (stat(Final.c_str(),&Buf) != 0)
1379       return "\nIndex-File: true";
1380
1381    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1382 }
1383                                                                         /*}}}*/
1384 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1385                            pkgAcquire::MethodConfig *Cfg)
1386 {
1387    Item::Done(Message,Size,Hash,Cfg);
1388
1389    // MetaIndexes are done in two passes: one to download the
1390    // metaindex with an appropriate method, and a second to verify it
1391    // with the gpgv method
1392
1393    if (AuthPass == true)
1394    {
1395       AuthDone(Message);
1396
1397       // all cool, move Release file into place
1398       Complete = true;
1399    }
1400    else
1401    {
1402       RetrievalDone(Message);
1403       if (!Complete)
1404          // Still more retrieving to do
1405          return;
1406
1407       if (SigFile == "")
1408       {
1409          // There was no signature file, so we are finished.  Download
1410          // the indexes and do only hashsum verification if possible
1411          MetaIndexParser->Load(DestFile);
1412          QueueIndexes(false);
1413       }
1414       else
1415       {
1416          // There was a signature file, so pass it to gpgv for
1417          // verification
1418
1419          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1420             std::cerr << "Metaindex acquired, queueing gpg verification ("
1421                       << SigFile << "," << DestFile << ")\n";
1422          AuthPass = true;
1423          Desc.URI = "gpgv:" + SigFile;
1424          QueueURI(Desc);
1425          Mode = "gpgv";
1426          return;
1427       }
1428    }
1429
1430    if (Complete == true)
1431    {
1432       string FinalFile = _config->FindDir("Dir::State::lists");
1433       FinalFile += URItoFileName(RealURI);
1434       if (SigFile == DestFile)
1435          SigFile = FinalFile;
1436       Rename(DestFile,FinalFile);
1437       chmod(FinalFile.c_str(),0644);
1438       DestFile = FinalFile;
1439    }
1440 }
1441                                                                         /*}}}*/
1442 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1443 {
1444    // We have just finished downloading a Release file (it is not
1445    // verified yet)
1446
1447    string FileName = LookupTag(Message,"Filename");
1448    if (FileName.empty() == true)
1449    {
1450       Status = StatError;
1451       ErrorText = "Method gave a blank filename";
1452       return;
1453    }
1454
1455    if (FileName != DestFile)
1456    {
1457       Local = true;
1458       Desc.URI = "copy:" + FileName;
1459       QueueURI(Desc);
1460       return;
1461    }
1462
1463    // make sure to verify against the right file on I-M-S hit
1464    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1465    if(IMSHit)
1466    {
1467       string FinalFile = _config->FindDir("Dir::State::lists");
1468       FinalFile += URItoFileName(RealURI);
1469       if (SigFile == DestFile)
1470       {
1471          SigFile = FinalFile;
1472          // constructor of pkgAcqMetaClearSig moved it out of the way,
1473          // now move it back in on IMS hit for the 'old' file
1474          string const OldClearSig = DestFile + ".reverify";
1475          if (RealFileExists(OldClearSig) == true)
1476             Rename(OldClearSig, FinalFile);
1477       }
1478       DestFile = FinalFile;
1479    }
1480    Complete = true;
1481 }
1482                                                                         /*}}}*/
1483 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1484 {
1485    // At this point, the gpgv method has succeeded, so there is a
1486    // valid signature from a key in the trusted keyring.  We
1487    // perform additional verification of its contents, and use them
1488    // to verify the indexes we are about to download
1489
1490    if (!MetaIndexParser->Load(DestFile))
1491    {
1492       Status = StatAuthError;
1493       ErrorText = MetaIndexParser->ErrorText;
1494       return;
1495    }
1496
1497    if (!VerifyVendor(Message))
1498    {
1499       return;
1500    }
1501
1502    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1503       std::cerr << "Signature verification succeeded: "
1504                 << DestFile << std::endl;
1505
1506    // Download further indexes with verification
1507    QueueIndexes(true);
1508
1509    // is it a clearsigned MetaIndex file?
1510    if (DestFile == SigFile)
1511       return;
1512
1513    // Done, move signature file into position
1514    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1515       URItoFileName(RealURI) + ".gpg";
1516    Rename(SigFile,VerifiedSigFile);
1517    chmod(VerifiedSigFile.c_str(),0644);
1518 }
1519                                                                         /*}}}*/
1520 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1521 {
1522 #if 0
1523    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1524     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1525    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1526    {
1527       Status = StatError;
1528       ErrorText = MetaIndexParser->ErrorText;
1529       return;
1530    }
1531 #endif
1532    bool transInRelease = false;
1533    {
1534       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1535       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1536          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1537          if (k->find("Translation-") != std::string::npos)
1538          {
1539             transInRelease = true;
1540             break;
1541          }
1542    }
1543
1544    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1545         Target != IndexTargets->end();
1546         ++Target)
1547    {
1548       HashString ExpectedIndexHash;
1549       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1550       bool compressedAvailable = false;
1551       if (Record == NULL)
1552       {
1553          if ((*Target)->IsOptional() == true)
1554          {
1555             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1556             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1557                if (MetaIndexParser->Exists(string((*Target)->MetaKey).append(".").append(*t)) == true)
1558                {
1559                   compressedAvailable = true;
1560                   break;
1561                }
1562          }
1563          else if (verify == true)
1564          {
1565             Status = StatAuthError;
1566             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1567             return;
1568          }
1569       }
1570       else
1571       {
1572          ExpectedIndexHash = Record->Hash;
1573          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1574          {
1575             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1576             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1577             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1578          }
1579          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1580          {
1581             Status = StatAuthError;
1582             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1583             return;
1584          }
1585       }
1586
1587       if ((*Target)->IsOptional() == true)
1588       {
1589          if ((*Target)->IsSubIndex() == true)
1590             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1591                                 (*Target)->ShortDesc, ExpectedIndexHash);
1592          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1593          {
1594             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1595                 MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)
1596                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1597                                    (*Target)->ShortDesc, ExpectedIndexHash);
1598             else
1599                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1600          }
1601          continue;
1602       }
1603
1604       /* Queue Packages file (either diff or full packages files, depending
1605          on the users option) - we also check if the PDiff Index file is listed
1606          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1607          instead, but passing the required info to it is to much hassle */
1608       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1609           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1610          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1611                              (*Target)->ShortDesc, ExpectedIndexHash);
1612       else
1613          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1614    }
1615 }
1616                                                                         /*}}}*/
1617 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1618 {
1619    string::size_type pos;
1620
1621    // check for missing sigs (that where not fatal because otherwise we had
1622    // bombed earlier)
1623    string missingkeys;
1624    string msg = _("There is no public key available for the "
1625                   "following key IDs:\n");
1626    pos = Message.find("NO_PUBKEY ");
1627    if (pos != std::string::npos)
1628    {
1629       string::size_type start = pos+strlen("NO_PUBKEY ");
1630       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1631       missingkeys += (Fingerprint);
1632    }
1633    if(!missingkeys.empty())
1634       _error->Warning("%s", string(msg+missingkeys).c_str());
1635
1636    string Transformed = MetaIndexParser->GetExpectedDist();
1637
1638    if (Transformed == "../project/experimental")
1639    {
1640       Transformed = "experimental";
1641    }
1642
1643    pos = Transformed.rfind('/');
1644    if (pos != string::npos)
1645    {
1646       Transformed = Transformed.substr(0, pos);
1647    }
1648
1649    if (Transformed == ".")
1650    {
1651       Transformed = "";
1652    }
1653
1654    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1655        MetaIndexParser->GetValidUntil() > 0) {
1656       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1657       if (invalid_since > 0)
1658          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1659          // the time since then the file is invalid - formated in the same way as in
1660          // the download progress display (e.g. 7d 3h 42min 1s)
1661          return _error->Error(
1662             _("Release file for %s is expired (invalid since %s). "
1663               "Updates for this repository will not be applied."),
1664             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1665    }
1666
1667    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1668    {
1669       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1670       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1671       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1672    }
1673
1674    if (MetaIndexParser->CheckDist(Transformed) == false)
1675    {
1676       // This might become fatal one day
1677 //       Status = StatAuthError;
1678 //       ErrorText = "Conflicting distribution; expected "
1679 //          + MetaIndexParser->GetExpectedDist() + " but got "
1680 //          + MetaIndexParser->GetDist();
1681 //       return false;
1682       if (!Transformed.empty())
1683       {
1684          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1685                          Desc.Description.c_str(),
1686                          Transformed.c_str(),
1687                          MetaIndexParser->GetDist().c_str());
1688       }
1689    }
1690
1691    return true;
1692 }
1693                                                                         /*}}}*/
1694 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1695 // ---------------------------------------------------------------------
1696 /* */
1697 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1698 {
1699    if (AuthPass == true)
1700    {
1701       // gpgv method failed, if we have a good signature
1702       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1703       if (DestFile != SigFile)
1704          LastGoodSigFile.append(".gpg");
1705       LastGoodSigFile.append(".reverify");
1706
1707       if(FileExists(LastGoodSigFile))
1708       {
1709          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1710          if (DestFile != SigFile)
1711             VerifiedSigFile.append(".gpg");
1712          Rename(LastGoodSigFile, VerifiedSigFile);
1713          Status = StatTransientNetworkError;
1714          _error->Warning(_("An error occurred during the signature "
1715                            "verification. The repository is not updated "
1716                            "and the previous index files will be used. "
1717                            "GPG error: %s: %s\n"),
1718                          Desc.Description.c_str(),
1719                          LookupTag(Message,"Message").c_str());
1720          RunScripts("APT::Update::Auth-Failure");
1721          return;
1722       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1723          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1724          _error->Error(_("GPG error: %s: %s"),
1725                          Desc.Description.c_str(),
1726                          LookupTag(Message,"Message").c_str());
1727          return;
1728       } else {
1729          _error->Warning(_("GPG error: %s: %s"),
1730                          Desc.Description.c_str(),
1731                          LookupTag(Message,"Message").c_str());
1732       }
1733       // gpgv method failed
1734       ReportMirrorFailure("GPGFailure");
1735    }
1736
1737    /* Always move the meta index, even if gpgv failed. This ensures
1738     * that PackageFile objects are correctly filled in */
1739    if (FileExists(DestFile)) {
1740       string FinalFile = _config->FindDir("Dir::State::lists");
1741       FinalFile += URItoFileName(RealURI);
1742       /* InRelease files become Release files, otherwise
1743        * they would be considered as trusted later on */
1744       if (SigFile == DestFile) {
1745          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1746                                        "Release");
1747          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1748                                        "Release");
1749          SigFile = FinalFile;
1750       }
1751       Rename(DestFile,FinalFile);
1752       chmod(FinalFile.c_str(),0644);
1753
1754       DestFile = FinalFile;
1755    }
1756
1757    // No Release file was present, or verification failed, so fall
1758    // back to queueing Packages files without verification
1759    QueueIndexes(false);
1760 }
1761                                                                         /*}}}*/
1762 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1763                 string const &URI, string const &URIDesc, string const &ShortDesc,
1764                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1765                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1766                 const vector<struct IndexTarget*>* IndexTargets,
1767                 indexRecords* MetaIndexParser) :
1768         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1769         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1770         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1771 {
1772    SigFile = DestFile;
1773
1774    // keep the old InRelease around in case of transistent network errors
1775    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1776    if (RealFileExists(Final) == true)
1777    {
1778       string const LastGoodSig = DestFile + ".reverify";
1779       Rename(Final,LastGoodSig);
1780    }
1781 }
1782                                                                         /*}}}*/
1783 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1784 {
1785    // if the file was never queued undo file-changes done in the constructor
1786    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1787    {
1788       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1789       string const LastGoodSig = DestFile + ".reverify";
1790       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1791          Rename(LastGoodSig, Final);
1792    }
1793 }
1794                                                                         /*}}}*/
1795 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1796 // ---------------------------------------------------------------------
1797 // FIXME: this can go away once the InRelease file is used widely
1798 string pkgAcqMetaClearSig::Custom600Headers()
1799 {
1800    string Final = _config->FindDir("Dir::State::lists");
1801    Final += URItoFileName(RealURI);
1802
1803    struct stat Buf;
1804    if (stat(Final.c_str(),&Buf) != 0)
1805    {
1806       Final = DestFile + ".reverify";
1807       if (stat(Final.c_str(),&Buf) != 0)
1808          return "\nIndex-File: true\nFail-Ignore: true\n";
1809    }
1810
1811    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1812 }
1813                                                                         /*}}}*/
1814 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1815 {
1816    if (AuthPass == false)
1817    {
1818       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1819       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1820       string FinalFile = _config->FindDir("Dir::State::lists");
1821       FinalFile.append(URItoFileName(RealURI));
1822       if (FileExists(FinalFile))
1823          unlink(FinalFile.c_str());
1824
1825       new pkgAcqMetaSig(Owner,
1826                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1827                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1828                         IndexTargets, MetaIndexParser);
1829       if (Cnf->LocalOnly == true ||
1830           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1831          Dequeue();
1832    }
1833    else
1834       pkgAcqMetaIndex::Failed(Message, Cnf);
1835 }
1836                                                                         /*}}}*/
1837 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1838 // ---------------------------------------------------------------------
1839 /* This just sets up the initial fetch environment and queues the first
1840    possibilitiy */
1841 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1842                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1843                              string &StoreFilename) :
1844                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1845                StoreFilename(StoreFilename), Vf(Version.FileList()),
1846                Trusted(false)
1847 {
1848    Retries = _config->FindI("Acquire::Retries",0);
1849
1850    if (Version.Arch() == 0)
1851    {
1852       _error->Error(_("I wasn't able to locate a file for the %s package. "
1853                       "This might mean you need to manually fix this package. "
1854                       "(due to missing arch)"),
1855                     Version.ParentPkg().Name());
1856       return;
1857    }
1858
1859    /* We need to find a filename to determine the extension. We make the
1860       assumption here that all the available sources for this version share
1861       the same extension.. */
1862    // Skip not source sources, they do not have file fields.
1863    for (; Vf.end() == false; ++Vf)
1864    {
1865       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1866          continue;
1867       break;
1868    }
1869
1870    // Does not really matter here.. we are going to fail out below
1871    if (Vf.end() != true)
1872    {
1873       // If this fails to get a file name we will bomb out below.
1874       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1875       if (_error->PendingError() == true)
1876          return;
1877
1878       // Generate the final file name as: package_version_arch.foo
1879       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1880                       QuoteString(Version.VerStr(),"_:") + '_' +
1881                       QuoteString(Version.Arch(),"_:.") +
1882                       "." + flExtension(Parse.FileName());
1883    }
1884
1885    // check if we have one trusted source for the package. if so, switch
1886    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1887    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1888    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1889    bool seenUntrusted = false;
1890    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1891    {
1892       pkgIndexFile *Index;
1893       if (Sources->FindIndex(i.File(),Index) == false)
1894          continue;
1895
1896       if (debugAuth == true)
1897          std::cerr << "Checking index: " << Index->Describe()
1898                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1899
1900       if (Index->IsTrusted() == true)
1901       {
1902          Trusted = true;
1903          if (allowUnauth == false)
1904             break;
1905       }
1906       else
1907          seenUntrusted = true;
1908    }
1909
1910    // "allow-unauthenticated" restores apts old fetching behaviour
1911    // that means that e.g. unauthenticated file:// uris are higher
1912    // priority than authenticated http:// uris
1913    if (allowUnauth == true && seenUntrusted == true)
1914       Trusted = false;
1915
1916    // Select a source
1917    if (QueueNext() == false && _error->PendingError() == false)
1918       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1919                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1920 }
1921                                                                         /*}}}*/
1922 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1923 // ---------------------------------------------------------------------
1924 /* This queues the next available file version for download. It checks if
1925    the archive is already available in the cache and stashs the MD5 for
1926    checking later. */
1927 bool pkgAcqArchive::QueueNext()
1928 {
1929    string const ForceHash = _config->Find("Acquire::ForceHash");
1930    for (; Vf.end() == false; ++Vf)
1931    {
1932       // Ignore not source sources
1933       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1934          continue;
1935
1936       // Try to cross match against the source list
1937       pkgIndexFile *Index;
1938       if (Sources->FindIndex(Vf.File(),Index) == false)
1939             continue;
1940
1941       // only try to get a trusted package from another source if that source
1942       // is also trusted
1943       if(Trusted && !Index->IsTrusted())
1944          continue;
1945
1946       // Grab the text package record
1947       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1948       if (_error->PendingError() == true)
1949          return false;
1950
1951       string PkgFile = Parse.FileName();
1952       if (ForceHash.empty() == false)
1953       {
1954          if(stringcasecmp(ForceHash, "sha512") == 0)
1955             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1956          else if(stringcasecmp(ForceHash, "sha256") == 0)
1957             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1958          else if (stringcasecmp(ForceHash, "sha1") == 0)
1959             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1960          else
1961             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1962       }
1963       else
1964       {
1965          string Hash;
1966          if ((Hash = Parse.SHA512Hash()).empty() == false)
1967             ExpectedHash = HashString("SHA512", Hash);
1968          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1969             ExpectedHash = HashString("SHA256", Hash);
1970          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1971             ExpectedHash = HashString("SHA1", Hash);
1972          else
1973             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1974       }
1975       if (PkgFile.empty() == true)
1976          return _error->Error(_("The package index files are corrupted. No Filename: "
1977                               "field for package %s."),
1978                               Version.ParentPkg().Name());
1979
1980       Desc.URI = Index->ArchiveURI(PkgFile);
1981       Desc.Description = Index->ArchiveInfo(Version);
1982       Desc.Owner = this;
1983       Desc.ShortDesc = Version.ParentPkg().Name();
1984
1985       // See if we already have the file. (Legacy filenames)
1986       FileSize = Version->Size;
1987       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1988       struct stat Buf;
1989       if (stat(FinalFile.c_str(),&Buf) == 0)
1990       {
1991          // Make sure the size matches
1992          if ((unsigned long long)Buf.st_size == Version->Size)
1993          {
1994             Complete = true;
1995             Local = true;
1996             Status = StatDone;
1997             StoreFilename = DestFile = FinalFile;
1998             return true;
1999          }
2000
2001          /* Hmm, we have a file and its size does not match, this means it is
2002             an old style mismatched arch */
2003          unlink(FinalFile.c_str());
2004       }
2005
2006       // Check it again using the new style output filenames
2007       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2008       if (stat(FinalFile.c_str(),&Buf) == 0)
2009       {
2010          // Make sure the size matches
2011          if ((unsigned long long)Buf.st_size == Version->Size)
2012          {
2013             Complete = true;
2014             Local = true;
2015             Status = StatDone;
2016             StoreFilename = DestFile = FinalFile;
2017             return true;
2018          }
2019
2020          /* Hmm, we have a file and its size does not match, this shouldnt
2021             happen.. */
2022          unlink(FinalFile.c_str());
2023       }
2024
2025       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2026
2027       // Check the destination file
2028       if (stat(DestFile.c_str(),&Buf) == 0)
2029       {
2030          // Hmm, the partial file is too big, erase it
2031          if ((unsigned long long)Buf.st_size > Version->Size)
2032             unlink(DestFile.c_str());
2033          else
2034             PartialSize = Buf.st_size;
2035       }
2036
2037       // Disables download of archives - useful if no real installation follows,
2038       // e.g. if we are just interested in proposed installation order
2039       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2040       {
2041          Complete = true;
2042          Local = true;
2043          Status = StatDone;
2044          StoreFilename = DestFile = FinalFile;
2045          return true;
2046       }
2047
2048       // Create the item
2049       Local = false;
2050       Desc.URI = Index->ArchiveURI(PkgFile);
2051       Desc.Description = Index->ArchiveInfo(Version);
2052       Desc.Owner = this;
2053       Desc.ShortDesc = Version.ParentPkg().Name();
2054       QueueURI(Desc);
2055
2056       ++Vf;
2057       return true;
2058    }
2059    return false;
2060 }
2061                                                                         /*}}}*/
2062 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2063 // ---------------------------------------------------------------------
2064 /* */
2065 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2066                          pkgAcquire::MethodConfig *Cfg)
2067 {
2068    Item::Done(Message,Size,CalcHash,Cfg);
2069
2070    // Check the size
2071    if (Size != Version->Size)
2072    {
2073       RenameOnError(SizeMismatch);
2074       return;
2075    }
2076
2077    // Check the hash
2078    if(ExpectedHash.toStr() != CalcHash)
2079    {
2080       RenameOnError(HashSumMismatch);
2081       return;
2082    }
2083
2084    // Grab the output filename
2085    string FileName = LookupTag(Message,"Filename");
2086    if (FileName.empty() == true)
2087    {
2088       Status = StatError;
2089       ErrorText = "Method gave a blank filename";
2090       return;
2091    }
2092
2093    Complete = true;
2094
2095    // Reference filename
2096    if (FileName != DestFile)
2097    {
2098       StoreFilename = DestFile = FileName;
2099       Local = true;
2100       return;
2101    }
2102
2103    // Done, move it into position
2104    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2105    FinalFile += flNotDir(StoreFilename);
2106    Rename(DestFile,FinalFile);
2107
2108    StoreFilename = DestFile = FinalFile;
2109    Complete = true;
2110 }
2111                                                                         /*}}}*/
2112 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2113 // ---------------------------------------------------------------------
2114 /* Here we try other sources */
2115 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2116 {
2117    ErrorText = LookupTag(Message,"Message");
2118
2119    /* We don't really want to retry on failed media swaps, this prevents
2120       that. An interesting observation is that permanent failures are not
2121       recorded. */
2122    if (Cnf->Removable == true &&
2123        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2124    {
2125       // Vf = Version.FileList();
2126       while (Vf.end() == false) ++Vf;
2127       StoreFilename = string();
2128       Item::Failed(Message,Cnf);
2129       return;
2130    }
2131
2132    if (QueueNext() == false)
2133    {
2134       // This is the retry counter
2135       if (Retries != 0 &&
2136           Cnf->LocalOnly == false &&
2137           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2138       {
2139          Retries--;
2140          Vf = Version.FileList();
2141          if (QueueNext() == true)
2142             return;
2143       }
2144
2145       StoreFilename = string();
2146       Item::Failed(Message,Cnf);
2147    }
2148 }
2149                                                                         /*}}}*/
2150 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2151 // ---------------------------------------------------------------------
2152 bool pkgAcqArchive::IsTrusted()
2153 {
2154    return Trusted;
2155 }
2156                                                                         /*}}}*/
2157 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2158 // ---------------------------------------------------------------------
2159 /* */
2160 void pkgAcqArchive::Finished()
2161 {
2162    if (Status == pkgAcquire::Item::StatDone &&
2163        Complete == true)
2164       return;
2165    StoreFilename = string();
2166 }
2167                                                                         /*}}}*/
2168 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2169 // ---------------------------------------------------------------------
2170 /* The file is added to the queue */
2171 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2172                        unsigned long long Size,string Dsc,string ShortDesc,
2173                        const string &DestDir, const string &DestFilename,
2174                        bool IsIndexFile) :
2175                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2176 {
2177    Retries = _config->FindI("Acquire::Retries",0);
2178
2179    if(!DestFilename.empty())
2180       DestFile = DestFilename;
2181    else if(!DestDir.empty())
2182       DestFile = DestDir + "/" + flNotDir(URI);
2183    else
2184       DestFile = flNotDir(URI);
2185
2186    // Create the item
2187    Desc.URI = URI;
2188    Desc.Description = Dsc;
2189    Desc.Owner = this;
2190
2191    // Set the short description to the archive component
2192    Desc.ShortDesc = ShortDesc;
2193
2194    // Get the transfer sizes
2195    FileSize = Size;
2196    struct stat Buf;
2197    if (stat(DestFile.c_str(),&Buf) == 0)
2198    {
2199       // Hmm, the partial file is too big, erase it
2200       if ((unsigned long long)Buf.st_size > Size)
2201          unlink(DestFile.c_str());
2202       else
2203          PartialSize = Buf.st_size;
2204    }
2205
2206    QueueURI(Desc);
2207 }
2208                                                                         /*}}}*/
2209 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2210 // ---------------------------------------------------------------------
2211 /* */
2212 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2213                       pkgAcquire::MethodConfig *Cnf)
2214 {
2215    Item::Done(Message,Size,CalcHash,Cnf);
2216
2217    // Check the hash
2218    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2219    {
2220       RenameOnError(HashSumMismatch);
2221       return;
2222    }
2223
2224    string FileName = LookupTag(Message,"Filename");
2225    if (FileName.empty() == true)
2226    {
2227       Status = StatError;
2228       ErrorText = "Method gave a blank filename";
2229       return;
2230    }
2231
2232    Complete = true;
2233
2234    // The files timestamp matches
2235    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2236       return;
2237
2238    // We have to copy it into place
2239    if (FileName != DestFile)
2240    {
2241       Local = true;
2242       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2243           Cnf->Removable == true)
2244       {
2245          Desc.URI = "copy:" + FileName;
2246          QueueURI(Desc);
2247          return;
2248       }
2249
2250       // Erase the file if it is a symlink so we can overwrite it
2251       struct stat St;
2252       if (lstat(DestFile.c_str(),&St) == 0)
2253       {
2254          if (S_ISLNK(St.st_mode) != 0)
2255             unlink(DestFile.c_str());
2256       }
2257
2258       // Symlink the file
2259       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2260       {
2261          ErrorText = "Link to " + DestFile + " failure ";
2262          Status = StatError;
2263          Complete = false;
2264       }
2265    }
2266 }
2267                                                                         /*}}}*/
2268 // AcqFile::Failed - Failure handler                                    /*{{{*/
2269 // ---------------------------------------------------------------------
2270 /* Here we try other sources */
2271 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2272 {
2273    ErrorText = LookupTag(Message,"Message");
2274
2275    // This is the retry counter
2276    if (Retries != 0 &&
2277        Cnf->LocalOnly == false &&
2278        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2279    {
2280       Retries--;
2281       QueueURI(Desc);
2282       return;
2283    }
2284
2285    Item::Failed(Message,Cnf);
2286 }
2287                                                                         /*}}}*/
2288 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2289 // ---------------------------------------------------------------------
2290 /* The only header we use is the last-modified header. */
2291 string pkgAcqFile::Custom600Headers()
2292 {
2293    if (IsIndexFile)
2294       return "\nIndex-File: true";
2295    return "";
2296 }
2297                                                                         /*}}}*/