apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by alot of item methods as thier final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 147 {
 148    if(FileExists(DestFile))
 149       Rename(DestFile, DestFile + ".FAILED");
 150
 151    switch (error)
 152    {
 153       case HashSumMismatch:
 154          ErrorText = _("Hash Sum mismatch");
 155          Status = StatAuthError;
 156          ReportMirrorFailure("HashChecksumFailure");
 157          break;
 158       case SizeMismatch:
 159          ErrorText = _("Size mismatch");
 160          Status = StatAuthError;
 161          ReportMirrorFailure("SizeFailure");
 162          break;
 163       case InvalidFormat:
 164          ErrorText = _("Invalid file format");
 165          Status = StatError;
 166          // do not report as usually its not the mirrors fault, but Portal/Proxy
 167          break;
 168    }
 169    return false;
 170 }
 171                                                                         /*}}}*/
 172 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 173 // ---------------------------------------------------------------------
 174 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 175 {
 176    // we only act if a mirror was used at all
 177    if(UsedMirror.empty())
 178       return;
 179 #if 0
 180    std::cerr << "\nReportMirrorFailure: "
 181              << UsedMirror
 182              << " Uri: " << DescURI()
 183              << " FailCode: "
 184              << FailCode << std::endl;
 185 #endif
 186    const char *Args[40];
 187    unsigned int i = 0;
 188    string report = _config->Find("Methods::Mirror::ProblemReporting",
 189                                  "/usr/lib/apt/apt-report-mirror-failure");
 190    if(!FileExists(report))
 191       return;
 192    Args[i++] = report.c_str();
 193    Args[i++] = UsedMirror.c_str();
 194    Args[i++] = DescURI().c_str();
 195    Args[i++] = FailCode.c_str();
 196    Args[i++] = NULL;
 197    pid_t pid = ExecFork();
 198    if(pid < 0)
 199    {
 200       _error->Error("ReportMirrorFailure Fork failed");
 201       return;
 202    }
 203    else if(pid == 0)
 204    {
 205       execvp(Args[0], (char**)Args);
 206       std::cerr << "Could not exec " << Args[0] << std::endl;
 207       _exit(100);
 208    }
 209    if(!ExecWait(pid, "report-mirror-failure"))
 210    {
 211       _error->Warning("Couldn't report problem to '%s'",
 212                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 213    }
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* Get a sub-index file based on checksums from a 'master' file and
 219    possibly query additional files */
 220 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 221                                  string const &URIDesc, string const &ShortDesc,
 222                                  HashString const &ExpectedHash)
 223    : Item(Owner), ExpectedHash(ExpectedHash)
 224 {
 225    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 226    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 227
 228    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 229    DestFile += URItoFileName(URI);
 230
 231    Desc.URI = URI;
 232    Desc.Description = URIDesc;
 233    Desc.Owner = this;
 234    Desc.ShortDesc = ShortDesc;
 235
 236    QueueURI(Desc);
 237
 238    if(Debug)
 239       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 240 }
 241                                                                         /*}}}*/
 242 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 243 // ---------------------------------------------------------------------
 244 /* The only header we use is the last-modified header. */
 245 string pkgAcqSubIndex::Custom600Headers()
 246 {
 247    string Final = _config->FindDir("Dir::State::lists");
 248    Final += URItoFileName(Desc.URI);
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true\nFail-Ignore: true\n";
 253    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255                                                                         /*}}}*/
 256 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 257 {
 258    if(Debug)
 259       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 260
 261    Complete = false;
 262    Status = StatDone;
 263    Dequeue();
 264
 265    // No good Index is provided
 266 }
 267                                                                         /*}}}*/
 268 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 269                            pkgAcquire::MethodConfig *Cnf)
 270 {
 271    if(Debug)
 272       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 273
 274    string FileName = LookupTag(Message,"Filename");
 275    if (FileName.empty() == true)
 276    {
 277       Status = StatError;
 278       ErrorText = "Method gave a blank filename";
 279       return;
 280    }
 281
 282    if (FileName != DestFile)
 283    {
 284       Local = true;
 285       Desc.URI = "copy:" + FileName;
 286       QueueURI(Desc);
 287       return;
 288    }
 289
 290    Item::Done(Message,Size,Md5Hash,Cnf);
 291
 292    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 293
 294    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 295    indexRecords SubIndexParser;
 296    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 297       Status = StatError;
 298       ErrorText = SubIndexParser.ErrorText;
 299       return;
 300    }
 301
 302    // sucess in downloading the index
 303    // rename the index
 304    if(Debug)
 305       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 306    Rename(DestFile,FinalFile);
 307    chmod(FinalFile.c_str(),0644);
 308    DestFile = FinalFile;
 309
 310    if(ParseIndex(DestFile) == false)
 311       return Failed("", NULL);
 312
 313    Complete = true;
 314    Status = StatDone;
 315    Dequeue();
 316    return;
 317 }
 318                                                                         /*}}}*/
 319 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 320 {
 321    indexRecords SubIndexParser;
 322    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 323       return false;
 324    // so something with the downloaded index
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* Get the DiffIndex file first and see if there are patches availabe
 331  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 332  * patches. If anything goes wrong in that process, it will fall back to
 333  * the original packages file
 334  */
 335 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 336                                  string URI,string URIDesc,string ShortDesc,
 337                                  HashString ExpectedHash)
 338    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 339      Description(URIDesc)
 340 {
 341
 342    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 343
 344    Desc.Description = URIDesc + "/DiffIndex";
 345    Desc.Owner = this;
 346    Desc.ShortDesc = ShortDesc;
 347    Desc.URI = URI + ".diff/Index";
 348
 349    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 350    DestFile += URItoFileName(URI) + string(".DiffIndex");
 351
 352    if(Debug)
 353       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 354
 355    // look for the current package file
 356    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 357    CurrentPackagesFile += URItoFileName(RealURI);
 358
 359    // FIXME: this file:/ check is a hack to prevent fetching
 360    //        from local sources. this is really silly, and
 361    //        should be fixed cleanly as soon as possible
 362    if(!FileExists(CurrentPackagesFile) ||
 363       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 364    {
 365       // we don't have a pkg file or we don't want to queue
 366       if(Debug)
 367          std::clog << "No index file, local or canceld by user" << std::endl;
 368       Failed("", NULL);
 369       return;
 370    }
 371
 372    if(Debug)
 373       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 374          << CurrentPackagesFile << std::endl;
 375
 376    QueueURI(Desc);
 377
 378 }
 379                                                                         /*}}}*/
 380 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* The only header we use is the last-modified header. */
 383 string pkgAcqDiffIndex::Custom600Headers()
 384 {
 385    string Final = _config->FindDir("Dir::State::lists");
 386    Final += URItoFileName(RealURI) + string(".IndexDiff");
 387
 388    if(Debug)
 389       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 390
 391    struct stat Buf;
 392    if (stat(Final.c_str(),&Buf) != 0)
 393       return "\nIndex-File: true";
 394
 395    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 396 }
 397                                                                         /*}}}*/
 398 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 399 {
 400    if(Debug)
 401       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 402          << std::endl;
 403
 404    pkgTagSection Tags;
 405    string ServerSha1;
 406    vector<DiffInfo> available_patches;
 407
 408    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 409    pkgTagFile TF(&Fd);
 410    if (_error->PendingError() == true)
 411       return false;
 412
 413    if(TF.Step(Tags) == true)
 414    {
 415       bool found = false;
 416       DiffInfo d;
 417       string size;
 418
 419       string const tmp = Tags.FindS("SHA1-Current");
 420       std::stringstream ss(tmp);
 421       ss >> ServerSha1 >> size;
 422       unsigned long const ServerSize = atol(size.c_str());
 423
 424       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 425       SHA1Summation SHA1;
 426       SHA1.AddFD(fd);
 427       string const local_sha1 = SHA1.Result();
 428
 429       if(local_sha1 == ServerSha1)
 430       {
 431          // we have the same sha1 as the server so we are done here
 432          if(Debug)
 433             std::clog << "Package file is up-to-date" << std::endl;
 434          // list cleanup needs to know that this file as well as the already
 435          // present index is ours, so we create an empty diff to save it for us
 436          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 437                ExpectedHash, ServerSha1, available_patches);
 438          return true;
 439       }
 440       else
 441       {
 442          if(Debug)
 443             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 444
 445          // check the historie and see what patches we need
 446          string const history = Tags.FindS("SHA1-History");
 447          std::stringstream hist(history);
 448          while(hist >> d.sha1 >> size >> d.file)
 449          {
 450             // read until the first match is found
 451             // from that point on, we probably need all diffs
 452             if(d.sha1 == local_sha1)
 453                found=true;
 454             else if (found == false)
 455                continue;
 456
 457             if(Debug)
 458                std::clog << "Need to get diff: " << d.file << std::endl;
 459             available_patches.push_back(d);
 460          }
 461
 462          if (available_patches.empty() == false)
 463          {
 464             // patching with too many files is rather slow compared to a fast download
 465             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 466             if (fileLimit != 0 && fileLimit < available_patches.size())
 467             {
 468                if (Debug)
 469                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 470                         << ") so fallback to complete download" << std::endl;
 471                return false;
 472             }
 473
 474             // see if the patches are too big
 475             found = false; // it was true and it will be true again at the end
 476             d = *available_patches.begin();
 477             string const firstPatch = d.file;
 478             unsigned long patchesSize = 0;
 479             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 480             while(patches >> d.sha1 >> size >> d.file)
 481             {
 482                if (firstPatch == d.file)
 483                   found = true;
 484                else if (found == false)
 485                   continue;
 486
 487                patchesSize += atol(size.c_str());
 488             }
 489             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 490             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 491             {
 492                if (Debug)
 493                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 494                         << ") so fallback to complete download" << std::endl;
 495                return false;
 496             }
 497          }
 498       }
 499
 500       // we have something, queue the next diff
 501       if(found)
 502       {
 503          // queue the diffs
 504          string::size_type const last_space = Description.rfind(" ");
 505          if(last_space != string::npos)
 506             Description.erase(last_space, Description.size()-last_space);
 507
 508          /* decide if we should download patches one by one or in one go:
 509             The first is good if the server merges patches, but many don't so client
 510             based merging can be attempt in which case the second is better.
 511             "bad things" will happen if patches are merged on the server,
 512             but client side merging is attempt as well */
 513          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 514          if (pdiff_merge == true)
 515          {
 516             // reprepro adds this flag if it has merged patches on the server
 517             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 518             pdiff_merge = (precedence != "merged");
 519          }
 520
 521          if (pdiff_merge == false)
 522             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 523                   ExpectedHash, ServerSha1, available_patches);
 524          else
 525          {
 526             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 527             for(size_t i = 0; i < available_patches.size(); ++i)
 528                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 529                      available_patches[i], diffs);
 530          }
 531
 532          Complete = false;
 533          Status = StatDone;
 534          Dequeue();
 535          return true;
 536       }
 537    }
 538
 539    // Nothing found, report and return false
 540    // Failing here is ok, if we return false later, the full
 541    // IndexFile is queued
 542    if(Debug)
 543       std::clog << "Can't find a patch in the index file" << std::endl;
 544    return false;
 545 }
 546                                                                         /*}}}*/
 547 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 548 {
 549    if(Debug)
 550       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 551                 << "Falling back to normal index file aquire" << std::endl;
 552
 553    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 554                    ExpectedHash);
 555
 556    Complete = false;
 557    Status = StatDone;
 558    Dequeue();
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 562                            pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading the index
 573    // rename the index
 574    FinalFile += string(".IndexDiff");
 575    if(Debug)
 576       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 577                 << std::endl;
 578    Rename(DestFile,FinalFile);
 579    chmod(FinalFile.c_str(),0644);
 580    DestFile = FinalFile;
 581
 582    if(!ParseDiffIndex(DestFile))
 583       return Failed("", NULL);
 584
 585    Complete = true;
 586    Status = StatDone;
 587    Dequeue();
 588    return;
 589 }
 590                                                                         /*}}}*/
 591 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* The package diff is added to the queue. one object is constructed
 594  * for each diff and the index
 595  */
 596 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 597                                    string URI,string URIDesc,string ShortDesc,
 598                                    HashString ExpectedHash,
 599                                    string ServerSha1,
 600                                    vector<DiffInfo> diffs)
 601    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 602      available_patches(diffs), ServerSha1(ServerSha1)
 603 {
 604
 605    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 606    DestFile += URItoFileName(URI);
 607
 608    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 609
 610    Description = URIDesc;
 611    Desc.Owner = this;
 612    Desc.ShortDesc = ShortDesc;
 613
 614    if(available_patches.empty() == true)
 615    {
 616       // we are done (yeah!)
 617       Finish(true);
 618    }
 619    else
 620    {
 621       // get the next diff
 622       State = StateFetchDiff;
 623       QueueNextDiff();
 624    }
 625 }
 626                                                                         /*}}}*/
 627 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 628 {
 629    if(Debug)
 630       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 631                 << "Falling back to normal index file aquire" << std::endl;
 632    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 633                    ExpectedHash);
 634    Finish();
 635 }
 636                                                                         /*}}}*/
 637 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 638 void pkgAcqIndexDiffs::Finish(bool allDone)
 639 {
 640    // we restore the original name, this is required, otherwise
 641    // the file will be cleaned
 642    if(allDone)
 643    {
 644       DestFile = _config->FindDir("Dir::State::lists");
 645       DestFile += URItoFileName(RealURI);
 646
 647       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 648       {
 649          RenameOnError(HashSumMismatch);
 650          Dequeue();
 651          return;
 652       }
 653
 654       // this is for the "real" finish
 655       Complete = true;
 656       Status = StatDone;
 657       Dequeue();
 658       if(Debug)
 659          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 660       return;
 661    }
 662
 663    if(Debug)
 664       std::clog << "Finishing: " << Desc.URI << std::endl;
 665    Complete = false;
 666    Status = StatDone;
 667    Dequeue();
 668    return;
 669 }
 670                                                                         /*}}}*/
 671 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 672 {
 673
 674    // calc sha1 of the just patched file
 675    string FinalFile = _config->FindDir("Dir::State::lists");
 676    FinalFile += URItoFileName(RealURI);
 677
 678    FileFd fd(FinalFile, FileFd::ReadOnly);
 679    SHA1Summation SHA1;
 680    SHA1.AddFD(fd);
 681    string local_sha1 = string(SHA1.Result());
 682    if(Debug)
 683       std::clog << "QueueNextDiff: "
 684                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 685
 686    // final file reached before all patches are applied
 687    if(local_sha1 == ServerSha1)
 688    {
 689       Finish(true);
 690       return true;
 691    }
 692
 693    // remove all patches until the next matching patch is found
 694    // this requires the Index file to be ordered
 695    for(vector<DiffInfo>::iterator I=available_patches.begin();
 696        available_patches.empty() == false &&
 697           I != available_patches.end() &&
 698           I->sha1 != local_sha1;
 699        ++I)
 700    {
 701       available_patches.erase(I);
 702    }
 703
 704    // error checking and falling back if no patch was found
 705    if(available_patches.empty() == true)
 706    {
 707       Failed("", NULL);
 708       return false;
 709    }
 710
 711    // queue the right diff
 712    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 713    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 714    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 715    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 716
 717    if(Debug)
 718       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 719
 720    QueueURI(Desc);
 721
 722    return true;
 723 }
 724                                                                         /*}}}*/
 725 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 726                             pkgAcquire::MethodConfig *Cnf)
 727 {
 728    if(Debug)
 729       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 730
 731    Item::Done(Message,Size,Md5Hash,Cnf);
 732
 733    string FinalFile;
 734    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 735
 736    // sucess in downloading a diff, enter ApplyDiff state
 737    if(State == StateFetchDiff)
 738    {
 739
 740       // rred excepts the patch as $FinalFile.ed
 741       Rename(DestFile,FinalFile+".ed");
 742
 743       if(Debug)
 744          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 745
 746       State = StateApplyDiff;
 747       Local = true;
 748       Desc.URI = "rred:" + FinalFile;
 749       QueueURI(Desc);
 750       Mode = "rred";
 751       return;
 752    }
 753
 754
 755    // success in download/apply a diff, queue next (if needed)
 756    if(State == StateApplyDiff)
 757    {
 758       // remove the just applied patch
 759       available_patches.erase(available_patches.begin());
 760
 761       // move into place
 762       if(Debug)
 763       {
 764          std::clog << "Moving patched file in place: " << std::endl
 765                    << DestFile << " -> " << FinalFile << std::endl;
 766       }
 767       Rename(DestFile,FinalFile);
 768       chmod(FinalFile.c_str(),0644);
 769
 770       // see if there is more to download
 771       if(available_patches.empty() == false) {
 772          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 773                               ExpectedHash, ServerSha1, available_patches);
 774          return Finish();
 775       } else
 776          return Finish(true);
 777    }
 778 }
 779                                                                         /*}}}*/
 780 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 781 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 782                                    string const &URI, string const &URIDesc,
 783                                    string const &ShortDesc, HashString const &ExpectedHash,
 784                                    DiffInfo const &patch,
 785                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 786    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 787      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 788 {
 789
 790    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 791    DestFile += URItoFileName(URI);
 792
 793    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 794
 795    Description = URIDesc;
 796    Desc.Owner = this;
 797    Desc.ShortDesc = ShortDesc;
 798
 799    Desc.URI = string(RealURI) + ".diff/" + patch.file + ".gz";
 800    Desc.Description = Description + " " + patch.file + string(".pdiff");
 801    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 802    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 803
 804    if(Debug)
 805       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 806
 807    QueueURI(Desc);
 808 }
 809                                                                         /*}}}*/
 810 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 811 {
 812    if(Debug)
 813       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 814    Complete = false;
 815    Status = StatDone;
 816    Dequeue();
 817
 818    // check if we are the first to fail, otherwise we are done here
 819    State = StateDoneDiff;
 820    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 821          I != allPatches->end(); ++I)
 822       if ((*I)->State == StateErrorDiff)
 823          return;
 824
 825    // first failure means we should fallback
 826    State = StateErrorDiff;
 827    std::clog << "Falling back to normal index file aquire" << std::endl;
 828    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 829                    ExpectedHash);
 830 }
 831                                                                         /*}}}*/
 832 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 833                             pkgAcquire::MethodConfig *Cnf)
 834 {
 835    if(Debug)
 836       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 837
 838    Item::Done(Message,Size,Md5Hash,Cnf);
 839
 840    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 841
 842    if (State == StateFetchDiff)
 843    {
 844       // rred expects the patch as $FinalFile.ed.$patchname.gz
 845       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 846
 847       // check if this is the last completed diff
 848       State = StateDoneDiff;
 849       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 850             I != allPatches->end(); ++I)
 851          if ((*I)->State != StateDoneDiff)
 852          {
 853             if(Debug)
 854                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 855             return;
 856          }
 857
 858       // this is the last completed diff, so we are ready to apply now
 859       State = StateApplyDiff;
 860
 861       if(Debug)
 862          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 863
 864       Local = true;
 865       Desc.URI = "rred:" + FinalFile;
 866       QueueURI(Desc);
 867       Mode = "rred";
 868       return;
 869    }
 870    // success in download/apply all diffs, clean up
 871    else if (State == StateApplyDiff)
 872    {
 873       // see if we really got the expected file
 874       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 875       {
 876          RenameOnError(HashSumMismatch);
 877          return;
 878       }
 879
 880       // move the result into place
 881       if(Debug)
 882          std::clog << "Moving patched file in place: " << std::endl
 883                    << DestFile << " -> " << FinalFile << std::endl;
 884       Rename(DestFile, FinalFile);
 885       chmod(FinalFile.c_str(), 0644);
 886
 887       // otherwise lists cleanup will eat the file
 888       DestFile = FinalFile;
 889
 890       // all set and done
 891       Complete = true;
 892       if(Debug)
 893          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 894    }
 895 }
 896                                                                         /*}}}*/
 897 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 898 // ---------------------------------------------------------------------
 899 /* The package file is added to the queue and a second class is
 900    instantiated to fetch the revision file */
 901 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 902                          string URI,string URIDesc,string ShortDesc,
 903                          HashString ExpectedHash, string comprExt)
 904    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 905 {
 906    if(comprExt.empty() == true)
 907    {
 908       // autoselect the compression method
 909       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 910       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 911          comprExt.append(*t).append(" ");
 912       if (comprExt.empty() == false)
 913          comprExt.erase(comprExt.end()-1);
 914    }
 915    CompressionExtension = comprExt;
 916
 917    Init(URI, URIDesc, ShortDesc);
 918 }
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 920                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 921    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 922 {
 923    // autoselect the compression method
 924    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 925    CompressionExtension = "";
 926    if (ExpectedHash.empty() == false)
 927    {
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 930             CompressionExtension.append(*t).append(" ");
 931    }
 932    else
 933    {
 934       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 935          CompressionExtension.append(*t).append(" ");
 936    }
 937    if (CompressionExtension.empty() == false)
 938       CompressionExtension.erase(CompressionExtension.end()-1);
 939
 940    // only verify non-optional targets, see acquire-item.h for a FIXME
 941    // to make this more flexible
 942    if (Target->IsOptional())
 943      Verify = false;
 944    else
 945      Verify = true;
 946
 947    Init(Target->URI, Target->Description, Target->ShortDesc);
 948 }
 949                                                                         /*}}}*/
 950 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 951 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 952    Decompression = false;
 953    Erase = false;
 954
 955    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 956    DestFile += URItoFileName(URI);
 957
 958    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 959    if (comprExt == "uncompressed")
 960       Desc.URI = URI;
 961    else
 962       Desc.URI = URI + '.' + comprExt;
 963
 964    Desc.Description = URIDesc;
 965    Desc.Owner = this;
 966    Desc.ShortDesc = ShortDesc;
 967
 968    QueueURI(Desc);
 969 }
 970                                                                         /*}}}*/
 971 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 972 // ---------------------------------------------------------------------
 973 /* The only header we use is the last-modified header. */
 974 string pkgAcqIndex::Custom600Headers()
 975 {
 976    string Final = _config->FindDir("Dir::State::lists");
 977    Final += URItoFileName(RealURI);
 978    if (_config->FindB("Acquire::GzipIndexes",false))
 979       Final += ".gz";
 980
 981    string msg = "\nIndex-File: true";
 982    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 983    //        seems to be difficult without breaking ABI
 984    if (ShortDesc().find("Translation") != 0)
 985       msg += "\nFail-Ignore: true";
 986    struct stat Buf;
 987    if (stat(Final.c_str(),&Buf) == 0)
 988       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 989
 990    return msg;
 991 }
 992                                                                         /*}}}*/
 993 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 994 {
 995    size_t const nextExt = CompressionExtension.find(' ');
 996    if (nextExt != std::string::npos)
 997    {
 998       CompressionExtension = CompressionExtension.substr(nextExt+1);
 999       Init(RealURI, Desc.Description, Desc.ShortDesc);
1000       return;
1001    }
1002
1003    // on decompression failure, remove bad versions in partial/
1004    if (Decompression && Erase) {
1005       string s = _config->FindDir("Dir::State::lists") + "partial/";
1006       s.append(URItoFileName(RealURI));
1007       unlink(s.c_str());
1008    }
1009
1010    Item::Failed(Message,Cnf);
1011 }
1012                                                                         /*}}}*/
1013 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1014 // ---------------------------------------------------------------------
1015 /* This goes through a number of states.. On the initial fetch the
1016    method could possibly return an alternate filename which points
1017    to the uncompressed version of the file. If this is so the file
1018    is copied into the partial directory. In all other cases the file
1019    is decompressed with a gzip uri. */
1020 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1021                        pkgAcquire::MethodConfig *Cfg)
1022 {
1023    Item::Done(Message,Size,Hash,Cfg);
1024
1025    if (Decompression == true)
1026    {
1027       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1028       {
1029          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1030          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1031       }
1032
1033       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1034       {
1035          RenameOnError(HashSumMismatch);
1036          return;
1037       }
1038
1039       /* Verify the index file for correctness (all indexes must
1040        * have a Package field) (LP: #346386) (Closes: #627642) */
1041       if (Verify == true)
1042       {
1043          FileFd fd(DestFile, FileFd::ReadOnly);
1044          // Only test for correctness if the file is not empty (empty is ok)
1045          if (fd.FileSize() > 0)
1046          {
1047             pkgTagSection sec;
1048             pkgTagFile tag(&fd);
1049
1050             // all our current indexes have a field 'Package' in each section
1051             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1052             {
1053                RenameOnError(InvalidFormat);
1054                return;
1055             }
1056          }
1057       }
1058
1059       // Done, move it into position
1060       string FinalFile = _config->FindDir("Dir::State::lists");
1061       FinalFile += URItoFileName(RealURI);
1062       Rename(DestFile,FinalFile);
1063       chmod(FinalFile.c_str(),0644);
1064
1065       /* We restore the original name to DestFile so that the clean operation
1066          will work OK */
1067       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1068       DestFile += URItoFileName(RealURI);
1069
1070       // Remove the compressed version.
1071       if (Erase == true)
1072          unlink(DestFile.c_str());
1073       return;
1074    }
1075
1076    Erase = false;
1077    Complete = true;
1078
1079    // Handle the unzipd case
1080    string FileName = LookupTag(Message,"Alt-Filename");
1081    if (FileName.empty() == false)
1082    {
1083       // The files timestamp matches
1084       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1085          return;
1086       Decompression = true;
1087       Local = true;
1088       DestFile += ".decomp";
1089       Desc.URI = "copy:" + FileName;
1090       QueueURI(Desc);
1091       Mode = "copy";
1092       return;
1093    }
1094
1095    FileName = LookupTag(Message,"Filename");
1096    if (FileName.empty() == true)
1097    {
1098       Status = StatError;
1099       ErrorText = "Method gave a blank filename";
1100    }
1101
1102    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1103
1104    // The files timestamp matches
1105    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1106        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1107           // Update DestFile for .gz suffix so that the clean operation keeps it
1108           DestFile += ".gz";
1109       return;
1110     }
1111
1112    if (FileName == DestFile)
1113       Erase = true;
1114    else
1115       Local = true;
1116
1117    string decompProg;
1118
1119    // If we enable compressed indexes and already have gzip, keep it
1120    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1121       string FinalFile = _config->FindDir("Dir::State::lists");
1122       FinalFile += URItoFileName(RealURI) + ".gz";
1123       Rename(DestFile,FinalFile);
1124       chmod(FinalFile.c_str(),0644);
1125
1126       // Update DestFile for .gz suffix so that the clean operation keeps it
1127       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1128       DestFile += URItoFileName(RealURI) + ".gz";
1129       return;
1130     }
1131
1132    // get the binary name for your used compression type
1133    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1134    if(decompProg.empty() == false);
1135    else if(compExt == "uncompressed")
1136       decompProg = "copy";
1137    else {
1138       _error->Error("Unsupported extension: %s", compExt.c_str());
1139       return;
1140    }
1141
1142    Decompression = true;
1143    DestFile += ".decomp";
1144    Desc.URI = decompProg + ":" + FileName;
1145    QueueURI(Desc);
1146
1147    // FIXME: this points to a c++ string that goes out of scope
1148    Mode = decompProg.c_str();
1149 }
1150                                                                         /*}}}*/
1151 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1152 // ---------------------------------------------------------------------
1153 /* The Translation file is added to the queue */
1154 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1155                             string URI,string URIDesc,string ShortDesc)
1156   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1157 {
1158 }
1159 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1160                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1161   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1162 {
1163 }
1164                                                                         /*}}}*/
1165 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1166 // ---------------------------------------------------------------------
1167 string pkgAcqIndexTrans::Custom600Headers()
1168 {
1169    string Final = _config->FindDir("Dir::State::lists");
1170    Final += URItoFileName(RealURI);
1171
1172    struct stat Buf;
1173    if (stat(Final.c_str(),&Buf) != 0)
1174       return "\nFail-Ignore: true\nIndex-File: true";
1175    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1176 }
1177                                                                         /*}}}*/
1178 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1179 // ---------------------------------------------------------------------
1180 /* */
1181 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1182 {
1183    size_t const nextExt = CompressionExtension.find(' ');
1184    if (nextExt != std::string::npos)
1185    {
1186       CompressionExtension = CompressionExtension.substr(nextExt+1);
1187       Init(RealURI, Desc.Description, Desc.ShortDesc);
1188       Status = StatIdle;
1189       return;
1190    }
1191
1192    if (Cnf->LocalOnly == true ||
1193        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1194    {
1195       // Ignore this
1196       Status = StatDone;
1197       Complete = false;
1198       Dequeue();
1199       return;
1200    }
1201
1202    Item::Failed(Message,Cnf);
1203 }
1204                                                                         /*}}}*/
1205 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1206                              string URI,string URIDesc,string ShortDesc,
1207                              string MetaIndexURI, string MetaIndexURIDesc,
1208                              string MetaIndexShortDesc,
1209                              const vector<IndexTarget*>* IndexTargets,
1210                              indexRecords* MetaIndexParser) :
1211    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1212    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1213    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1214 {
1215    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1216    DestFile += URItoFileName(URI);
1217
1218    // remove any partial downloaded sig-file in partial/.
1219    // it may confuse proxies and is too small to warrant a
1220    // partial download anyway
1221    unlink(DestFile.c_str());
1222
1223    // Create the item
1224    Desc.Description = URIDesc;
1225    Desc.Owner = this;
1226    Desc.ShortDesc = ShortDesc;
1227    Desc.URI = URI;
1228
1229    string Final = _config->FindDir("Dir::State::lists");
1230    Final += URItoFileName(RealURI);
1231    if (RealFileExists(Final) == true)
1232    {
1233       // File was already in place.  It needs to be re-downloaded/verified
1234       // because Release might have changed, we do give it a differnt
1235       // name than DestFile because otherwise the http method will
1236       // send If-Range requests and there are too many broken servers
1237       // out there that do not understand them
1238       LastGoodSig = DestFile+".reverify";
1239       Rename(Final,LastGoodSig);
1240    }
1241
1242    QueueURI(Desc);
1243 }
1244                                                                         /*}}}*/
1245 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1246 {
1247    // if the file was never queued undo file-changes done in the constructor
1248    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1249          LastGoodSig.empty() == false)
1250    {
1251       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1252       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1253          Rename(LastGoodSig, Final);
1254    }
1255
1256 }
1257                                                                         /*}}}*/
1258 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1259 // ---------------------------------------------------------------------
1260 /* The only header we use is the last-modified header. */
1261 string pkgAcqMetaSig::Custom600Headers()
1262 {
1263    struct stat Buf;
1264    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1265       return "\nIndex-File: true";
1266
1267    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1268 }
1269
1270 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1271                          pkgAcquire::MethodConfig *Cfg)
1272 {
1273    Item::Done(Message,Size,MD5,Cfg);
1274
1275    string FileName = LookupTag(Message,"Filename");
1276    if (FileName.empty() == true)
1277    {
1278       Status = StatError;
1279       ErrorText = "Method gave a blank filename";
1280       return;
1281    }
1282
1283    if (FileName != DestFile)
1284    {
1285       // We have to copy it into place
1286       Local = true;
1287       Desc.URI = "copy:" + FileName;
1288       QueueURI(Desc);
1289       return;
1290    }
1291
1292    Complete = true;
1293
1294    // put the last known good file back on i-m-s hit (it will
1295    // be re-verified again)
1296    // Else do nothing, we have the new file in DestFile then
1297    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1298       Rename(LastGoodSig, DestFile);
1299
1300    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1301    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1302                        MetaIndexShortDesc,  DestFile, IndexTargets,
1303                        MetaIndexParser);
1304
1305 }
1306                                                                         /*}}}*/
1307 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1308 {
1309    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1310
1311    // if we get a network error we fail gracefully
1312    if(Status == StatTransientNetworkError)
1313    {
1314       Item::Failed(Message,Cnf);
1315       // move the sigfile back on transient network failures
1316       if(FileExists(LastGoodSig))
1317          Rename(LastGoodSig,Final);
1318
1319       // set the status back to , Item::Failed likes to reset it
1320       Status = pkgAcquire::Item::StatTransientNetworkError;
1321       return;
1322    }
1323
1324    // Delete any existing sigfile when the acquire failed
1325    unlink(Final.c_str());
1326
1327    // queue a pkgAcqMetaIndex with no sigfile
1328    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1329                        "", IndexTargets, MetaIndexParser);
1330
1331    if (Cnf->LocalOnly == true ||
1332        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1333    {
1334       // Ignore this
1335       Status = StatDone;
1336       Complete = false;
1337       Dequeue();
1338       return;
1339    }
1340
1341    Item::Failed(Message,Cnf);
1342 }
1343                                                                         /*}}}*/
1344 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1345                                  string URI,string URIDesc,string ShortDesc,
1346                                  string SigFile,
1347                                  const vector<struct IndexTarget*>* IndexTargets,
1348                                  indexRecords* MetaIndexParser) :
1349    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1350    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1351 {
1352    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1353    DestFile += URItoFileName(URI);
1354
1355    // Create the item
1356    Desc.Description = URIDesc;
1357    Desc.Owner = this;
1358    Desc.ShortDesc = ShortDesc;
1359    Desc.URI = URI;
1360
1361    QueueURI(Desc);
1362 }
1363                                                                         /*}}}*/
1364 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1365 // ---------------------------------------------------------------------
1366 /* The only header we use is the last-modified header. */
1367 string pkgAcqMetaIndex::Custom600Headers()
1368 {
1369    string Final = _config->FindDir("Dir::State::lists");
1370    Final += URItoFileName(RealURI);
1371
1372    struct stat Buf;
1373    if (stat(Final.c_str(),&Buf) != 0)
1374       return "\nIndex-File: true";
1375
1376    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1377 }
1378                                                                         /*}}}*/
1379 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1380                            pkgAcquire::MethodConfig *Cfg)
1381 {
1382    Item::Done(Message,Size,Hash,Cfg);
1383
1384    // MetaIndexes are done in two passes: one to download the
1385    // metaindex with an appropriate method, and a second to verify it
1386    // with the gpgv method
1387
1388    if (AuthPass == true)
1389    {
1390       AuthDone(Message);
1391
1392       // all cool, move Release file into place
1393       Complete = true;
1394    }
1395    else
1396    {
1397       RetrievalDone(Message);
1398       if (!Complete)
1399          // Still more retrieving to do
1400          return;
1401
1402       if (SigFile == "")
1403       {
1404          // There was no signature file, so we are finished.  Download
1405          // the indexes and do only hashsum verification if possible
1406          MetaIndexParser->Load(DestFile);
1407          QueueIndexes(false);
1408       }
1409       else
1410       {
1411          // There was a signature file, so pass it to gpgv for
1412          // verification
1413
1414          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1415             std::cerr << "Metaindex acquired, queueing gpg verification ("
1416                       << SigFile << "," << DestFile << ")\n";
1417          AuthPass = true;
1418          Desc.URI = "gpgv:" + SigFile;
1419          QueueURI(Desc);
1420          Mode = "gpgv";
1421          return;
1422       }
1423    }
1424
1425    if (Complete == true)
1426    {
1427       string FinalFile = _config->FindDir("Dir::State::lists");
1428       FinalFile += URItoFileName(RealURI);
1429       if (SigFile == DestFile)
1430          SigFile = FinalFile;
1431       Rename(DestFile,FinalFile);
1432       chmod(FinalFile.c_str(),0644);
1433       DestFile = FinalFile;
1434    }
1435 }
1436                                                                         /*}}}*/
1437 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1438 {
1439    // We have just finished downloading a Release file (it is not
1440    // verified yet)
1441
1442    string FileName = LookupTag(Message,"Filename");
1443    if (FileName.empty() == true)
1444    {
1445       Status = StatError;
1446       ErrorText = "Method gave a blank filename";
1447       return;
1448    }
1449
1450    if (FileName != DestFile)
1451    {
1452       Local = true;
1453       Desc.URI = "copy:" + FileName;
1454       QueueURI(Desc);
1455       return;
1456    }
1457
1458    // make sure to verify against the right file on I-M-S hit
1459    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1460    if(IMSHit)
1461    {
1462       string FinalFile = _config->FindDir("Dir::State::lists");
1463       FinalFile += URItoFileName(RealURI);
1464       if (SigFile == DestFile)
1465       {
1466          SigFile = FinalFile;
1467          // constructor of pkgAcqMetaClearSig moved it out of the way,
1468          // now move it back in on IMS hit for the 'old' file
1469          string const OldClearSig = DestFile + ".reverify";
1470          if (RealFileExists(OldClearSig) == true)
1471             Rename(OldClearSig, FinalFile);
1472       }
1473       DestFile = FinalFile;
1474    }
1475    Complete = true;
1476 }
1477                                                                         /*}}}*/
1478 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1479 {
1480    // At this point, the gpgv method has succeeded, so there is a
1481    // valid signature from a key in the trusted keyring.  We
1482    // perform additional verification of its contents, and use them
1483    // to verify the indexes we are about to download
1484
1485    if (!MetaIndexParser->Load(DestFile))
1486    {
1487       Status = StatAuthError;
1488       ErrorText = MetaIndexParser->ErrorText;
1489       return;
1490    }
1491
1492    if (!VerifyVendor(Message))
1493    {
1494       return;
1495    }
1496
1497    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1498       std::cerr << "Signature verification succeeded: "
1499                 << DestFile << std::endl;
1500
1501    // Download further indexes with verification
1502    QueueIndexes(true);
1503
1504    // is it a clearsigned MetaIndex file?
1505    if (DestFile == SigFile)
1506       return;
1507
1508    // Done, move signature file into position
1509    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1510       URItoFileName(RealURI) + ".gpg";
1511    Rename(SigFile,VerifiedSigFile);
1512    chmod(VerifiedSigFile.c_str(),0644);
1513 }
1514                                                                         /*}}}*/
1515 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1516 {
1517 #if 0
1518    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1519     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1520    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1521    {
1522       Status = StatError;
1523       ErrorText = MetaIndexParser->ErrorText;
1524       return;
1525    }
1526 #endif
1527    bool transInRelease = false;
1528    {
1529       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1530       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1531          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1532          if (k->find("Translation-") != std::string::npos)
1533          {
1534             transInRelease = true;
1535             break;
1536          }
1537    }
1538
1539    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1540         Target != IndexTargets->end();
1541         ++Target)
1542    {
1543       HashString ExpectedIndexHash;
1544       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1545       bool compressedAvailable = false;
1546       if (Record == NULL)
1547       {
1548          if ((*Target)->IsOptional() == true)
1549          {
1550             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1551             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1552                if (MetaIndexParser->Exists(string((*Target)->MetaKey).append(".").append(*t)) == true)
1553                {
1554                   compressedAvailable = true;
1555                   break;
1556                }
1557          }
1558          else if (verify == true)
1559          {
1560             Status = StatAuthError;
1561             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1562             return;
1563          }
1564       }
1565       else
1566       {
1567          ExpectedIndexHash = Record->Hash;
1568          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1569          {
1570             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1571             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1572             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1573          }
1574          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1575          {
1576             Status = StatAuthError;
1577             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1578             return;
1579          }
1580       }
1581
1582       if ((*Target)->IsOptional() == true)
1583       {
1584          if ((*Target)->IsSubIndex() == true)
1585             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1586                                 (*Target)->ShortDesc, ExpectedIndexHash);
1587          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1588          {
1589             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1590                 MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)
1591                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1592                                    (*Target)->ShortDesc, ExpectedIndexHash);
1593             else
1594                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1595          }
1596          continue;
1597       }
1598
1599       /* Queue Packages file (either diff or full packages files, depending
1600          on the users option) - we also check if the PDiff Index file is listed
1601          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1602          instead, but passing the required info to it is to much hassle */
1603       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1604           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1605          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1606                              (*Target)->ShortDesc, ExpectedIndexHash);
1607       else
1608          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1609    }
1610 }
1611                                                                         /*}}}*/
1612 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1613 {
1614    string::size_type pos;
1615
1616    // check for missing sigs (that where not fatal because otherwise we had
1617    // bombed earlier)
1618    string missingkeys;
1619    string msg = _("There is no public key available for the "
1620                   "following key IDs:\n");
1621    pos = Message.find("NO_PUBKEY ");
1622    if (pos != std::string::npos)
1623    {
1624       string::size_type start = pos+strlen("NO_PUBKEY ");
1625       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1626       missingkeys += (Fingerprint);
1627    }
1628    if(!missingkeys.empty())
1629       _error->Warning("%s", string(msg+missingkeys).c_str());
1630
1631    string Transformed = MetaIndexParser->GetExpectedDist();
1632
1633    if (Transformed == "../project/experimental")
1634    {
1635       Transformed = "experimental";
1636    }
1637
1638    pos = Transformed.rfind('/');
1639    if (pos != string::npos)
1640    {
1641       Transformed = Transformed.substr(0, pos);
1642    }
1643
1644    if (Transformed == ".")
1645    {
1646       Transformed = "";
1647    }
1648
1649    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1650        MetaIndexParser->GetValidUntil() > 0) {
1651       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1652       if (invalid_since > 0)
1653          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1654          // the time since then the file is invalid - formated in the same way as in
1655          // the download progress display (e.g. 7d 3h 42min 1s)
1656          return _error->Error(
1657             _("Release file for %s is expired (invalid since %s). "
1658               "Updates for this repository will not be applied."),
1659             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1660    }
1661
1662    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1663    {
1664       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1665       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1666       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1667    }
1668
1669    if (MetaIndexParser->CheckDist(Transformed) == false)
1670    {
1671       // This might become fatal one day
1672 //       Status = StatAuthError;
1673 //       ErrorText = "Conflicting distribution; expected "
1674 //          + MetaIndexParser->GetExpectedDist() + " but got "
1675 //          + MetaIndexParser->GetDist();
1676 //       return false;
1677       if (!Transformed.empty())
1678       {
1679          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1680                          Desc.Description.c_str(),
1681                          Transformed.c_str(),
1682                          MetaIndexParser->GetDist().c_str());
1683       }
1684    }
1685
1686    return true;
1687 }
1688                                                                         /*}}}*/
1689 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1690 // ---------------------------------------------------------------------
1691 /* */
1692 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1693 {
1694    if (AuthPass == true)
1695    {
1696       // gpgv method failed, if we have a good signature
1697       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1698       if (DestFile != SigFile)
1699          LastGoodSigFile.append(".gpg");
1700       LastGoodSigFile.append(".reverify");
1701
1702       if(FileExists(LastGoodSigFile))
1703       {
1704          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1705          if (DestFile != SigFile)
1706             VerifiedSigFile.append(".gpg");
1707          Rename(LastGoodSigFile, VerifiedSigFile);
1708          Status = StatTransientNetworkError;
1709          _error->Warning(_("An error occurred during the signature "
1710                            "verification. The repository is not updated "
1711                            "and the previous index files will be used. "
1712                            "GPG error: %s: %s\n"),
1713                          Desc.Description.c_str(),
1714                          LookupTag(Message,"Message").c_str());
1715          RunScripts("APT::Update::Auth-Failure");
1716          return;
1717       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1718          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1719          _error->Error(_("GPG error: %s: %s"),
1720                          Desc.Description.c_str(),
1721                          LookupTag(Message,"Message").c_str());
1722          return;
1723       } else {
1724          _error->Warning(_("GPG error: %s: %s"),
1725                          Desc.Description.c_str(),
1726                          LookupTag(Message,"Message").c_str());
1727       }
1728       // gpgv method failed
1729       ReportMirrorFailure("GPGFailure");
1730    }
1731
1732    /* Always move the meta index, even if gpgv failed. This ensures
1733     * that PackageFile objects are correctly filled in */
1734    if (FileExists(DestFile)) {
1735       string FinalFile = _config->FindDir("Dir::State::lists");
1736       FinalFile += URItoFileName(RealURI);
1737       /* InRelease files become Release files, otherwise
1738        * they would be considered as trusted later on */
1739       if (SigFile == DestFile) {
1740          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1741                                        "Release");
1742          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1743                                        "Release");
1744          SigFile = FinalFile;
1745       }
1746       Rename(DestFile,FinalFile);
1747       chmod(FinalFile.c_str(),0644);
1748
1749       DestFile = FinalFile;
1750    }
1751
1752    // No Release file was present, or verification failed, so fall
1753    // back to queueing Packages files without verification
1754    QueueIndexes(false);
1755 }
1756                                                                         /*}}}*/
1757 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1758                 string const &URI, string const &URIDesc, string const &ShortDesc,
1759                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1760                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1761                 const vector<struct IndexTarget*>* IndexTargets,
1762                 indexRecords* MetaIndexParser) :
1763         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1764         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1765         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1766 {
1767    SigFile = DestFile;
1768
1769    // keep the old InRelease around in case of transistent network errors
1770    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1771    if (RealFileExists(Final) == true)
1772    {
1773       string const LastGoodSig = DestFile + ".reverify";
1774       Rename(Final,LastGoodSig);
1775    }
1776 }
1777                                                                         /*}}}*/
1778 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1779 {
1780    // if the file was never queued undo file-changes done in the constructor
1781    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1782    {
1783       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1784       string const LastGoodSig = DestFile + ".reverify";
1785       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1786          Rename(LastGoodSig, Final);
1787    }
1788 }
1789                                                                         /*}}}*/
1790 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1791 // ---------------------------------------------------------------------
1792 // FIXME: this can go away once the InRelease file is used widely
1793 string pkgAcqMetaClearSig::Custom600Headers()
1794 {
1795    string Final = _config->FindDir("Dir::State::lists");
1796    Final += URItoFileName(RealURI);
1797
1798    struct stat Buf;
1799    if (stat(Final.c_str(),&Buf) != 0)
1800    {
1801       Final = DestFile + ".reverify";
1802       if (stat(Final.c_str(),&Buf) != 0)
1803          return "\nIndex-File: true\nFail-Ignore: true\n";
1804    }
1805
1806    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1807 }
1808                                                                         /*}}}*/
1809 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1810 {
1811    if (AuthPass == false)
1812    {
1813       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1814       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1815       string FinalFile = _config->FindDir("Dir::State::lists");
1816       FinalFile.append(URItoFileName(RealURI));
1817       if (FileExists(FinalFile))
1818          unlink(FinalFile.c_str());
1819
1820       new pkgAcqMetaSig(Owner,
1821                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1822                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1823                         IndexTargets, MetaIndexParser);
1824       if (Cnf->LocalOnly == true ||
1825           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1826          Dequeue();
1827    }
1828    else
1829       pkgAcqMetaIndex::Failed(Message, Cnf);
1830 }
1831                                                                         /*}}}*/
1832 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1833 // ---------------------------------------------------------------------
1834 /* This just sets up the initial fetch environment and queues the first
1835    possibilitiy */
1836 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1837                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1838                              string &StoreFilename) :
1839                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1840                StoreFilename(StoreFilename), Vf(Version.FileList()),
1841                Trusted(false)
1842 {
1843    Retries = _config->FindI("Acquire::Retries",0);
1844
1845    if (Version.Arch() == 0)
1846    {
1847       _error->Error(_("I wasn't able to locate a file for the %s package. "
1848                       "This might mean you need to manually fix this package. "
1849                       "(due to missing arch)"),
1850                     Version.ParentPkg().Name());
1851       return;
1852    }
1853
1854    /* We need to find a filename to determine the extension. We make the
1855       assumption here that all the available sources for this version share
1856       the same extension.. */
1857    // Skip not source sources, they do not have file fields.
1858    for (; Vf.end() == false; ++Vf)
1859    {
1860       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1861          continue;
1862       break;
1863    }
1864
1865    // Does not really matter here.. we are going to fail out below
1866    if (Vf.end() != true)
1867    {
1868       // If this fails to get a file name we will bomb out below.
1869       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1870       if (_error->PendingError() == true)
1871          return;
1872
1873       // Generate the final file name as: package_version_arch.foo
1874       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1875                       QuoteString(Version.VerStr(),"_:") + '_' +
1876                       QuoteString(Version.Arch(),"_:.") +
1877                       "." + flExtension(Parse.FileName());
1878    }
1879
1880    // check if we have one trusted source for the package. if so, switch
1881    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1882    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1883    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1884    bool seenUntrusted = false;
1885    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1886    {
1887       pkgIndexFile *Index;
1888       if (Sources->FindIndex(i.File(),Index) == false)
1889          continue;
1890
1891       if (debugAuth == true)
1892          std::cerr << "Checking index: " << Index->Describe()
1893                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1894
1895       if (Index->IsTrusted() == true)
1896       {
1897          Trusted = true;
1898          if (allowUnauth == false)
1899             break;
1900       }
1901       else
1902          seenUntrusted = true;
1903    }
1904
1905    // "allow-unauthenticated" restores apts old fetching behaviour
1906    // that means that e.g. unauthenticated file:// uris are higher
1907    // priority than authenticated http:// uris
1908    if (allowUnauth == true && seenUntrusted == true)
1909       Trusted = false;
1910
1911    // Select a source
1912    if (QueueNext() == false && _error->PendingError() == false)
1913       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1914                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1915 }
1916                                                                         /*}}}*/
1917 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1918 // ---------------------------------------------------------------------
1919 /* This queues the next available file version for download. It checks if
1920    the archive is already available in the cache and stashs the MD5 for
1921    checking later. */
1922 bool pkgAcqArchive::QueueNext()
1923 {
1924    string const ForceHash = _config->Find("Acquire::ForceHash");
1925    for (; Vf.end() == false; ++Vf)
1926    {
1927       // Ignore not source sources
1928       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1929          continue;
1930
1931       // Try to cross match against the source list
1932       pkgIndexFile *Index;
1933       if (Sources->FindIndex(Vf.File(),Index) == false)
1934             continue;
1935
1936       // only try to get a trusted package from another source if that source
1937       // is also trusted
1938       if(Trusted && !Index->IsTrusted())
1939          continue;
1940
1941       // Grab the text package record
1942       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1943       if (_error->PendingError() == true)
1944          return false;
1945
1946       string PkgFile = Parse.FileName();
1947       if (ForceHash.empty() == false)
1948       {
1949          if(stringcasecmp(ForceHash, "sha512") == 0)
1950             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1951          else if(stringcasecmp(ForceHash, "sha256") == 0)
1952             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1953          else if (stringcasecmp(ForceHash, "sha1") == 0)
1954             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1955          else
1956             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1957       }
1958       else
1959       {
1960          string Hash;
1961          if ((Hash = Parse.SHA512Hash()).empty() == false)
1962             ExpectedHash = HashString("SHA512", Hash);
1963          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1964             ExpectedHash = HashString("SHA256", Hash);
1965          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1966             ExpectedHash = HashString("SHA1", Hash);
1967          else
1968             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1969       }
1970       if (PkgFile.empty() == true)
1971          return _error->Error(_("The package index files are corrupted. No Filename: "
1972                               "field for package %s."),
1973                               Version.ParentPkg().Name());
1974
1975       Desc.URI = Index->ArchiveURI(PkgFile);
1976       Desc.Description = Index->ArchiveInfo(Version);
1977       Desc.Owner = this;
1978       Desc.ShortDesc = Version.ParentPkg().Name();
1979
1980       // See if we already have the file. (Legacy filenames)
1981       FileSize = Version->Size;
1982       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1983       struct stat Buf;
1984       if (stat(FinalFile.c_str(),&Buf) == 0)
1985       {
1986          // Make sure the size matches
1987          if ((unsigned long long)Buf.st_size == Version->Size)
1988          {
1989             Complete = true;
1990             Local = true;
1991             Status = StatDone;
1992             StoreFilename = DestFile = FinalFile;
1993             return true;
1994          }
1995
1996          /* Hmm, we have a file and its size does not match, this means it is
1997             an old style mismatched arch */
1998          unlink(FinalFile.c_str());
1999       }
2000
2001       // Check it again using the new style output filenames
2002       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2003       if (stat(FinalFile.c_str(),&Buf) == 0)
2004       {
2005          // Make sure the size matches
2006          if ((unsigned long long)Buf.st_size == Version->Size)
2007          {
2008             Complete = true;
2009             Local = true;
2010             Status = StatDone;
2011             StoreFilename = DestFile = FinalFile;
2012             return true;
2013          }
2014
2015          /* Hmm, we have a file and its size does not match, this shouldnt
2016             happen.. */
2017          unlink(FinalFile.c_str());
2018       }
2019
2020       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2021
2022       // Check the destination file
2023       if (stat(DestFile.c_str(),&Buf) == 0)
2024       {
2025          // Hmm, the partial file is too big, erase it
2026          if ((unsigned long long)Buf.st_size > Version->Size)
2027             unlink(DestFile.c_str());
2028          else
2029             PartialSize = Buf.st_size;
2030       }
2031
2032       // Disables download of archives - useful if no real installation follows,
2033       // e.g. if we are just interested in proposed installation order
2034       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2035       {
2036          Complete = true;
2037          Local = true;
2038          Status = StatDone;
2039          StoreFilename = DestFile = FinalFile;
2040          return true;
2041       }
2042
2043       // Create the item
2044       Local = false;
2045       Desc.URI = Index->ArchiveURI(PkgFile);
2046       Desc.Description = Index->ArchiveInfo(Version);
2047       Desc.Owner = this;
2048       Desc.ShortDesc = Version.ParentPkg().Name();
2049       QueueURI(Desc);
2050
2051       ++Vf;
2052       return true;
2053    }
2054    return false;
2055 }
2056                                                                         /*}}}*/
2057 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2058 // ---------------------------------------------------------------------
2059 /* */
2060 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2061                          pkgAcquire::MethodConfig *Cfg)
2062 {
2063    Item::Done(Message,Size,CalcHash,Cfg);
2064
2065    // Check the size
2066    if (Size != Version->Size)
2067    {
2068       RenameOnError(SizeMismatch);
2069       return;
2070    }
2071
2072    // Check the hash
2073    if(ExpectedHash.toStr() != CalcHash)
2074    {
2075       RenameOnError(HashSumMismatch);
2076       return;
2077    }
2078
2079    // Grab the output filename
2080    string FileName = LookupTag(Message,"Filename");
2081    if (FileName.empty() == true)
2082    {
2083       Status = StatError;
2084       ErrorText = "Method gave a blank filename";
2085       return;
2086    }
2087
2088    Complete = true;
2089
2090    // Reference filename
2091    if (FileName != DestFile)
2092    {
2093       StoreFilename = DestFile = FileName;
2094       Local = true;
2095       return;
2096    }
2097
2098    // Done, move it into position
2099    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2100    FinalFile += flNotDir(StoreFilename);
2101    Rename(DestFile,FinalFile);
2102
2103    StoreFilename = DestFile = FinalFile;
2104    Complete = true;
2105 }
2106                                                                         /*}}}*/
2107 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2108 // ---------------------------------------------------------------------
2109 /* Here we try other sources */
2110 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2111 {
2112    ErrorText = LookupTag(Message,"Message");
2113
2114    /* We don't really want to retry on failed media swaps, this prevents
2115       that. An interesting observation is that permanent failures are not
2116       recorded. */
2117    if (Cnf->Removable == true &&
2118        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2119    {
2120       // Vf = Version.FileList();
2121       while (Vf.end() == false) ++Vf;
2122       StoreFilename = string();
2123       Item::Failed(Message,Cnf);
2124       return;
2125    }
2126
2127    if (QueueNext() == false)
2128    {
2129       // This is the retry counter
2130       if (Retries != 0 &&
2131           Cnf->LocalOnly == false &&
2132           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2133       {
2134          Retries--;
2135          Vf = Version.FileList();
2136          if (QueueNext() == true)
2137             return;
2138       }
2139
2140       StoreFilename = string();
2141       Item::Failed(Message,Cnf);
2142    }
2143 }
2144                                                                         /*}}}*/
2145 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2146 // ---------------------------------------------------------------------
2147 bool pkgAcqArchive::IsTrusted()
2148 {
2149    return Trusted;
2150 }
2151                                                                         /*}}}*/
2152 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2153 // ---------------------------------------------------------------------
2154 /* */
2155 void pkgAcqArchive::Finished()
2156 {
2157    if (Status == pkgAcquire::Item::StatDone &&
2158        Complete == true)
2159       return;
2160    StoreFilename = string();
2161 }
2162                                                                         /*}}}*/
2163 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2164 // ---------------------------------------------------------------------
2165 /* The file is added to the queue */
2166 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2167                        unsigned long long Size,string Dsc,string ShortDesc,
2168                        const string &DestDir, const string &DestFilename,
2169                        bool IsIndexFile) :
2170                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2171 {
2172    Retries = _config->FindI("Acquire::Retries",0);
2173
2174    if(!DestFilename.empty())
2175       DestFile = DestFilename;
2176    else if(!DestDir.empty())
2177       DestFile = DestDir + "/" + flNotDir(URI);
2178    else
2179       DestFile = flNotDir(URI);
2180
2181    // Create the item
2182    Desc.URI = URI;
2183    Desc.Description = Dsc;
2184    Desc.Owner = this;
2185
2186    // Set the short description to the archive component
2187    Desc.ShortDesc = ShortDesc;
2188
2189    // Get the transfer sizes
2190    FileSize = Size;
2191    struct stat Buf;
2192    if (stat(DestFile.c_str(),&Buf) == 0)
2193    {
2194       // Hmm, the partial file is too big, erase it
2195       if ((unsigned long long)Buf.st_size > Size)
2196          unlink(DestFile.c_str());
2197       else
2198          PartialSize = Buf.st_size;
2199    }
2200
2201    QueueURI(Desc);
2202 }
2203                                                                         /*}}}*/
2204 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2205 // ---------------------------------------------------------------------
2206 /* */
2207 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2208                       pkgAcquire::MethodConfig *Cnf)
2209 {
2210    Item::Done(Message,Size,CalcHash,Cnf);
2211
2212    // Check the hash
2213    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2214    {
2215       RenameOnError(HashSumMismatch);
2216       return;
2217    }
2218
2219    string FileName = LookupTag(Message,"Filename");
2220    if (FileName.empty() == true)
2221    {
2222       Status = StatError;
2223       ErrorText = "Method gave a blank filename";
2224       return;
2225    }
2226
2227    Complete = true;
2228
2229    // The files timestamp matches
2230    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2231       return;
2232
2233    // We have to copy it into place
2234    if (FileName != DestFile)
2235    {
2236       Local = true;
2237       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2238           Cnf->Removable == true)
2239       {
2240          Desc.URI = "copy:" + FileName;
2241          QueueURI(Desc);
2242          return;
2243       }
2244
2245       // Erase the file if it is a symlink so we can overwrite it
2246       struct stat St;
2247       if (lstat(DestFile.c_str(),&St) == 0)
2248       {
2249          if (S_ISLNK(St.st_mode) != 0)
2250             unlink(DestFile.c_str());
2251       }
2252
2253       // Symlink the file
2254       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2255       {
2256          ErrorText = "Link to " + DestFile + " failure ";
2257          Status = StatError;
2258          Complete = false;
2259       }
2260    }
2261 }
2262                                                                         /*}}}*/
2263 // AcqFile::Failed - Failure handler                                    /*{{{*/
2264 // ---------------------------------------------------------------------
2265 /* Here we try other sources */
2266 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2267 {
2268    ErrorText = LookupTag(Message,"Message");
2269
2270    // This is the retry counter
2271    if (Retries != 0 &&
2272        Cnf->LocalOnly == false &&
2273        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2274    {
2275       Retries--;
2276       QueueURI(Desc);
2277       return;
2278    }
2279
2280    Item::Failed(Message,Cnf);
2281 }
2282                                                                         /*}}}*/
2283 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2284 // ---------------------------------------------------------------------
2285 /* The only header we use is the last-modified header. */
2286 string pkgAcqFile::Custom600Headers()
2287 {
2288    if (IsIndexFile)
2289       return "\nIndex-File: true";
2290    return "";
2291 }
2292                                                                         /*}}}*/