apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
  54 {
  55    if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
  56       return;
  57    std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
  58    for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
  59       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  60    std::cerr << " Actual Hash: " << std::endl;
  61    for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
  62       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  63 }
  64                                                                         /*}}}*/
  65
  66 // Acquire::Item::Item - Constructor                                    /*{{{*/
  67 pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) :
  68    Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
  69    Local(false), QueueCounter(0), ExpectedAdditionalItems(0),
  70    ExpectedHashes(ExpectedHashes)
  71 {
  72    Owner->Add(this);
  73    Status = StatIdle;
  74 }
  75                                                                         /*}}}*/
  76 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* */
  79 pkgAcquire::Item::~Item()
  80 {
  81    Owner->Remove(this);
  82 }
  83                                                                         /*}}}*/
  84 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* We return to an idle state if there are still other queues that could
  87    fetch this object */
  88 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  89 {
  90    Status = StatIdle;
  91    ErrorText = LookupTag(Message,"Message");
  92    UsedMirror =  LookupTag(Message,"UsedMirror");
  93    if (QueueCounter <= 1)
  94    {
  95       /* This indicates that the file is not available right now but might
  96          be sometime later. If we do a retry cycle then this should be
  97          retried [CDROMs] */
  98       if (Cnf->LocalOnly == true &&
  99           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 100       {
 101          Status = StatIdle;
 102          Dequeue();
 103          return;
 104       }
 105
 106       Status = StatError;
 107       Dequeue();
 108    }
 109
 110    // report mirror failure back to LP if we actually use a mirror
 111    string FailReason = LookupTag(Message, "FailReason");
 112    if(FailReason.size() != 0)
 113       ReportMirrorFailure(FailReason);
 114    else
 115       ReportMirrorFailure(ErrorText);
 116 }
 117                                                                         /*}}}*/
 118 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* Stash status and the file size. Note that setting Complete means
 121    sub-phases of the acquire process such as decompresion are operating */
 122 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 123 {
 124    Status = StatFetching;
 125    if (FileSize == 0 && Complete == false)
 126       FileSize = Size;
 127 }
 128                                                                         /*}}}*/
 129 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 130 // ---------------------------------------------------------------------
 131 /* */
 132 void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/,
 133                             pkgAcquire::MethodConfig * /*Cnf*/)
 134 {
 135    // We just downloaded something..
 136    string FileName = LookupTag(Message,"Filename");
 137    UsedMirror =  LookupTag(Message,"UsedMirror");
 138    if (Complete == false && !Local && FileName == DestFile)
 139    {
 140       if (Owner->Log != 0)
 141          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 142    }
 143
 144    if (FileSize == 0)
 145       FileSize= Size;
 146    Status = StatDone;
 147    ErrorText = string();
 148    Owner->Dequeue(this);
 149 }
 150                                                                         /*}}}*/
 151 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 152 // ---------------------------------------------------------------------
 153 /* This helper function is used by a lot of item methods as their final
 154    step */
 155 void pkgAcquire::Item::Rename(string From,string To)
 156 {
 157    if (rename(From.c_str(),To.c_str()) != 0)
 158    {
 159       char S[300];
 160       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 161               From.c_str(),To.c_str());
 162       Status = StatError;
 163       ErrorText = S;
 164    }
 165 }
 166                                                                         /*}}}*/
 167 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 168 {
 169    if(FileExists(DestFile))
 170       Rename(DestFile, DestFile + ".FAILED");
 171
 172    switch (error)
 173    {
 174       case HashSumMismatch:
 175          ErrorText = _("Hash Sum mismatch");
 176          Status = StatAuthError;
 177          ReportMirrorFailure("HashChecksumFailure");
 178          break;
 179       case SizeMismatch:
 180          ErrorText = _("Size mismatch");
 181          Status = StatAuthError;
 182          ReportMirrorFailure("SizeFailure");
 183          break;
 184       case InvalidFormat:
 185          ErrorText = _("Invalid file format");
 186          Status = StatError;
 187          // do not report as usually its not the mirrors fault, but Portal/Proxy
 188          break;
 189    }
 190    return false;
 191 }
 192                                                                         /*}}}*/
 193 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 194 // ---------------------------------------------------------------------
 195 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 196 {
 197    // we only act if a mirror was used at all
 198    if(UsedMirror.empty())
 199       return;
 200 #if 0
 201    std::cerr << "\nReportMirrorFailure: "
 202              << UsedMirror
 203              << " Uri: " << DescURI()
 204              << " FailCode: "
 205              << FailCode << std::endl;
 206 #endif
 207    const char *Args[40];
 208    unsigned int i = 0;
 209    string report = _config->Find("Methods::Mirror::ProblemReporting",
 210                                  "/usr/lib/apt/apt-report-mirror-failure");
 211    if(!FileExists(report))
 212       return;
 213    Args[i++] = report.c_str();
 214    Args[i++] = UsedMirror.c_str();
 215    Args[i++] = DescURI().c_str();
 216    Args[i++] = FailCode.c_str();
 217    Args[i++] = NULL;
 218    pid_t pid = ExecFork();
 219    if(pid < 0)
 220    {
 221       _error->Error("ReportMirrorFailure Fork failed");
 222       return;
 223    }
 224    else if(pid == 0)
 225    {
 226       execvp(Args[0], (char**)Args);
 227       std::cerr << "Could not exec " << Args[0] << std::endl;
 228       _exit(100);
 229    }
 230    if(!ExecWait(pid, "report-mirror-failure"))
 231    {
 232       _error->Warning("Couldn't report problem to '%s'",
 233                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 234    }
 235 }
 236                                                                         /*}}}*/
 237 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 238 // ---------------------------------------------------------------------
 239 /* Get a sub-index file based on checksums from a 'master' file and
 240    possibly query additional files */
 241 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 242                                  string const &URIDesc, string const &ShortDesc,
 243                                  HashStringList const &ExpectedHashes)
 244    : Item(Owner, ExpectedHashes)
 245 {
 246    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 247    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 248
 249    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 250    DestFile += URItoFileName(URI);
 251
 252    Desc.URI = URI;
 253    Desc.Description = URIDesc;
 254    Desc.Owner = this;
 255    Desc.ShortDesc = ShortDesc;
 256
 257    QueueURI(Desc);
 258
 259    if(Debug)
 260       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 261 }
 262                                                                         /*}}}*/
 263 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 264 // ---------------------------------------------------------------------
 265 /* The only header we use is the last-modified header. */
 266 string pkgAcqSubIndex::Custom600Headers() const
 267 {
 268    string Final = _config->FindDir("Dir::State::lists");
 269    Final += URItoFileName(Desc.URI);
 270
 271    struct stat Buf;
 272    if (stat(Final.c_str(),&Buf) != 0)
 273       return "\nIndex-File: true\nFail-Ignore: true\n";
 274    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 278 {
 279    if(Debug)
 280       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 281
 282    Complete = false;
 283    Status = StatDone;
 284    Dequeue();
 285
 286    // No good Index is provided
 287 }
 288                                                                         /*}}}*/
 289 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,  /*{{{*/
 290                            pkgAcquire::MethodConfig *Cnf)
 291 {
 292    if(Debug)
 293       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 294
 295    string FileName = LookupTag(Message,"Filename");
 296    if (FileName.empty() == true)
 297    {
 298       Status = StatError;
 299       ErrorText = "Method gave a blank filename";
 300       return;
 301    }
 302
 303    if (FileName != DestFile)
 304    {
 305       Local = true;
 306       Desc.URI = "copy:" + FileName;
 307       QueueURI(Desc);
 308       return;
 309    }
 310
 311    Item::Done(Message, Size, Hashes, Cnf);
 312
 313    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 314
 315    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 316    indexRecords SubIndexParser;
 317    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 318       Status = StatError;
 319       ErrorText = SubIndexParser.ErrorText;
 320       return;
 321    }
 322
 323    // success in downloading the index
 324    // rename the index
 325    if(Debug)
 326       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 327    Rename(DestFile,FinalFile);
 328    chmod(FinalFile.c_str(),0644);
 329    DestFile = FinalFile;
 330
 331    if(ParseIndex(DestFile) == false)
 332       return Failed("", NULL);
 333
 334    Complete = true;
 335    Status = StatDone;
 336    Dequeue();
 337    return;
 338 }
 339                                                                         /*}}}*/
 340 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 341 {
 342    indexRecords SubIndexParser;
 343    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 344       return false;
 345    // so something with the downloaded index
 346    return true;
 347 }
 348                                                                         /*}}}*/
 349 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 350 // ---------------------------------------------------------------------
 351 /* Get the DiffIndex file first and see if there are patches available
 352  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 353  * patches. If anything goes wrong in that process, it will fall back to
 354  * the original packages file
 355  */
 356 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 357                                  IndexTarget const * const Target,
 358                                  HashStringList const &ExpectedHashes,
 359                                  indexRecords *MetaIndexParser)
 360    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser)
 361 {
 362
 363    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 364
 365    RealURI = Target->URI;
 366    Desc.Owner = this;
 367    Desc.Description = Target->Description + "/DiffIndex";
 368    Desc.ShortDesc = Target->ShortDesc;
 369    Desc.URI = Target->URI + ".diff/Index";
 370
 371    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 372    DestFile += URItoFileName(Target->URI) + string(".DiffIndex");
 373
 374    if(Debug)
 375       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 376
 377    // look for the current package file
 378    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 379    CurrentPackagesFile += URItoFileName(RealURI);
 380
 381    // FIXME: this file:/ check is a hack to prevent fetching
 382    //        from local sources. this is really silly, and
 383    //        should be fixed cleanly as soon as possible
 384    if(!FileExists(CurrentPackagesFile) ||
 385       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 386    {
 387       // we don't have a pkg file or we don't want to queue
 388       if(Debug)
 389          std::clog << "No index file, local or canceld by user" << std::endl;
 390       Failed("", NULL);
 391       return;
 392    }
 393
 394    if(Debug)
 395       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 396          << CurrentPackagesFile << std::endl;
 397
 398    QueueURI(Desc);
 399
 400 }
 401                                                                         /*}}}*/
 402 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 403 // ---------------------------------------------------------------------
 404 /* The only header we use is the last-modified header. */
 405 string pkgAcqDiffIndex::Custom600Headers() const
 406 {
 407    string Final = _config->FindDir("Dir::State::lists");
 408    Final += URItoFileName(RealURI) + string(".IndexDiff");
 409
 410    if(Debug)
 411       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 412
 413    struct stat Buf;
 414    if (stat(Final.c_str(),&Buf) != 0)
 415       return "\nIndex-File: true";
 416
 417    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 418 }
 419                                                                         /*}}}*/
 420 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 421 {
 422    if(Debug)
 423       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 424          << std::endl;
 425
 426    pkgTagSection Tags;
 427    string ServerSha1;
 428    vector<DiffInfo> available_patches;
 429
 430    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 431    pkgTagFile TF(&Fd);
 432    if (_error->PendingError() == true)
 433       return false;
 434
 435    if(TF.Step(Tags) == true)
 436    {
 437       bool found = false;
 438       DiffInfo d;
 439       string size;
 440
 441       string const tmp = Tags.FindS("SHA1-Current");
 442       std::stringstream ss(tmp);
 443       ss >> ServerSha1 >> size;
 444       unsigned long const ServerSize = atol(size.c_str());
 445
 446       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 447       SHA1Summation SHA1;
 448       SHA1.AddFD(fd);
 449       string const local_sha1 = SHA1.Result();
 450
 451       if(local_sha1 == ServerSha1)
 452       {
 453          // we have the same sha1 as the server so we are done here
 454          if(Debug)
 455             std::clog << "Package file is up-to-date" << std::endl;
 456          // list cleanup needs to know that this file as well as the already
 457          // present index is ours, so we create an empty diff to save it for us
 458          new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 459                               ServerSha1, available_patches);
 460          return true;
 461       }
 462       else
 463       {
 464          if(Debug)
 465             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 466
 467          // check the historie and see what patches we need
 468          string const history = Tags.FindS("SHA1-History");
 469          std::stringstream hist(history);
 470          while(hist >> d.sha1 >> size >> d.file)
 471          {
 472             // read until the first match is found
 473             // from that point on, we probably need all diffs
 474             if(d.sha1 == local_sha1)
 475                found=true;
 476             else if (found == false)
 477                continue;
 478
 479             if(Debug)
 480                std::clog << "Need to get diff: " << d.file << std::endl;
 481             available_patches.push_back(d);
 482          }
 483
 484          if (available_patches.empty() == false)
 485          {
 486             // patching with too many files is rather slow compared to a fast download
 487             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 488             if (fileLimit != 0 && fileLimit < available_patches.size())
 489             {
 490                if (Debug)
 491                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 492                         << ") so fallback to complete download" << std::endl;
 493                return false;
 494             }
 495
 496             // see if the patches are too big
 497             found = false; // it was true and it will be true again at the end
 498             d = *available_patches.begin();
 499             string const firstPatch = d.file;
 500             unsigned long patchesSize = 0;
 501             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 502             while(patches >> d.sha1 >> size >> d.file)
 503             {
 504                if (firstPatch == d.file)
 505                   found = true;
 506                else if (found == false)
 507                   continue;
 508
 509                patchesSize += atol(size.c_str());
 510             }
 511             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 512             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 513             {
 514                if (Debug)
 515                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 516                         << ") so fallback to complete download" << std::endl;
 517                return false;
 518             }
 519          }
 520       }
 521
 522       // we have something, queue the next diff
 523       if(found)
 524       {
 525          // queue the diffs
 526          string::size_type const last_space = Description.rfind(" ");
 527          if(last_space != string::npos)
 528             Description.erase(last_space, Description.size()-last_space);
 529
 530          /* decide if we should download patches one by one or in one go:
 531             The first is good if the server merges patches, but many don't so client
 532             based merging can be attempt in which case the second is better.
 533             "bad things" will happen if patches are merged on the server,
 534             but client side merging is attempt as well */
 535          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 536          if (pdiff_merge == true)
 537          {
 538             // reprepro adds this flag if it has merged patches on the server
 539             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 540             pdiff_merge = (precedence != "merged");
 541          }
 542
 543          if (pdiff_merge == false)
 544          {
 545             new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 546                                  ServerSha1, available_patches);
 547          }
 548          else
 549          {
 550             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 551             for(size_t i = 0; i < available_patches.size(); ++i)
 552                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target,
 553                                                        ExpectedHashes,
 554                                                        MetaIndexParser,
 555                                                        available_patches[i],
 556                                                        diffs);
 557          }
 558
 559          Complete = false;
 560          Status = StatDone;
 561          Dequeue();
 562          return true;
 563       }
 564    }
 565
 566    // Nothing found, report and return false
 567    // Failing here is ok, if we return false later, the full
 568    // IndexFile is queued
 569    if(Debug)
 570       std::clog << "Can't find a patch in the index file" << std::endl;
 571    return false;
 572 }
 573                                                                         /*}}}*/
 574 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 575 {
 576    if(Debug)
 577       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 578                 << "Falling back to normal index file acquire" << std::endl;
 579
 580    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 581
 582    Complete = false;
 583    Status = StatDone;
 584    Dequeue();
 585 }
 586                                                                         /*}}}*/
 587 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
 588                            pkgAcquire::MethodConfig *Cnf)
 589 {
 590    if(Debug)
 591       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 592
 593    Item::Done(Message, Size, Hashes, Cnf);
 594
 595    string FinalFile;
 596    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 597
 598    // success in downloading the index
 599    // rename the index
 600    FinalFile += string(".IndexDiff");
 601    if(Debug)
 602       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 603                 << std::endl;
 604    Rename(DestFile,FinalFile);
 605    chmod(FinalFile.c_str(),0644);
 606    DestFile = FinalFile;
 607
 608    if(!ParseDiffIndex(DestFile))
 609       return Failed("", NULL);
 610
 611    Complete = true;
 612    Status = StatDone;
 613    Dequeue();
 614    return;
 615 }
 616                                                                         /*}}}*/
 617 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 618 // ---------------------------------------------------------------------
 619 /* The package diff is added to the queue. one object is constructed
 620  * for each diff and the index
 621  */
 622 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 623                                    struct IndexTarget const * const Target,
 624                                    HashStringList const &ExpectedHashes,
 625                                    indexRecords *MetaIndexParser,
 626                                    string ServerSha1,
 627                                    vector<DiffInfo> diffs)
 628    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 629      available_patches(diffs), ServerSha1(ServerSha1)
 630 {
 631
 632    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 633    DestFile += URItoFileName(Target->URI);
 634
 635    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 636
 637    RealURI = Target->URI;
 638    Desc.Owner = this;
 639    Description = Target->Description;
 640    Desc.ShortDesc = Target->ShortDesc;
 641
 642    if(available_patches.empty() == true)
 643    {
 644       // we are done (yeah!)
 645       Finish(true);
 646    }
 647    else
 648    {
 649       // get the next diff
 650       State = StateFetchDiff;
 651       QueueNextDiff();
 652    }
 653 }
 654                                                                         /*}}}*/
 655 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 656 {
 657    if(Debug)
 658       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 659                 << "Falling back to normal index file acquire" << std::endl;
 660    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 661    Finish();
 662 }
 663                                                                         /*}}}*/
 664 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 665 void pkgAcqIndexDiffs::Finish(bool allDone)
 666 {
 667    // we restore the original name, this is required, otherwise
 668    // the file will be cleaned
 669    if(allDone)
 670    {
 671       DestFile = _config->FindDir("Dir::State::lists");
 672       DestFile += URItoFileName(RealURI);
 673
 674       if(HashSums().usable() && !HashSums().VerifyFile(DestFile))
 675       {
 676          RenameOnError(HashSumMismatch);
 677          Dequeue();
 678          return;
 679       }
 680
 681       // this is for the "real" finish
 682       Complete = true;
 683       Status = StatDone;
 684       Dequeue();
 685       if(Debug)
 686          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 687       return;
 688    }
 689
 690    if(Debug)
 691       std::clog << "Finishing: " << Desc.URI << std::endl;
 692    Complete = false;
 693    Status = StatDone;
 694    Dequeue();
 695    return;
 696 }
 697                                                                         /*}}}*/
 698 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 699 {
 700
 701    // calc sha1 of the just patched file
 702    string FinalFile = _config->FindDir("Dir::State::lists");
 703    FinalFile += URItoFileName(RealURI);
 704
 705    FileFd fd(FinalFile, FileFd::ReadOnly);
 706    SHA1Summation SHA1;
 707    SHA1.AddFD(fd);
 708    string local_sha1 = string(SHA1.Result());
 709    if(Debug)
 710       std::clog << "QueueNextDiff: "
 711                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 712
 713    // final file reached before all patches are applied
 714    if(local_sha1 == ServerSha1)
 715    {
 716       Finish(true);
 717       return true;
 718    }
 719
 720    // remove all patches until the next matching patch is found
 721    // this requires the Index file to be ordered
 722    for(vector<DiffInfo>::iterator I=available_patches.begin();
 723        available_patches.empty() == false &&
 724           I != available_patches.end() &&
 725           I->sha1 != local_sha1;
 726        ++I)
 727    {
 728       available_patches.erase(I);
 729    }
 730
 731    // error checking and falling back if no patch was found
 732    if(available_patches.empty() == true)
 733    {
 734       Failed("", NULL);
 735       return false;
 736    }
 737
 738    // queue the right diff
 739    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 740    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 741    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 742    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 743
 744    if(Debug)
 745       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 746
 747    QueueURI(Desc);
 748
 749    return true;
 750 }
 751                                                                         /*}}}*/
 752 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes,       /*{{{*/
 753                             pkgAcquire::MethodConfig *Cnf)
 754 {
 755    if(Debug)
 756       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 757
 758    Item::Done(Message, Size, Hashes, Cnf);
 759
 760    string FinalFile;
 761    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 762
 763    // success in downloading a diff, enter ApplyDiff state
 764    if(State == StateFetchDiff)
 765    {
 766
 767       // rred excepts the patch as $FinalFile.ed
 768       Rename(DestFile,FinalFile+".ed");
 769
 770       if(Debug)
 771          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 772
 773       State = StateApplyDiff;
 774       Local = true;
 775       Desc.URI = "rred:" + FinalFile;
 776       QueueURI(Desc);
 777       Mode = "rred";
 778       return;
 779    }
 780
 781
 782    // success in download/apply a diff, queue next (if needed)
 783    if(State == StateApplyDiff)
 784    {
 785       // remove the just applied patch
 786       available_patches.erase(available_patches.begin());
 787       unlink((FinalFile + ".ed").c_str());
 788
 789       // move into place
 790       if(Debug)
 791       {
 792          std::clog << "Moving patched file in place: " << std::endl
 793                    << DestFile << " -> " << FinalFile << std::endl;
 794       }
 795       Rename(DestFile,FinalFile);
 796       chmod(FinalFile.c_str(),0644);
 797
 798       // see if there is more to download
 799       if(available_patches.empty() == false) {
 800          new pkgAcqIndexDiffs(Owner, Target,
 801                               ExpectedHashes, MetaIndexParser,
 802                               ServerSha1, available_patches);
 803          return Finish();
 804       } else
 805          return Finish(true);
 806    }
 807 }
 808                                                                         /*}}}*/
 809 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 810 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 811                                              struct IndexTarget const * const Target,
 812                                              HashStringList const &ExpectedHashes,
 813                                              indexRecords *MetaIndexParser,
 814                                              DiffInfo const &patch,
 815                                              std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 816    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 817      patch(patch), allPatches(allPatches), State(StateFetchDiff)
 818 {
 819
 820    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 821    DestFile += URItoFileName(Target->URI);
 822
 823    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 824
 825    RealURI = Target->URI;
 826    Desc.Owner = this;
 827    Description = Target->Description;
 828    Desc.ShortDesc = Target->ShortDesc;
 829
 830    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 831    Desc.Description = Description + " " + patch.file + string(".pdiff");
 832    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 833    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 834
 835    if(Debug)
 836       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 837
 838    QueueURI(Desc);
 839 }
 840                                                                         /*}}}*/
 841 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 842 {
 843    if(Debug)
 844       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 845    Complete = false;
 846    Status = StatDone;
 847    Dequeue();
 848
 849    // check if we are the first to fail, otherwise we are done here
 850    State = StateDoneDiff;
 851    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 852          I != allPatches->end(); ++I)
 853       if ((*I)->State == StateErrorDiff)
 854          return;
 855
 856    // first failure means we should fallback
 857    State = StateErrorDiff;
 858    std::clog << "Falling back to normal index file acquire" << std::endl;
 859    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 860 }
 861                                                                         /*}}}*/
 862 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes,   /*{{{*/
 863                             pkgAcquire::MethodConfig *Cnf)
 864 {
 865    if(Debug)
 866       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 867
 868    Item::Done(Message,Size,Hashes,Cnf);
 869
 870    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 871
 872    if (State == StateFetchDiff)
 873    {
 874       // rred expects the patch as $FinalFile.ed.$patchname.gz
 875       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 876
 877       // check if this is the last completed diff
 878       State = StateDoneDiff;
 879       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 880             I != allPatches->end(); ++I)
 881          if ((*I)->State != StateDoneDiff)
 882          {
 883             if(Debug)
 884                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 885             return;
 886          }
 887
 888       // this is the last completed diff, so we are ready to apply now
 889       State = StateApplyDiff;
 890
 891       if(Debug)
 892          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 893
 894       Local = true;
 895       Desc.URI = "rred:" + FinalFile;
 896       QueueURI(Desc);
 897       Mode = "rred";
 898       return;
 899    }
 900    // success in download/apply all diffs, clean up
 901    else if (State == StateApplyDiff)
 902    {
 903       // see if we really got the expected file
 904       if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile))
 905       {
 906          RenameOnError(HashSumMismatch);
 907          return;
 908       }
 909
 910       // move the result into place
 911       if(Debug)
 912          std::clog << "Moving patched file in place: " << std::endl
 913                    << DestFile << " -> " << FinalFile << std::endl;
 914       Rename(DestFile, FinalFile);
 915       chmod(FinalFile.c_str(), 0644);
 916
 917       // otherwise lists cleanup will eat the file
 918       DestFile = FinalFile;
 919
 920       // ensure the ed's are gone regardless of list-cleanup
 921       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 922             I != allPatches->end(); ++I)
 923       {
 924             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 925             unlink(patch.c_str());
 926       }
 927
 928       // all set and done
 929       Complete = true;
 930       if(Debug)
 931          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 932    }
 933 }
 934                                                                         /*}}}*/
 935 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* The package file is added to the queue and a second class is
 938    instantiated to fetch the revision file */
 939 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 940                          string URI,string URIDesc,string ShortDesc,
 941                          HashStringList const  &ExpectedHash, string comprExt)
 942    : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI)
 943 {
 944    if(comprExt.empty() == true)
 945    {
 946       // autoselect the compression method
 947       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 948       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 949          comprExt.append(*t).append(" ");
 950       if (comprExt.empty() == false)
 951          comprExt.erase(comprExt.end()-1);
 952    }
 953    CompressionExtension = comprExt;
 954
 955    Verify = true;
 956
 957    Init(URI, URIDesc, ShortDesc);
 958 }
 959 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 960                          HashStringList const &ExpectedHash,
 961                          indexRecords *MetaIndexParser)
 962    : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser),
 963      RealURI(Target->URI)
 964 {
 965    // autoselect the compression method
 966    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 967    CompressionExtension = "";
 968    if (ExpectedHashes.usable())
 969    {
 970       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 971          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 972             CompressionExtension.append(*t).append(" ");
 973    }
 974    else
 975    {
 976       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 977          CompressionExtension.append(*t).append(" ");
 978    }
 979    if (CompressionExtension.empty() == false)
 980       CompressionExtension.erase(CompressionExtension.end()-1);
 981
 982    // only verify non-optional targets, see acquire-item.h for a FIXME
 983    // to make this more flexible
 984    if (Target->IsOptional())
 985      Verify = false;
 986    else
 987      Verify = true;
 988
 989    Init(Target->URI, Target->Description, Target->ShortDesc);
 990 }
 991                                                                         /*}}}*/
 992 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 993 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 994    Decompression = false;
 995    Erase = false;
 996
 997    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 998    DestFile += URItoFileName(URI);
 999
1000    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1001    std::string MetaKey;
1002    if (comprExt == "uncompressed")
1003    {
1004       Desc.URI = URI;
1005       if(Target)
1006          MetaKey = string(Target->MetaKey);
1007    }
1008    else
1009    {
1010       Desc.URI = URI + '.' + comprExt;
1011       if(Target)
1012          MetaKey = string(Target->MetaKey) + '.' + comprExt;
1013    }
1014
1015    // load the filesize
1016    if(MetaIndexParser)
1017    {
1018       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1019       if(Record)
1020          FileSize = Record->Size;
1021    }
1022
1023    Desc.Description = URIDesc;
1024    Desc.Owner = this;
1025    Desc.ShortDesc = ShortDesc;
1026
1027    QueueURI(Desc);
1028 }
1029                                                                         /*}}}*/
1030 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1031 // ---------------------------------------------------------------------
1032 /* The only header we use is the last-modified header. */
1033 string pkgAcqIndex::Custom600Headers() const
1034 {
1035    string Final = _config->FindDir("Dir::State::lists");
1036    Final += URItoFileName(RealURI);
1037    if (_config->FindB("Acquire::GzipIndexes",false))
1038       Final += ".gz";
1039
1040    string msg = "\nIndex-File: true";
1041    // FIXME: this really should use "IndexTarget::IsOptional()" but that
1042    //        seems to be difficult without breaking ABI
1043    if (ShortDesc().find("Translation") != 0)
1044       msg += "\nFail-Ignore: true";
1045    struct stat Buf;
1046    if (stat(Final.c_str(),&Buf) == 0)
1047       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1048
1049    return msg;
1050 }
1051                                                                         /*}}}*/
1052 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1053 {
1054    size_t const nextExt = CompressionExtension.find(' ');
1055    if (nextExt != std::string::npos)
1056    {
1057       CompressionExtension = CompressionExtension.substr(nextExt+1);
1058       Init(RealURI, Desc.Description, Desc.ShortDesc);
1059       return;
1060    }
1061
1062    // on decompression failure, remove bad versions in partial/
1063    if (Decompression && Erase) {
1064       string s = _config->FindDir("Dir::State::lists") + "partial/";
1065       s.append(URItoFileName(RealURI));
1066       unlink(s.c_str());
1067    }
1068
1069    Item::Failed(Message,Cnf);
1070 }
1071                                                                         /*}}}*/
1072 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1073 // ---------------------------------------------------------------------
1074 /* This goes through a number of states.. On the initial fetch the
1075    method could possibly return an alternate filename which points
1076    to the uncompressed version of the file. If this is so the file
1077    is copied into the partial directory. In all other cases the file
1078    is decompressed with a gzip uri. */
1079 void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,
1080                        pkgAcquire::MethodConfig *Cfg)
1081 {
1082    Item::Done(Message,Size,Hashes,Cfg);
1083
1084    if (Decompression == true)
1085    {
1086       if (ExpectedHashes.usable() && ExpectedHashes != Hashes)
1087       {
1088          RenameOnError(HashSumMismatch);
1089          printHashSumComparision(RealURI, ExpectedHashes, Hashes);
1090          return;
1091       }
1092
1093       /* Verify the index file for correctness (all indexes must
1094        * have a Package field) (LP: #346386) (Closes: #627642) */
1095       if (Verify == true)
1096       {
1097          FileFd fd(DestFile, FileFd::ReadOnly);
1098          // Only test for correctness if the file is not empty (empty is ok)
1099          if (fd.FileSize() > 0)
1100          {
1101             pkgTagSection sec;
1102             pkgTagFile tag(&fd);
1103
1104             // all our current indexes have a field 'Package' in each section
1105             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1106             {
1107                RenameOnError(InvalidFormat);
1108                return;
1109             }
1110          }
1111       }
1112
1113       // Done, move it into position
1114       string FinalFile = _config->FindDir("Dir::State::lists");
1115       FinalFile += URItoFileName(RealURI);
1116       Rename(DestFile,FinalFile);
1117       chmod(FinalFile.c_str(),0644);
1118
1119       /* We restore the original name to DestFile so that the clean operation
1120          will work OK */
1121       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1122       DestFile += URItoFileName(RealURI);
1123
1124       // Remove the compressed version.
1125       if (Erase == true)
1126          unlink(DestFile.c_str());
1127
1128       return;
1129    }
1130
1131    Erase = false;
1132    Complete = true;
1133
1134    // Handle the unzipd case
1135    string FileName = LookupTag(Message,"Alt-Filename");
1136    if (FileName.empty() == false)
1137    {
1138       // The files timestamp matches
1139       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1140          return;
1141       Decompression = true;
1142       Local = true;
1143       DestFile += ".decomp";
1144       Desc.URI = "copy:" + FileName;
1145       QueueURI(Desc);
1146       Mode = "copy";
1147       return;
1148    }
1149
1150    FileName = LookupTag(Message,"Filename");
1151    if (FileName.empty() == true)
1152    {
1153       Status = StatError;
1154       ErrorText = "Method gave a blank filename";
1155    }
1156
1157    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1158
1159    // The files timestamp matches
1160    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1161        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1162           // Update DestFile for .gz suffix so that the clean operation keeps it
1163           DestFile += ".gz";
1164       return;
1165     }
1166
1167    if (FileName == DestFile)
1168       Erase = true;
1169    else
1170       Local = true;
1171
1172    string decompProg;
1173
1174    // If we enable compressed indexes and already have gzip, keep it
1175    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1176       string FinalFile = _config->FindDir("Dir::State::lists");
1177       FinalFile += URItoFileName(RealURI) + ".gz";
1178       Rename(DestFile,FinalFile);
1179       chmod(FinalFile.c_str(),0644);
1180
1181       // Update DestFile for .gz suffix so that the clean operation keeps it
1182       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1183       DestFile += URItoFileName(RealURI) + ".gz";
1184       return;
1185     }
1186
1187    // get the binary name for your used compression type
1188    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1189    if(decompProg.empty() == false);
1190    else if(compExt == "uncompressed")
1191       decompProg = "copy";
1192    else {
1193       _error->Error("Unsupported extension: %s", compExt.c_str());
1194       return;
1195    }
1196
1197    Decompression = true;
1198    DestFile += ".decomp";
1199    Desc.URI = decompProg + ":" + FileName;
1200    QueueURI(Desc);
1201
1202    // FIXME: this points to a c++ string that goes out of scope
1203    Mode = decompProg.c_str();
1204 }
1205                                                                         /*}}}*/
1206 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1207 // ---------------------------------------------------------------------
1208 /* The Translation file is added to the queue */
1209 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1210                             string URI,string URIDesc,string ShortDesc)
1211   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "")
1212 {
1213 }
1214 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target,
1215                          HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser)
1216   : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser)
1217 {
1218    // load the filesize
1219    indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
1220    if(Record)
1221       FileSize = Record->Size;
1222 }
1223                                                                         /*}}}*/
1224 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1225 // ---------------------------------------------------------------------
1226 string pkgAcqIndexTrans::Custom600Headers() const
1227 {
1228    string Final = _config->FindDir("Dir::State::lists");
1229    Final += URItoFileName(RealURI);
1230
1231    struct stat Buf;
1232    if (stat(Final.c_str(),&Buf) != 0)
1233       return "\nFail-Ignore: true\nIndex-File: true";
1234    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1235 }
1236                                                                         /*}}}*/
1237 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1238 // ---------------------------------------------------------------------
1239 /* */
1240 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1241 {
1242    size_t const nextExt = CompressionExtension.find(' ');
1243    if (nextExt != std::string::npos)
1244    {
1245       CompressionExtension = CompressionExtension.substr(nextExt+1);
1246       Init(RealURI, Desc.Description, Desc.ShortDesc);
1247       Status = StatIdle;
1248       return;
1249    }
1250
1251    if (Cnf->LocalOnly == true ||
1252        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1253    {
1254       // Ignore this
1255       Status = StatDone;
1256       Complete = false;
1257       Dequeue();
1258       return;
1259    }
1260
1261    Item::Failed(Message,Cnf);
1262 }
1263                                                                         /*}}}*/
1264 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1265                              string URI,string URIDesc,string ShortDesc,
1266                              string MetaIndexURI, string MetaIndexURIDesc,
1267                              string MetaIndexShortDesc,
1268                              const vector<IndexTarget*>* IndexTargets,
1269                              indexRecords* MetaIndexParser) :
1270    Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI),
1271    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1272    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1273 {
1274    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1275    DestFile += URItoFileName(URI);
1276
1277    // remove any partial downloaded sig-file in partial/.
1278    // it may confuse proxies and is too small to warrant a
1279    // partial download anyway
1280    unlink(DestFile.c_str());
1281
1282    // Create the item
1283    Desc.Description = URIDesc;
1284    Desc.Owner = this;
1285    Desc.ShortDesc = ShortDesc;
1286    Desc.URI = URI;
1287
1288    string Final = _config->FindDir("Dir::State::lists");
1289    Final += URItoFileName(RealURI);
1290    if (RealFileExists(Final) == true)
1291    {
1292       // File was already in place.  It needs to be re-downloaded/verified
1293       // because Release might have changed, we do give it a different
1294       // name than DestFile because otherwise the http method will
1295       // send If-Range requests and there are too many broken servers
1296       // out there that do not understand them
1297       LastGoodSig = DestFile+".reverify";
1298       Rename(Final,LastGoodSig);
1299    }
1300
1301    // we expect the indextargets + one additional Release file
1302    ExpectedAdditionalItems = IndexTargets->size() + 1;
1303
1304    QueueURI(Desc);
1305 }
1306                                                                         /*}}}*/
1307 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1308 {
1309    // if the file was never queued undo file-changes done in the constructor
1310    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1311          LastGoodSig.empty() == false)
1312    {
1313       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1314       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1315          Rename(LastGoodSig, Final);
1316    }
1317
1318 }
1319                                                                         /*}}}*/
1320 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1321 // ---------------------------------------------------------------------
1322 /* The only header we use is the last-modified header. */
1323 string pkgAcqMetaSig::Custom600Headers() const
1324 {
1325    struct stat Buf;
1326    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1327       return "\nIndex-File: true";
1328
1329    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1330 }
1331
1332 void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes,
1333                          pkgAcquire::MethodConfig *Cfg)
1334 {
1335    Item::Done(Message, Size, Hashes, Cfg);
1336
1337    string FileName = LookupTag(Message,"Filename");
1338    if (FileName.empty() == true)
1339    {
1340       Status = StatError;
1341       ErrorText = "Method gave a blank filename";
1342       return;
1343    }
1344
1345    if (FileName != DestFile)
1346    {
1347       // We have to copy it into place
1348       Local = true;
1349       Desc.URI = "copy:" + FileName;
1350       QueueURI(Desc);
1351       return;
1352    }
1353
1354    Complete = true;
1355
1356    // at this point pkgAcqMetaIndex takes over
1357    ExpectedAdditionalItems = 0;
1358
1359    // put the last known good file back on i-m-s hit (it will
1360    // be re-verified again)
1361    // Else do nothing, we have the new file in DestFile then
1362    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1363       Rename(LastGoodSig, DestFile);
1364
1365    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1366    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1367                        MetaIndexShortDesc,  DestFile, IndexTargets,
1368                        MetaIndexParser);
1369
1370 }
1371                                                                         /*}}}*/
1372 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1373 {
1374    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1375
1376    // at this point pkgAcqMetaIndex takes over
1377    ExpectedAdditionalItems = 0;
1378
1379    // if we get a network error we fail gracefully
1380    if(Status == StatTransientNetworkError)
1381    {
1382       Item::Failed(Message,Cnf);
1383       // move the sigfile back on transient network failures
1384       if(FileExists(LastGoodSig))
1385          Rename(LastGoodSig,Final);
1386
1387       // set the status back to , Item::Failed likes to reset it
1388       Status = pkgAcquire::Item::StatTransientNetworkError;
1389       return;
1390    }
1391
1392    // Delete any existing sigfile when the acquire failed
1393    unlink(Final.c_str());
1394
1395    // queue a pkgAcqMetaIndex with no sigfile
1396    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1397                        "", IndexTargets, MetaIndexParser);
1398
1399    if (Cnf->LocalOnly == true ||
1400        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1401    {
1402       // Ignore this
1403       Status = StatDone;
1404       Complete = false;
1405       Dequeue();
1406       return;
1407    }
1408
1409    Item::Failed(Message,Cnf);
1410 }
1411                                                                         /*}}}*/
1412 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1413                                  string URI,string URIDesc,string ShortDesc,
1414                                  string SigFile,
1415                                  const vector<IndexTarget*>* IndexTargets,
1416                                  indexRecords* MetaIndexParser) :
1417    Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1418    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1419 {
1420    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1421    DestFile += URItoFileName(URI);
1422
1423    // Create the item
1424    Desc.Description = URIDesc;
1425    Desc.Owner = this;
1426    Desc.ShortDesc = ShortDesc;
1427    Desc.URI = URI;
1428
1429    // we expect more item
1430    ExpectedAdditionalItems = IndexTargets->size();
1431
1432    QueueURI(Desc);
1433 }
1434                                                                         /*}}}*/
1435 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1436 // ---------------------------------------------------------------------
1437 /* The only header we use is the last-modified header. */
1438 string pkgAcqMetaIndex::Custom600Headers() const
1439 {
1440    string Final = _config->FindDir("Dir::State::lists");
1441    Final += URItoFileName(RealURI);
1442
1443    struct stat Buf;
1444    if (stat(Final.c_str(),&Buf) != 0)
1445       return "\nIndex-File: true";
1446
1447    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1448 }
1449                                                                         /*}}}*/
1450 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
1451                            pkgAcquire::MethodConfig *Cfg)
1452 {
1453    Item::Done(Message,Size,Hashes,Cfg);
1454
1455    // MetaIndexes are done in two passes: one to download the
1456    // metaindex with an appropriate method, and a second to verify it
1457    // with the gpgv method
1458
1459    if (AuthPass == true)
1460    {
1461       AuthDone(Message);
1462
1463       // all cool, move Release file into place
1464       Complete = true;
1465    }
1466    else
1467    {
1468       RetrievalDone(Message);
1469       if (!Complete)
1470          // Still more retrieving to do
1471          return;
1472
1473       if (SigFile == "")
1474       {
1475          // There was no signature file, so we are finished.  Download
1476          // the indexes and do only hashsum verification if possible
1477          MetaIndexParser->Load(DestFile);
1478          QueueIndexes(false);
1479       }
1480       else
1481       {
1482          // FIXME: move this into pkgAcqMetaClearSig::Done on the next
1483          //        ABI break
1484
1485          // if we expect a ClearTextSignature (InRelase), ensure that
1486          // this is what we get and if not fail to queue a
1487          // Release/Release.gpg, see #346386
1488          if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
1489          {
1490             Failed(Message, Cfg);
1491             return;
1492          }
1493
1494          // There was a signature file, so pass it to gpgv for
1495          // verification
1496          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1497             std::cerr << "Metaindex acquired, queueing gpg verification ("
1498                       << SigFile << "," << DestFile << ")\n";
1499          AuthPass = true;
1500          Desc.URI = "gpgv:" + SigFile;
1501          QueueURI(Desc);
1502          Mode = "gpgv";
1503          return;
1504       }
1505    }
1506
1507    if (Complete == true)
1508    {
1509       string FinalFile = _config->FindDir("Dir::State::lists");
1510       FinalFile += URItoFileName(RealURI);
1511       if (SigFile == DestFile)
1512          SigFile = FinalFile;
1513       Rename(DestFile,FinalFile);
1514       chmod(FinalFile.c_str(),0644);
1515       DestFile = FinalFile;
1516    }
1517 }
1518                                                                         /*}}}*/
1519 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1520 {
1521    // We have just finished downloading a Release file (it is not
1522    // verified yet)
1523
1524    string FileName = LookupTag(Message,"Filename");
1525    if (FileName.empty() == true)
1526    {
1527       Status = StatError;
1528       ErrorText = "Method gave a blank filename";
1529       return;
1530    }
1531
1532    if (FileName != DestFile)
1533    {
1534       Local = true;
1535       Desc.URI = "copy:" + FileName;
1536       QueueURI(Desc);
1537       return;
1538    }
1539
1540    // make sure to verify against the right file on I-M-S hit
1541    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1542    if(IMSHit)
1543    {
1544       string FinalFile = _config->FindDir("Dir::State::lists");
1545       FinalFile += URItoFileName(RealURI);
1546       if (SigFile == DestFile)
1547       {
1548          SigFile = FinalFile;
1549          // constructor of pkgAcqMetaClearSig moved it out of the way,
1550          // now move it back in on IMS hit for the 'old' file
1551          string const OldClearSig = DestFile + ".reverify";
1552          if (RealFileExists(OldClearSig) == true)
1553             Rename(OldClearSig, FinalFile);
1554       }
1555       DestFile = FinalFile;
1556    }
1557    Complete = true;
1558 }
1559                                                                         /*}}}*/
1560 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1561 {
1562    // At this point, the gpgv method has succeeded, so there is a
1563    // valid signature from a key in the trusted keyring.  We
1564    // perform additional verification of its contents, and use them
1565    // to verify the indexes we are about to download
1566
1567    if (!MetaIndexParser->Load(DestFile))
1568    {
1569       Status = StatAuthError;
1570       ErrorText = MetaIndexParser->ErrorText;
1571       return;
1572    }
1573
1574    if (!VerifyVendor(Message))
1575    {
1576       return;
1577    }
1578
1579    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1580       std::cerr << "Signature verification succeeded: "
1581                 << DestFile << std::endl;
1582
1583    // Download further indexes with verification
1584    QueueIndexes(true);
1585
1586    // is it a clearsigned MetaIndex file?
1587    if (DestFile == SigFile)
1588       return;
1589
1590    // Done, move signature file into position
1591    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1592       URItoFileName(RealURI) + ".gpg";
1593    Rename(SigFile,VerifiedSigFile);
1594    chmod(VerifiedSigFile.c_str(),0644);
1595 }
1596                                                                         /*}}}*/
1597 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1598 {
1599 #if 0
1600    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1601     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1602    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1603    {
1604       Status = StatError;
1605       ErrorText = MetaIndexParser->ErrorText;
1606       return;
1607    }
1608 #endif
1609    bool transInRelease = false;
1610    {
1611       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1612       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1613          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1614          if (k->find("Translation-") != std::string::npos)
1615          {
1616             transInRelease = true;
1617             break;
1618          }
1619    }
1620
1621    // at this point the real Items are loaded in the fetcher
1622    ExpectedAdditionalItems = 0;
1623    for (vector <IndexTarget*>::const_iterator Target = IndexTargets->begin();
1624         Target != IndexTargets->end();
1625         ++Target)
1626    {
1627       HashStringList ExpectedIndexHashes;
1628       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1629       bool compressedAvailable = false;
1630       if (Record == NULL)
1631       {
1632          if ((*Target)->IsOptional() == true)
1633          {
1634             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1635             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1636                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1637                {
1638                   compressedAvailable = true;
1639                   break;
1640                }
1641          }
1642          else if (verify == true)
1643          {
1644             Status = StatAuthError;
1645             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1646             return;
1647          }
1648       }
1649       else
1650       {
1651          ExpectedIndexHashes = Record->Hashes;
1652          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1653          {
1654             std::cerr << "Queueing: " << (*Target)->URI << std::endl
1655                << "Expected Hash:" << std::endl;
1656             for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
1657                std::cerr <<  "\t- " << hs->toStr() << std::endl;
1658             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1659          }
1660          if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false)
1661          {
1662             Status = StatAuthError;
1663             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1664             return;
1665          }
1666       }
1667
1668       if ((*Target)->IsOptional() == true)
1669       {
1670          if ((*Target)->IsSubIndex() == true)
1671             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1672                                 (*Target)->ShortDesc, ExpectedIndexHashes);
1673          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1674          {
1675             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1676                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1677                new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1678             else
1679                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1680          }
1681          continue;
1682       }
1683
1684       /* Queue Packages file (either diff or full packages files, depending
1685          on the users option) - we also check if the PDiff Index file is listed
1686          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1687          instead, but passing the required info to it is to much hassle */
1688       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1689           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1690          new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1691       else
1692          new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1693    }
1694 }
1695                                                                         /*}}}*/
1696 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1697 {
1698    string::size_type pos;
1699
1700    // check for missing sigs (that where not fatal because otherwise we had
1701    // bombed earlier)
1702    string missingkeys;
1703    string msg = _("There is no public key available for the "
1704                   "following key IDs:\n");
1705    pos = Message.find("NO_PUBKEY ");
1706    if (pos != std::string::npos)
1707    {
1708       string::size_type start = pos+strlen("NO_PUBKEY ");
1709       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1710       missingkeys += (Fingerprint);
1711    }
1712    if(!missingkeys.empty())
1713       _error->Warning("%s", (msg + missingkeys).c_str());
1714
1715    string Transformed = MetaIndexParser->GetExpectedDist();
1716
1717    if (Transformed == "../project/experimental")
1718    {
1719       Transformed = "experimental";
1720    }
1721
1722    pos = Transformed.rfind('/');
1723    if (pos != string::npos)
1724    {
1725       Transformed = Transformed.substr(0, pos);
1726    }
1727
1728    if (Transformed == ".")
1729    {
1730       Transformed = "";
1731    }
1732
1733    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1734        MetaIndexParser->GetValidUntil() > 0) {
1735       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1736       if (invalid_since > 0)
1737          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1738          // the time since then the file is invalid - formated in the same way as in
1739          // the download progress display (e.g. 7d 3h 42min 1s)
1740          return _error->Error(
1741             _("Release file for %s is expired (invalid since %s). "
1742               "Updates for this repository will not be applied."),
1743             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1744    }
1745
1746    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1747    {
1748       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1749       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1750       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1751    }
1752
1753    if (MetaIndexParser->CheckDist(Transformed) == false)
1754    {
1755       // This might become fatal one day
1756 //       Status = StatAuthError;
1757 //       ErrorText = "Conflicting distribution; expected "
1758 //          + MetaIndexParser->GetExpectedDist() + " but got "
1759 //          + MetaIndexParser->GetDist();
1760 //       return false;
1761       if (!Transformed.empty())
1762       {
1763          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1764                          Desc.Description.c_str(),
1765                          Transformed.c_str(),
1766                          MetaIndexParser->GetDist().c_str());
1767       }
1768    }
1769
1770    return true;
1771 }
1772                                                                         /*}}}*/
1773 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1774 // ---------------------------------------------------------------------
1775 /* */
1776 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1777 {
1778    if (AuthPass == true)
1779    {
1780       // gpgv method failed, if we have a good signature
1781       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1782       if (DestFile != SigFile)
1783          LastGoodSigFile.append(".gpg");
1784       LastGoodSigFile.append(".reverify");
1785
1786       if(FileExists(LastGoodSigFile))
1787       {
1788          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1789          if (DestFile != SigFile)
1790             VerifiedSigFile.append(".gpg");
1791          Rename(LastGoodSigFile, VerifiedSigFile);
1792          Status = StatTransientNetworkError;
1793          _error->Warning(_("An error occurred during the signature "
1794                            "verification. The repository is not updated "
1795                            "and the previous index files will be used. "
1796                            "GPG error: %s: %s\n"),
1797                          Desc.Description.c_str(),
1798                          LookupTag(Message,"Message").c_str());
1799          RunScripts("APT::Update::Auth-Failure");
1800          return;
1801       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1802          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1803          _error->Error(_("GPG error: %s: %s"),
1804                          Desc.Description.c_str(),
1805                          LookupTag(Message,"Message").c_str());
1806          return;
1807       } else {
1808          _error->Warning(_("GPG error: %s: %s"),
1809                          Desc.Description.c_str(),
1810                          LookupTag(Message,"Message").c_str());
1811       }
1812       // gpgv method failed
1813       ReportMirrorFailure("GPGFailure");
1814    }
1815
1816    /* Always move the meta index, even if gpgv failed. This ensures
1817     * that PackageFile objects are correctly filled in */
1818    if (FileExists(DestFile)) {
1819       string FinalFile = _config->FindDir("Dir::State::lists");
1820       FinalFile += URItoFileName(RealURI);
1821       /* InRelease files become Release files, otherwise
1822        * they would be considered as trusted later on */
1823       if (SigFile == DestFile) {
1824          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1825                                        "Release");
1826          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1827                                        "Release");
1828          SigFile = FinalFile;
1829       }
1830       Rename(DestFile,FinalFile);
1831       chmod(FinalFile.c_str(),0644);
1832
1833       DestFile = FinalFile;
1834    }
1835
1836    // No Release file was present, or verification failed, so fall
1837    // back to queueing Packages files without verification
1838    QueueIndexes(false);
1839 }
1840                                                                         /*}}}*/
1841 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1842                 string const &URI, string const &URIDesc, string const &ShortDesc,
1843                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1844                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1845                 const vector<IndexTarget*>* IndexTargets,
1846                 indexRecords* MetaIndexParser) :
1847         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1848         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1849         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1850 {
1851    SigFile = DestFile;
1852
1853    // index targets + (worst case:) Release/Release.gpg
1854    ExpectedAdditionalItems = IndexTargets->size() + 2;
1855
1856
1857    // keep the old InRelease around in case of transistent network errors
1858    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1859    if (RealFileExists(Final) == true)
1860    {
1861       string const LastGoodSig = DestFile + ".reverify";
1862       Rename(Final,LastGoodSig);
1863    }
1864 }
1865                                                                         /*}}}*/
1866 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1867 {
1868    // if the file was never queued undo file-changes done in the constructor
1869    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1870    {
1871       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1872       string const LastGoodSig = DestFile + ".reverify";
1873       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1874          Rename(LastGoodSig, Final);
1875    }
1876 }
1877                                                                         /*}}}*/
1878 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1879 // ---------------------------------------------------------------------
1880 // FIXME: this can go away once the InRelease file is used widely
1881 string pkgAcqMetaClearSig::Custom600Headers() const
1882 {
1883    string Final = _config->FindDir("Dir::State::lists");
1884    Final += URItoFileName(RealURI);
1885
1886    struct stat Buf;
1887    if (stat(Final.c_str(),&Buf) != 0)
1888    {
1889       Final = DestFile + ".reverify";
1890       if (stat(Final.c_str(),&Buf) != 0)
1891          return "\nIndex-File: true\nFail-Ignore: true\n";
1892    }
1893
1894    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1895 }
1896                                                                         /*}}}*/
1897 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1898 {
1899    // we failed, we will not get additional items from this method
1900    ExpectedAdditionalItems = 0;
1901
1902    if (AuthPass == false)
1903    {
1904       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1905       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1906       string FinalFile = _config->FindDir("Dir::State::lists");
1907       FinalFile.append(URItoFileName(RealURI));
1908       if (FileExists(FinalFile))
1909          unlink(FinalFile.c_str());
1910
1911       new pkgAcqMetaSig(Owner,
1912                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1913                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1914                         IndexTargets, MetaIndexParser);
1915       if (Cnf->LocalOnly == true ||
1916           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1917          Dequeue();
1918    }
1919    else
1920       pkgAcqMetaIndex::Failed(Message, Cnf);
1921 }
1922                                                                         /*}}}*/
1923 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1924 // ---------------------------------------------------------------------
1925 /* This just sets up the initial fetch environment and queues the first
1926    possibilitiy */
1927 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1928                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1929                              string &StoreFilename) :
1930                Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs),
1931                StoreFilename(StoreFilename), Vf(Version.FileList()),
1932                Trusted(false)
1933 {
1934    Retries = _config->FindI("Acquire::Retries",0);
1935
1936    if (Version.Arch() == 0)
1937    {
1938       _error->Error(_("I wasn't able to locate a file for the %s package. "
1939                       "This might mean you need to manually fix this package. "
1940                       "(due to missing arch)"),
1941                     Version.ParentPkg().FullName().c_str());
1942       return;
1943    }
1944
1945    /* We need to find a filename to determine the extension. We make the
1946       assumption here that all the available sources for this version share
1947       the same extension.. */
1948    // Skip not source sources, they do not have file fields.
1949    for (; Vf.end() == false; ++Vf)
1950    {
1951       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1952          continue;
1953       break;
1954    }
1955
1956    // Does not really matter here.. we are going to fail out below
1957    if (Vf.end() != true)
1958    {
1959       // If this fails to get a file name we will bomb out below.
1960       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1961       if (_error->PendingError() == true)
1962          return;
1963
1964       // Generate the final file name as: package_version_arch.foo
1965       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1966                       QuoteString(Version.VerStr(),"_:") + '_' +
1967                       QuoteString(Version.Arch(),"_:.") +
1968                       "." + flExtension(Parse.FileName());
1969    }
1970
1971    // check if we have one trusted source for the package. if so, switch
1972    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1973    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1974    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1975    bool seenUntrusted = false;
1976    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1977    {
1978       pkgIndexFile *Index;
1979       if (Sources->FindIndex(i.File(),Index) == false)
1980          continue;
1981
1982       if (debugAuth == true)
1983          std::cerr << "Checking index: " << Index->Describe()
1984                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1985
1986       if (Index->IsTrusted() == true)
1987       {
1988          Trusted = true;
1989          if (allowUnauth == false)
1990             break;
1991       }
1992       else
1993          seenUntrusted = true;
1994    }
1995
1996    // "allow-unauthenticated" restores apts old fetching behaviour
1997    // that means that e.g. unauthenticated file:// uris are higher
1998    // priority than authenticated http:// uris
1999    if (allowUnauth == true && seenUntrusted == true)
2000       Trusted = false;
2001
2002    // Select a source
2003    if (QueueNext() == false && _error->PendingError() == false)
2004       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
2005                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
2006 }
2007                                                                         /*}}}*/
2008 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
2009 // ---------------------------------------------------------------------
2010 /* This queues the next available file version for download. It checks if
2011    the archive is already available in the cache and stashs the MD5 for
2012    checking later. */
2013 bool pkgAcqArchive::QueueNext()
2014 {
2015    for (; Vf.end() == false; ++Vf)
2016    {
2017       // Ignore not source sources
2018       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2019          continue;
2020
2021       // Try to cross match against the source list
2022       pkgIndexFile *Index;
2023       if (Sources->FindIndex(Vf.File(),Index) == false)
2024             continue;
2025
2026       // only try to get a trusted package from another source if that source
2027       // is also trusted
2028       if(Trusted && !Index->IsTrusted())
2029          continue;
2030
2031       // Grab the text package record
2032       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2033       if (_error->PendingError() == true)
2034          return false;
2035
2036       string PkgFile = Parse.FileName();
2037       ExpectedHashes = Parse.Hashes();
2038
2039       if (PkgFile.empty() == true)
2040          return _error->Error(_("The package index files are corrupted. No Filename: "
2041                               "field for package %s."),
2042                               Version.ParentPkg().Name());
2043
2044       Desc.URI = Index->ArchiveURI(PkgFile);
2045       Desc.Description = Index->ArchiveInfo(Version);
2046       Desc.Owner = this;
2047       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2048
2049       // See if we already have the file. (Legacy filenames)
2050       FileSize = Version->Size;
2051       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2052       struct stat Buf;
2053       if (stat(FinalFile.c_str(),&Buf) == 0)
2054       {
2055          // Make sure the size matches
2056          if ((unsigned long long)Buf.st_size == Version->Size)
2057          {
2058             Complete = true;
2059             Local = true;
2060             Status = StatDone;
2061             StoreFilename = DestFile = FinalFile;
2062             return true;
2063          }
2064
2065          /* Hmm, we have a file and its size does not match, this means it is
2066             an old style mismatched arch */
2067          unlink(FinalFile.c_str());
2068       }
2069
2070       // Check it again using the new style output filenames
2071       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2072       if (stat(FinalFile.c_str(),&Buf) == 0)
2073       {
2074          // Make sure the size matches
2075          if ((unsigned long long)Buf.st_size == Version->Size)
2076          {
2077             Complete = true;
2078             Local = true;
2079             Status = StatDone;
2080             StoreFilename = DestFile = FinalFile;
2081             return true;
2082          }
2083
2084          /* Hmm, we have a file and its size does not match, this shouldn't
2085             happen.. */
2086          unlink(FinalFile.c_str());
2087       }
2088
2089       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2090
2091       // Check the destination file
2092       if (stat(DestFile.c_str(),&Buf) == 0)
2093       {
2094          // Hmm, the partial file is too big, erase it
2095          if ((unsigned long long)Buf.st_size > Version->Size)
2096             unlink(DestFile.c_str());
2097          else
2098             PartialSize = Buf.st_size;
2099       }
2100
2101       // Disables download of archives - useful if no real installation follows,
2102       // e.g. if we are just interested in proposed installation order
2103       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2104       {
2105          Complete = true;
2106          Local = true;
2107          Status = StatDone;
2108          StoreFilename = DestFile = FinalFile;
2109          return true;
2110       }
2111
2112       // Create the item
2113       Local = false;
2114       QueueURI(Desc);
2115
2116       ++Vf;
2117       return true;
2118    }
2119    return false;
2120 }
2121                                                                         /*}}}*/
2122 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2123 // ---------------------------------------------------------------------
2124 /* */
2125 void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes,
2126                          pkgAcquire::MethodConfig *Cfg)
2127 {
2128    Item::Done(Message, Size, CalcHashes, Cfg);
2129
2130    // Check the size
2131    if (Size != Version->Size)
2132    {
2133       RenameOnError(SizeMismatch);
2134       return;
2135    }
2136
2137    // FIXME: could this empty() check impose *any* sort of security issue?
2138    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2139    {
2140       RenameOnError(HashSumMismatch);
2141       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2142       return;
2143    }
2144
2145    // Grab the output filename
2146    string FileName = LookupTag(Message,"Filename");
2147    if (FileName.empty() == true)
2148    {
2149       Status = StatError;
2150       ErrorText = "Method gave a blank filename";
2151       return;
2152    }
2153
2154    Complete = true;
2155
2156    // Reference filename
2157    if (FileName != DestFile)
2158    {
2159       StoreFilename = DestFile = FileName;
2160       Local = true;
2161       return;
2162    }
2163
2164    // Done, move it into position
2165    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2166    FinalFile += flNotDir(StoreFilename);
2167    Rename(DestFile,FinalFile);
2168
2169    StoreFilename = DestFile = FinalFile;
2170    Complete = true;
2171 }
2172                                                                         /*}}}*/
2173 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2174 // ---------------------------------------------------------------------
2175 /* Here we try other sources */
2176 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2177 {
2178    ErrorText = LookupTag(Message,"Message");
2179
2180    /* We don't really want to retry on failed media swaps, this prevents
2181       that. An interesting observation is that permanent failures are not
2182       recorded. */
2183    if (Cnf->Removable == true &&
2184        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2185    {
2186       // Vf = Version.FileList();
2187       while (Vf.end() == false) ++Vf;
2188       StoreFilename = string();
2189       Item::Failed(Message,Cnf);
2190       return;
2191    }
2192
2193    if (QueueNext() == false)
2194    {
2195       // This is the retry counter
2196       if (Retries != 0 &&
2197           Cnf->LocalOnly == false &&
2198           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2199       {
2200          Retries--;
2201          Vf = Version.FileList();
2202          if (QueueNext() == true)
2203             return;
2204       }
2205
2206       StoreFilename = string();
2207       Item::Failed(Message,Cnf);
2208    }
2209 }
2210                                                                         /*}}}*/
2211 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2212 // ---------------------------------------------------------------------
2213 APT_PURE bool pkgAcqArchive::IsTrusted() const
2214 {
2215    return Trusted;
2216 }
2217                                                                         /*}}}*/
2218 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2219 // ---------------------------------------------------------------------
2220 /* */
2221 void pkgAcqArchive::Finished()
2222 {
2223    if (Status == pkgAcquire::Item::StatDone &&
2224        Complete == true)
2225       return;
2226    StoreFilename = string();
2227 }
2228                                                                         /*}}}*/
2229 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2230 // ---------------------------------------------------------------------
2231 /* The file is added to the queue */
2232 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes,
2233                        unsigned long long Size,string Dsc,string ShortDesc,
2234                        const string &DestDir, const string &DestFilename,
2235                        bool IsIndexFile) :
2236                        Item(Owner, Hashes), IsIndexFile(IsIndexFile)
2237 {
2238    Retries = _config->FindI("Acquire::Retries",0);
2239
2240    if(!DestFilename.empty())
2241       DestFile = DestFilename;
2242    else if(!DestDir.empty())
2243       DestFile = DestDir + "/" + flNotDir(URI);
2244    else
2245       DestFile = flNotDir(URI);
2246
2247    // Create the item
2248    Desc.URI = URI;
2249    Desc.Description = Dsc;
2250    Desc.Owner = this;
2251
2252    // Set the short description to the archive component
2253    Desc.ShortDesc = ShortDesc;
2254
2255    // Get the transfer sizes
2256    FileSize = Size;
2257    struct stat Buf;
2258    if (stat(DestFile.c_str(),&Buf) == 0)
2259    {
2260       // Hmm, the partial file is too big, erase it
2261       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2262          unlink(DestFile.c_str());
2263       else
2264          PartialSize = Buf.st_size;
2265    }
2266
2267    QueueURI(Desc);
2268 }
2269                                                                         /*}}}*/
2270 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2271 // ---------------------------------------------------------------------
2272 /* */
2273 void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes,
2274                       pkgAcquire::MethodConfig *Cnf)
2275 {
2276    Item::Done(Message,Size,CalcHashes,Cnf);
2277
2278    // Check the hash
2279    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2280    {
2281       RenameOnError(HashSumMismatch);
2282       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2283       return;
2284    }
2285
2286    string FileName = LookupTag(Message,"Filename");
2287    if (FileName.empty() == true)
2288    {
2289       Status = StatError;
2290       ErrorText = "Method gave a blank filename";
2291       return;
2292    }
2293
2294    Complete = true;
2295
2296    // The files timestamp matches
2297    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2298       return;
2299
2300    // We have to copy it into place
2301    if (FileName != DestFile)
2302    {
2303       Local = true;
2304       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2305           Cnf->Removable == true)
2306       {
2307          Desc.URI = "copy:" + FileName;
2308          QueueURI(Desc);
2309          return;
2310       }
2311
2312       // Erase the file if it is a symlink so we can overwrite it
2313       struct stat St;
2314       if (lstat(DestFile.c_str(),&St) == 0)
2315       {
2316          if (S_ISLNK(St.st_mode) != 0)
2317             unlink(DestFile.c_str());
2318       }
2319
2320       // Symlink the file
2321       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2322       {
2323          ErrorText = "Link to " + DestFile + " failure ";
2324          Status = StatError;
2325          Complete = false;
2326       }
2327    }
2328 }
2329                                                                         /*}}}*/
2330 // AcqFile::Failed - Failure handler                                    /*{{{*/
2331 // ---------------------------------------------------------------------
2332 /* Here we try other sources */
2333 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2334 {
2335    ErrorText = LookupTag(Message,"Message");
2336
2337    // This is the retry counter
2338    if (Retries != 0 &&
2339        Cnf->LocalOnly == false &&
2340        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2341    {
2342       Retries--;
2343       QueueURI(Desc);
2344       return;
2345    }
2346
2347    Item::Failed(Message,Cnf);
2348 }
2349                                                                         /*}}}*/
2350 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2351 // ---------------------------------------------------------------------
2352 /* The only header we use is the last-modified header. */
2353 string pkgAcqFile::Custom600Headers() const
2354 {
2355    if (IsIndexFile)
2356       return "\nIndex-File: true";
2357    return "";
2358 }
2359                                                                         /*}}}*/