apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include <errno.h>
  32 #include <string>
  33 #include <sstream>
  34 #include <stdio.h>
  35 #include <ctime>
  36
  37 #include <apti18n.h>
  38                                                                         /*}}}*/
  39
  40 using namespace std;
  41
  42 // Acquire::Item::Item - Constructor                                    /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  46                        PartialSize(0), Mode(0), ID(0), Complete(false),
  47                        Local(false), QueueCounter(0)
  48 {
  49    Owner->Add(this);
  50    Status = StatIdle;
  51 }
  52                                                                         /*}}}*/
  53 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::~Item()
  57 {
  58    Owner->Remove(this);
  59 }
  60                                                                         /*}}}*/
  61 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  62 // ---------------------------------------------------------------------
  63 /* We return to an idle state if there are still other queues that could
  64    fetch this object */
  65 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  66 {
  67    Status = StatIdle;
  68    ErrorText = LookupTag(Message,"Message");
  69    UsedMirror =  LookupTag(Message,"UsedMirror");
  70    if (QueueCounter <= 1)
  71    {
  72       /* This indicates that the file is not available right now but might
  73          be sometime later. If we do a retry cycle then this should be
  74          retried [CDROMs] */
  75       if (Cnf->LocalOnly == true &&
  76           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  77       {
  78          Status = StatIdle;
  79          Dequeue();
  80          return;
  81       }
  82
  83       Status = StatError;
  84       Dequeue();
  85    }
  86
  87    // report mirror failure back to LP if we actually use a mirror
  88    string FailReason = LookupTag(Message, "FailReason");
  89    if(FailReason.size() != 0)
  90       ReportMirrorFailure(FailReason);
  91    else
  92       ReportMirrorFailure(ErrorText);
  93 }
  94                                                                         /*}}}*/
  95 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* Stash status and the file size. Note that setting Complete means
  98    sub-phases of the acquire process such as decompresion are operating */
  99 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 100 {
 101    Status = StatFetching;
 102    if (FileSize == 0 && Complete == false)
 103       FileSize = Size;
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* */
 109 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 110                             pkgAcquire::MethodConfig *Cnf)
 111 {
 112    // We just downloaded something..
 113    string FileName = LookupTag(Message,"Filename");
 114    UsedMirror =  LookupTag(Message,"UsedMirror");
 115    if (Complete == false && !Local && FileName == DestFile)
 116    {
 117       if (Owner->Log != 0)
 118          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 119    }
 120
 121    if (FileSize == 0)
 122       FileSize= Size;
 123    Status = StatDone;
 124    ErrorText = string();
 125    Owner->Dequeue(this);
 126 }
 127                                                                         /*}}}*/
 128 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 129 // ---------------------------------------------------------------------
 130 /* This helper function is used by alot of item methods as thier final
 131    step */
 132 void pkgAcquire::Item::Rename(string From,string To)
 133 {
 134    if (rename(From.c_str(),To.c_str()) != 0)
 135    {
 136       char S[300];
 137       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 138               From.c_str(),To.c_str());
 139       Status = StatError;
 140       ErrorText = S;
 141    }
 142 }
 143                                                                         /*}}}*/
 144 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 145 // ---------------------------------------------------------------------
 146 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 147 {
 148    // we only act if a mirror was used at all
 149    if(UsedMirror.empty())
 150       return;
 151 #if 0
 152    std::cerr << "\nReportMirrorFailure: "
 153              << UsedMirror
 154              << " Uri: " << DescURI()
 155              << " FailCode: "
 156              << FailCode << std::endl;
 157 #endif
 158    const char *Args[40];
 159    unsigned int i = 0;
 160    string report = _config->Find("Methods::Mirror::ProblemReporting",
 161                                  "/usr/lib/apt/apt-report-mirror-failure");
 162    if(!FileExists(report))
 163       return;
 164    Args[i++] = report.c_str();
 165    Args[i++] = UsedMirror.c_str();
 166    Args[i++] = DescURI().c_str();
 167    Args[i++] = FailCode.c_str();
 168    Args[i++] = NULL;
 169    pid_t pid = ExecFork();
 170    if(pid < 0)
 171    {
 172       _error->Error("ReportMirrorFailure Fork failed");
 173       return;
 174    }
 175    else if(pid == 0)
 176    {
 177       execvp(Args[0], (char**)Args);
 178       std::cerr << "Could not exec " << Args[0] << std::endl;
 179       _exit(100);
 180    }
 181    if(!ExecWait(pid, "report-mirror-failure"))
 182    {
 183       _error->Warning("Couldn't report problem to '%s'",
 184                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 185    }
 186 }
 187                                                                         /*}}}*/
 188 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 189 // ---------------------------------------------------------------------
 190 /* Get the Index file first and see if there are languages available
 191  * If so, create a pkgAcqIndexTrans for the found language(s).
 192  */
 193 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 194                                  string const &URIDesc, string const &ShortDesc,
 195                                  HashString const &ExpectedHash)
 196    : Item(Owner), ExpectedHash(ExpectedHash)
 197 {
 198    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 199
 200    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 201    DestFile += URItoFileName(URI);
 202
 203    Desc.URI = URI;
 204    Desc.Description = URIDesc;
 205    Desc.Owner = this;
 206    Desc.ShortDesc = ShortDesc;
 207
 208    QueueURI(Desc);
 209
 210    if(Debug)
 211       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 212 }
 213                                                                         /*}}}*/
 214 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 215 // ---------------------------------------------------------------------
 216 /* The only header we use is the last-modified header. */
 217 string pkgAcqSubIndex::Custom600Headers()
 218 {
 219    string Final = _config->FindDir("Dir::State::lists");
 220    Final += URItoFileName(Desc.URI);
 221
 222    struct stat Buf;
 223    if (stat(Final.c_str(),&Buf) != 0)
 224       return "\nIndex-File: true\nFail-Ignore: true\n";
 225    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 226 }
 227                                                                         /*}}}*/
 228 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 229 {
 230    if(Debug)
 231       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 232
 233    Complete = false;
 234    Status = StatDone;
 235    Dequeue();
 236
 237    // No good Index is provided, so try guessing
 238    std::vector<std::string> langs = APT::Configuration::getLanguages(true);
 239    for (std::vector<std::string>::const_iterator l = langs.begin();
 240         l != langs.end(); ++l)
 241    {
 242       if (*l == "none") continue;
 243       string const file = "Translation-" + *l;
 244       new pkgAcqIndexTrans(Owner, Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file),
 245                 Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file),
 246                 file);
 247    }
 248 }
 249                                                                         /*}}}*/
 250 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 251                            pkgAcquire::MethodConfig *Cnf)
 252 {
 253    if(Debug)
 254       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 255
 256    string FileName = LookupTag(Message,"Filename");
 257    if (FileName.empty() == true)
 258    {
 259       Status = StatError;
 260       ErrorText = "Method gave a blank filename";
 261       return;
 262    }
 263
 264    if (FileName != DestFile)
 265    {
 266       Local = true;
 267       Desc.URI = "copy:" + FileName;
 268       QueueURI(Desc);
 269       return;
 270    }
 271
 272    Item::Done(Message,Size,Md5Hash,Cnf);
 273
 274    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 275
 276    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 277    indexRecords SubIndexParser;
 278    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 279       Status = StatError;
 280       ErrorText = SubIndexParser.ErrorText;
 281       return;
 282    }
 283
 284    // sucess in downloading the index
 285    // rename the index
 286    if(Debug)
 287       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 288    Rename(DestFile,FinalFile);
 289    chmod(FinalFile.c_str(),0644);
 290    DestFile = FinalFile;
 291
 292    if(ParseIndex(DestFile) == false)
 293       return Failed("", NULL);
 294
 295    Complete = true;
 296    Status = StatDone;
 297    Dequeue();
 298    return;
 299 }
 300                                                                         /*}}}*/
 301 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 302 {
 303    indexRecords SubIndexParser;
 304    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 305       return false;
 306
 307    std::vector<std::string> lang = APT::Configuration::getLanguages(true);
 308    for (std::vector<std::string>::const_iterator l = lang.begin();
 309         l != lang.end(); ++l)
 310    {
 311       if (*l == "none")
 312          continue;
 313
 314       string file = "Translation-" + *l;
 315       indexRecords::checkSum const *Record = SubIndexParser.Lookup(file);
 316       HashString expected;
 317       if (Record == NULL)
 318       {
 319          // FIXME: the Index file provided by debian currently only includes bz2 records
 320          Record = SubIndexParser.Lookup(file + ".bz2");
 321          if (Record == NULL)
 322             continue;
 323       }
 324       else
 325       {
 326          expected = Record->Hash;
 327          if (expected.empty() == true)
 328             continue;
 329       }
 330
 331       IndexTarget target;
 332       target.Description = Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file);
 333       target.MetaKey = file;
 334       target.ShortDesc = file;
 335       target.URI = Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file);
 336       new pkgAcqIndexTrans(Owner, &target, expected, &SubIndexParser);
 337    }
 338    return true;
 339 }
 340                                                                         /*}}}*/
 341 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* Get the DiffIndex file first and see if there are patches availabe
 344  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 345  * patches. If anything goes wrong in that process, it will fall back to
 346  * the original packages file
 347  */
 348 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 349                                  string URI,string URIDesc,string ShortDesc,
 350                                  HashString ExpectedHash)
 351    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 352      Description(URIDesc)
 353 {
 354
 355    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 356
 357    Desc.Description = URIDesc + "/DiffIndex";
 358    Desc.Owner = this;
 359    Desc.ShortDesc = ShortDesc;
 360    Desc.URI = URI + ".diff/Index";
 361
 362    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 363    DestFile += URItoFileName(URI) + string(".DiffIndex");
 364
 365    if(Debug)
 366       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 367
 368    // look for the current package file
 369    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 370    CurrentPackagesFile += URItoFileName(RealURI);
 371
 372    // FIXME: this file:/ check is a hack to prevent fetching
 373    //        from local sources. this is really silly, and
 374    //        should be fixed cleanly as soon as possible
 375    if(!FileExists(CurrentPackagesFile) ||
 376       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 377    {
 378       // we don't have a pkg file or we don't want to queue
 379       if(Debug)
 380          std::clog << "No index file, local or canceld by user" << std::endl;
 381       Failed("", NULL);
 382       return;
 383    }
 384
 385    if(Debug)
 386       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 387                 << CurrentPackagesFile << std::endl;
 388
 389    QueueURI(Desc);
 390
 391 }
 392                                                                         /*}}}*/
 393 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 394 // ---------------------------------------------------------------------
 395 /* The only header we use is the last-modified header. */
 396 string pkgAcqDiffIndex::Custom600Headers()
 397 {
 398    string Final = _config->FindDir("Dir::State::lists");
 399    Final += URItoFileName(RealURI) + string(".IndexDiff");
 400
 401    if(Debug)
 402       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 403
 404    struct stat Buf;
 405    if (stat(Final.c_str(),&Buf) != 0)
 406       return "\nIndex-File: true";
 407
 408    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 409 }
 410                                                                         /*}}}*/
 411 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 412 {
 413    if(Debug)
 414       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 415                 << std::endl;
 416
 417    pkgTagSection Tags;
 418    string ServerSha1;
 419    vector<DiffInfo> available_patches;
 420
 421    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 422    pkgTagFile TF(&Fd);
 423    if (_error->PendingError() == true)
 424       return false;
 425
 426    if(TF.Step(Tags) == true)
 427    {
 428       bool found = false;
 429       DiffInfo d;
 430       string size;
 431
 432       string const tmp = Tags.FindS("SHA1-Current");
 433       std::stringstream ss(tmp);
 434       ss >> ServerSha1 >> size;
 435       unsigned long const ServerSize = atol(size.c_str());
 436
 437       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 438       SHA1Summation SHA1;
 439       SHA1.AddFD(fd.Fd(), fd.Size());
 440       string const local_sha1 = SHA1.Result();
 441
 442       if(local_sha1 == ServerSha1)
 443       {
 444          // we have the same sha1 as the server
 445          if(Debug)
 446             std::clog << "Package file is up-to-date" << std::endl;
 447          // set found to true, this will queue a pkgAcqIndexDiffs with
 448          // a empty availabe_patches
 449          found = true;
 450       }
 451       else
 452       {
 453          if(Debug)
 454             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 455
 456          // check the historie and see what patches we need
 457          string const history = Tags.FindS("SHA1-History");
 458          std::stringstream hist(history);
 459          while(hist >> d.sha1 >> size >> d.file)
 460          {
 461             // read until the first match is found
 462             // from that point on, we probably need all diffs
 463             if(d.sha1 == local_sha1)
 464                found=true;
 465             else if (found == false)
 466                continue;
 467
 468             if(Debug)
 469                std::clog << "Need to get diff: " << d.file << std::endl;
 470             available_patches.push_back(d);
 471          }
 472
 473          if (available_patches.empty() == false)
 474          {
 475             // patching with too many files is rather slow compared to a fast download
 476             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 477             if (fileLimit != 0 && fileLimit < available_patches.size())
 478             {
 479                if (Debug)
 480                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 481                         << ") so fallback to complete download" << std::endl;
 482                return false;
 483             }
 484
 485             // see if the patches are too big
 486             found = false; // it was true and it will be true again at the end
 487             d = *available_patches.begin();
 488             string const firstPatch = d.file;
 489             unsigned long patchesSize = 0;
 490             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 491             while(patches >> d.sha1 >> size >> d.file)
 492             {
 493                if (firstPatch == d.file)
 494                   found = true;
 495                else if (found == false)
 496                   continue;
 497
 498                patchesSize += atol(size.c_str());
 499             }
 500             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 501             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 502             {
 503                if (Debug)
 504                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 505                         << ") so fallback to complete download" << std::endl;
 506                return false;
 507             }
 508          }
 509       }
 510
 511       // we have something, queue the next diff
 512       if(found)
 513       {
 514          // queue the diffs
 515          string::size_type const last_space = Description.rfind(" ");
 516          if(last_space != string::npos)
 517             Description.erase(last_space, Description.size()-last_space);
 518          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 519                               ExpectedHash, ServerSha1, available_patches);
 520          Complete = false;
 521          Status = StatDone;
 522          Dequeue();
 523          return true;
 524       }
 525    }
 526
 527    // Nothing found, report and return false
 528    // Failing here is ok, if we return false later, the full
 529    // IndexFile is queued
 530    if(Debug)
 531       std::clog << "Can't find a patch in the index file" << std::endl;
 532    return false;
 533 }
 534                                                                         /*}}}*/
 535 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 536 {
 537    if(Debug)
 538       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 539                 << "Falling back to normal index file aquire" << std::endl;
 540
 541    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 542                    ExpectedHash);
 543
 544    Complete = false;
 545    Status = StatDone;
 546    Dequeue();
 547 }
 548                                                                         /*}}}*/
 549 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 550                            pkgAcquire::MethodConfig *Cnf)
 551 {
 552    if(Debug)
 553       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 554
 555    Item::Done(Message,Size,Md5Hash,Cnf);
 556
 557    string FinalFile;
 558    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 559
 560    // sucess in downloading the index
 561    // rename the index
 562    FinalFile += string(".IndexDiff");
 563    if(Debug)
 564       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 565                 << std::endl;
 566    Rename(DestFile,FinalFile);
 567    chmod(FinalFile.c_str(),0644);
 568    DestFile = FinalFile;
 569
 570    if(!ParseDiffIndex(DestFile))
 571       return Failed("", NULL);
 572
 573    Complete = true;
 574    Status = StatDone;
 575    Dequeue();
 576    return;
 577 }
 578                                                                         /*}}}*/
 579 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 580 // ---------------------------------------------------------------------
 581 /* The package diff is added to the queue. one object is constructed
 582  * for each diff and the index
 583  */
 584 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 585                                    string URI,string URIDesc,string ShortDesc,
 586                                    HashString ExpectedHash,
 587                                    string ServerSha1,
 588                                    vector<DiffInfo> diffs)
 589    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 590      available_patches(diffs), ServerSha1(ServerSha1)
 591 {
 592
 593    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 594    DestFile += URItoFileName(URI);
 595
 596    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 597
 598    Description = URIDesc;
 599    Desc.Owner = this;
 600    Desc.ShortDesc = ShortDesc;
 601
 602    if(available_patches.size() == 0)
 603    {
 604       // we are done (yeah!)
 605       Finish(true);
 606    }
 607    else
 608    {
 609       // get the next diff
 610       State = StateFetchDiff;
 611       QueueNextDiff();
 612    }
 613 }
 614                                                                         /*}}}*/
 615 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 616 {
 617    if(Debug)
 618       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 619                 << "Falling back to normal index file aquire" << std::endl;
 620    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 621                    ExpectedHash);
 622    Finish();
 623 }
 624                                                                         /*}}}*/
 625 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 626 void pkgAcqIndexDiffs::Finish(bool allDone)
 627 {
 628    // we restore the original name, this is required, otherwise
 629    // the file will be cleaned
 630    if(allDone)
 631    {
 632       DestFile = _config->FindDir("Dir::State::lists");
 633       DestFile += URItoFileName(RealURI);
 634
 635       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 636       {
 637          Status = StatAuthError;
 638          ErrorText = _("MD5Sum mismatch");
 639          Rename(DestFile,DestFile + ".FAILED");
 640          Dequeue();
 641          return;
 642       }
 643
 644       // this is for the "real" finish
 645       Complete = true;
 646       Status = StatDone;
 647       Dequeue();
 648       if(Debug)
 649          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 650       return;
 651    }
 652
 653    if(Debug)
 654       std::clog << "Finishing: " << Desc.URI << std::endl;
 655    Complete = false;
 656    Status = StatDone;
 657    Dequeue();
 658    return;
 659 }
 660                                                                         /*}}}*/
 661 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 662 {
 663
 664    // calc sha1 of the just patched file
 665    string FinalFile = _config->FindDir("Dir::State::lists");
 666    FinalFile += URItoFileName(RealURI);
 667
 668    FileFd fd(FinalFile, FileFd::ReadOnly);
 669    SHA1Summation SHA1;
 670    SHA1.AddFD(fd.Fd(), fd.Size());
 671    string local_sha1 = string(SHA1.Result());
 672    if(Debug)
 673       std::clog << "QueueNextDiff: "
 674                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 675
 676    // final file reached before all patches are applied
 677    if(local_sha1 == ServerSha1)
 678    {
 679       Finish(true);
 680       return true;
 681    }
 682
 683    // remove all patches until the next matching patch is found
 684    // this requires the Index file to be ordered
 685    for(vector<DiffInfo>::iterator I=available_patches.begin();
 686        available_patches.size() > 0 &&
 687           I != available_patches.end() &&
 688           (*I).sha1 != local_sha1;
 689        I++)
 690    {
 691       available_patches.erase(I);
 692    }
 693
 694    // error checking and falling back if no patch was found
 695    if(available_patches.size() == 0)
 696    {
 697       Failed("", NULL);
 698       return false;
 699    }
 700
 701    // queue the right diff
 702    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 703    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 704    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 705    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 706
 707    if(Debug)
 708       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 709
 710    QueueURI(Desc);
 711
 712    return true;
 713 }
 714                                                                         /*}}}*/
 715 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 716                             pkgAcquire::MethodConfig *Cnf)
 717 {
 718    if(Debug)
 719       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 720
 721    Item::Done(Message,Size,Md5Hash,Cnf);
 722
 723    string FinalFile;
 724    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 725
 726    // sucess in downloading a diff, enter ApplyDiff state
 727    if(State == StateFetchDiff)
 728    {
 729
 730       // rred excepts the patch as $FinalFile.ed
 731       Rename(DestFile,FinalFile+".ed");
 732
 733       if(Debug)
 734          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 735
 736       State = StateApplyDiff;
 737       Local = true;
 738       Desc.URI = "rred:" + FinalFile;
 739       QueueURI(Desc);
 740       Mode = "rred";
 741       return;
 742    }
 743
 744
 745    // success in download/apply a diff, queue next (if needed)
 746    if(State == StateApplyDiff)
 747    {
 748       // remove the just applied patch
 749       available_patches.erase(available_patches.begin());
 750
 751       // move into place
 752       if(Debug)
 753       {
 754          std::clog << "Moving patched file in place: " << std::endl
 755                    << DestFile << " -> " << FinalFile << std::endl;
 756       }
 757       Rename(DestFile,FinalFile);
 758       chmod(FinalFile.c_str(),0644);
 759
 760       // see if there is more to download
 761       if(available_patches.size() > 0) {
 762          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 763                               ExpectedHash, ServerSha1, available_patches);
 764          return Finish();
 765       } else
 766          return Finish(true);
 767    }
 768 }
 769                                                                         /*}}}*/
 770 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 771 // ---------------------------------------------------------------------
 772 /* The package file is added to the queue and a second class is
 773    instantiated to fetch the revision file */
 774 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 775                          string URI,string URIDesc,string ShortDesc,
 776                          HashString ExpectedHash, string comprExt)
 777    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 778 {
 779    if(comprExt.empty() == true)
 780    {
 781       // autoselect the compression method
 782       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 783       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 784          comprExt.append(*t).append(" ");
 785       if (comprExt.empty() == false)
 786          comprExt.erase(comprExt.end()-1);
 787    }
 788    CompressionExtension = comprExt;
 789
 790    Init(URI, URIDesc, ShortDesc);
 791 }
 792 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 793                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 794    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 795 {
 796    // autoselect the compression method
 797    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 798    CompressionExtension = "";
 799    if (ExpectedHash.empty() == false)
 800    {
 801       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 802          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 803             CompressionExtension.append(*t).append(" ");
 804    }
 805    else
 806    {
 807       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 808          CompressionExtension.append(*t).append(" ");
 809    }
 810    if (CompressionExtension.empty() == false)
 811       CompressionExtension.erase(CompressionExtension.end()-1);
 812
 813    Init(Target->URI, Target->Description, Target->ShortDesc);
 814 }
 815                                                                         /*}}}*/
 816 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 817 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 818    Decompression = false;
 819    Erase = false;
 820
 821    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 822    DestFile += URItoFileName(URI);
 823
 824    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 825    if (comprExt == "uncompressed")
 826       Desc.URI = URI;
 827    else
 828       Desc.URI = URI + '.' + comprExt;
 829
 830    Desc.Description = URIDesc;
 831    Desc.Owner = this;
 832    Desc.ShortDesc = ShortDesc;
 833
 834    QueueURI(Desc);
 835 }
 836                                                                         /*}}}*/
 837 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 838 // ---------------------------------------------------------------------
 839 /* The only header we use is the last-modified header. */
 840 string pkgAcqIndex::Custom600Headers()
 841 {
 842    string Final = _config->FindDir("Dir::State::lists");
 843    Final += URItoFileName(RealURI);
 844    if (_config->FindB("Acquire::GzipIndexes",false))
 845       Final += ".gz";
 846
 847    string msg = "\nIndex-File: true";
 848    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 849    //        seems to be difficult without breaking ABI
 850    if (ShortDesc().find("Translation") != 0)
 851       msg += "\nFail-Ignore: true";
 852    struct stat Buf;
 853    if (stat(Final.c_str(),&Buf) == 0)
 854       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 855
 856    return msg;
 857 }
 858                                                                         /*}}}*/
 859 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 860 {
 861    size_t const nextExt = CompressionExtension.find(' ');
 862    if (nextExt != std::string::npos)
 863    {
 864       CompressionExtension = CompressionExtension.substr(nextExt+1);
 865       Init(RealURI, Desc.Description, Desc.ShortDesc);
 866       return;
 867    }
 868
 869    // on decompression failure, remove bad versions in partial/
 870    if (Decompression && Erase) {
 871       string s = _config->FindDir("Dir::State::lists") + "partial/";
 872       s.append(URItoFileName(RealURI));
 873       unlink(s.c_str());
 874    }
 875
 876    Item::Failed(Message,Cnf);
 877 }
 878                                                                         /*}}}*/
 879 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 880 // ---------------------------------------------------------------------
 881 /* This goes through a number of states.. On the initial fetch the
 882    method could possibly return an alternate filename which points
 883    to the uncompressed version of the file. If this is so the file
 884    is copied into the partial directory. In all other cases the file
 885    is decompressed with a gzip uri. */
 886 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
 887                        pkgAcquire::MethodConfig *Cfg)
 888 {
 889    Item::Done(Message,Size,Hash,Cfg);
 890
 891    if (Decompression == true)
 892    {
 893       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 894       {
 895          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 896          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 897       }
 898
 899       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 900       {
 901          Status = StatAuthError;
 902          ErrorText = _("Hash Sum mismatch");
 903          Rename(DestFile,DestFile + ".FAILED");
 904          ReportMirrorFailure("HashChecksumFailure");
 905          return;
 906       }
 907
 908       /* Verify the index file for correctness (all indexes must
 909        * have a Package field) (LP: #346386) (Closes: #627642) */
 910       {
 911          FileFd fd(DestFile, FileFd::ReadOnly);
 912          pkgTagSection sec;
 913          pkgTagFile tag(&fd);
 914
 915          // Only test for correctness if the file is not empty (empty is ok)
 916          if (fd.Size() > 0) {
 917             if (_error->PendingError() || !tag.Step(sec)) {
 918                Status = StatError;
 919                _error->DumpErrors();
 920                Rename(DestFile,DestFile + ".FAILED");
 921                return;
 922             } else if (!sec.Exists("Package")) {
 923                Status = StatError;
 924                ErrorText = ("Encountered a section with no Package: header");
 925                Rename(DestFile,DestFile + ".FAILED");
 926                return;
 927             }
 928          }
 929       }
 930
 931       // Done, move it into position
 932       string FinalFile = _config->FindDir("Dir::State::lists");
 933       FinalFile += URItoFileName(RealURI);
 934       Rename(DestFile,FinalFile);
 935       chmod(FinalFile.c_str(),0644);
 936
 937       /* We restore the original name to DestFile so that the clean operation
 938          will work OK */
 939       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 940       DestFile += URItoFileName(RealURI);
 941
 942       // Remove the compressed version.
 943       if (Erase == true)
 944          unlink(DestFile.c_str());
 945       return;
 946    }
 947
 948    Erase = false;
 949    Complete = true;
 950
 951    // Handle the unzipd case
 952    string FileName = LookupTag(Message,"Alt-Filename");
 953    if (FileName.empty() == false)
 954    {
 955       // The files timestamp matches
 956       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 957          return;
 958       Decompression = true;
 959       Local = true;
 960       DestFile += ".decomp";
 961       Desc.URI = "copy:" + FileName;
 962       QueueURI(Desc);
 963       Mode = "copy";
 964       return;
 965    }
 966
 967    FileName = LookupTag(Message,"Filename");
 968    if (FileName.empty() == true)
 969    {
 970       Status = StatError;
 971       ErrorText = "Method gave a blank filename";
 972    }
 973
 974    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 975
 976    // The files timestamp matches
 977    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 978        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 979           // Update DestFile for .gz suffix so that the clean operation keeps it
 980           DestFile += ".gz";
 981       return;
 982     }
 983
 984    if (FileName == DestFile)
 985       Erase = true;
 986    else
 987       Local = true;
 988
 989    string decompProg;
 990
 991    // If we enable compressed indexes and already have gzip, keep it
 992    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 993       string FinalFile = _config->FindDir("Dir::State::lists");
 994       FinalFile += URItoFileName(RealURI) + ".gz";
 995       Rename(DestFile,FinalFile);
 996       chmod(FinalFile.c_str(),0644);
 997
 998       // Update DestFile for .gz suffix so that the clean operation keeps it
 999       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1000       DestFile += URItoFileName(RealURI) + ".gz";
1001       return;
1002     }
1003
1004    // get the binary name for your used compression type
1005    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1006    if(decompProg.empty() == false);
1007    else if(compExt == "uncompressed")
1008       decompProg = "copy";
1009    else {
1010       _error->Error("Unsupported extension: %s", compExt.c_str());
1011       return;
1012    }
1013
1014    Decompression = true;
1015    DestFile += ".decomp";
1016    Desc.URI = decompProg + ":" + FileName;
1017    QueueURI(Desc);
1018    Mode = decompProg.c_str();
1019 }
1020                                                                         /*}}}*/
1021 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1022 // ---------------------------------------------------------------------
1023 /* The Translation file is added to the queue */
1024 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1025                             string URI,string URIDesc,string ShortDesc)
1026   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1027 {
1028 }
1029 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1030                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1031   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1032 {
1033 }
1034                                                                         /*}}}*/
1035 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1036 // ---------------------------------------------------------------------
1037 string pkgAcqIndexTrans::Custom600Headers()
1038 {
1039    string Final = _config->FindDir("Dir::State::lists");
1040    Final += URItoFileName(RealURI);
1041
1042    struct stat Buf;
1043    if (stat(Final.c_str(),&Buf) != 0)
1044       return "\nFail-Ignore: true\nIndex-File: true";
1045    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1046 }
1047                                                                         /*}}}*/
1048 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1049 // ---------------------------------------------------------------------
1050 /* */
1051 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1052 {
1053    size_t const nextExt = CompressionExtension.find(' ');
1054    if (nextExt != std::string::npos)
1055    {
1056       CompressionExtension = CompressionExtension.substr(nextExt+1);
1057       Init(RealURI, Desc.Description, Desc.ShortDesc);
1058       Status = StatIdle;
1059       return;
1060    }
1061
1062    if (Cnf->LocalOnly == true ||
1063        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1064    {
1065       // Ignore this
1066       Status = StatDone;
1067       Complete = false;
1068       Dequeue();
1069       return;
1070    }
1071
1072    Item::Failed(Message,Cnf);
1073 }
1074                                                                         /*}}}*/
1075 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1076                              string URI,string URIDesc,string ShortDesc,
1077                              string MetaIndexURI, string MetaIndexURIDesc,
1078                              string MetaIndexShortDesc,
1079                              const vector<IndexTarget*>* IndexTargets,
1080                              indexRecords* MetaIndexParser) :
1081    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1082    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1083    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1084 {
1085    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1086    DestFile += URItoFileName(URI);
1087
1088    // remove any partial downloaded sig-file in partial/.
1089    // it may confuse proxies and is too small to warrant a
1090    // partial download anyway
1091    unlink(DestFile.c_str());
1092
1093    // Create the item
1094    Desc.Description = URIDesc;
1095    Desc.Owner = this;
1096    Desc.ShortDesc = ShortDesc;
1097    Desc.URI = URI;
1098
1099    string Final = _config->FindDir("Dir::State::lists");
1100    Final += URItoFileName(RealURI);
1101    struct stat Buf;
1102    if (stat(Final.c_str(),&Buf) == 0)
1103    {
1104       // File was already in place.  It needs to be re-downloaded/verified
1105       // because Release might have changed, we do give it a differnt
1106       // name than DestFile because otherwise the http method will
1107       // send If-Range requests and there are too many broken servers
1108       // out there that do not understand them
1109       LastGoodSig = DestFile+".reverify";
1110       Rename(Final,LastGoodSig);
1111    }
1112
1113    QueueURI(Desc);
1114 }
1115                                                                         /*}}}*/
1116 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1117 // ---------------------------------------------------------------------
1118 /* The only header we use is the last-modified header. */
1119 string pkgAcqMetaSig::Custom600Headers()
1120 {
1121    struct stat Buf;
1122    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1123       return "\nIndex-File: true";
1124
1125    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1126 }
1127
1128 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1129                          pkgAcquire::MethodConfig *Cfg)
1130 {
1131    Item::Done(Message,Size,MD5,Cfg);
1132
1133    string FileName = LookupTag(Message,"Filename");
1134    if (FileName.empty() == true)
1135    {
1136       Status = StatError;
1137       ErrorText = "Method gave a blank filename";
1138       return;
1139    }
1140
1141    if (FileName != DestFile)
1142    {
1143       // We have to copy it into place
1144       Local = true;
1145       Desc.URI = "copy:" + FileName;
1146       QueueURI(Desc);
1147       return;
1148    }
1149
1150    Complete = true;
1151
1152    // put the last known good file back on i-m-s hit (it will
1153    // be re-verified again)
1154    // Else do nothing, we have the new file in DestFile then
1155    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1156       Rename(LastGoodSig, DestFile);
1157
1158    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1159    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1160                        MetaIndexShortDesc,  DestFile, IndexTargets,
1161                        MetaIndexParser);
1162
1163 }
1164                                                                         /*}}}*/
1165 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1166 {
1167    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1168
1169    // if we get a network error we fail gracefully
1170    if(Status == StatTransientNetworkError)
1171    {
1172       Item::Failed(Message,Cnf);
1173       // move the sigfile back on transient network failures
1174       if(FileExists(LastGoodSig))
1175          Rename(LastGoodSig,Final);
1176
1177       // set the status back to , Item::Failed likes to reset it
1178       Status = pkgAcquire::Item::StatTransientNetworkError;
1179       return;
1180    }
1181
1182    // Delete any existing sigfile when the acquire failed
1183    unlink(Final.c_str());
1184
1185    // queue a pkgAcqMetaIndex with no sigfile
1186    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1187                        "", IndexTargets, MetaIndexParser);
1188
1189    if (Cnf->LocalOnly == true ||
1190        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1191    {
1192       // Ignore this
1193       Status = StatDone;
1194       Complete = false;
1195       Dequeue();
1196       return;
1197    }
1198
1199    Item::Failed(Message,Cnf);
1200 }
1201                                                                         /*}}}*/
1202 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1203                                  string URI,string URIDesc,string ShortDesc,
1204                                  string SigFile,
1205                                  const vector<struct IndexTarget*>* IndexTargets,
1206                                  indexRecords* MetaIndexParser) :
1207    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1208    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1209 {
1210    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1211    DestFile += URItoFileName(URI);
1212
1213    // Create the item
1214    Desc.Description = URIDesc;
1215    Desc.Owner = this;
1216    Desc.ShortDesc = ShortDesc;
1217    Desc.URI = URI;
1218
1219    QueueURI(Desc);
1220 }
1221                                                                         /*}}}*/
1222 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1223 // ---------------------------------------------------------------------
1224 /* The only header we use is the last-modified header. */
1225 string pkgAcqMetaIndex::Custom600Headers()
1226 {
1227    string Final = _config->FindDir("Dir::State::lists");
1228    Final += URItoFileName(RealURI);
1229
1230    struct stat Buf;
1231    if (stat(Final.c_str(),&Buf) != 0)
1232       return "\nIndex-File: true";
1233
1234    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1235 }
1236                                                                         /*}}}*/
1237 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1238                            pkgAcquire::MethodConfig *Cfg)
1239 {
1240    Item::Done(Message,Size,Hash,Cfg);
1241
1242    // MetaIndexes are done in two passes: one to download the
1243    // metaindex with an appropriate method, and a second to verify it
1244    // with the gpgv method
1245
1246    if (AuthPass == true)
1247    {
1248       AuthDone(Message);
1249
1250       // all cool, move Release file into place
1251       Complete = true;
1252    }
1253    else
1254    {
1255       RetrievalDone(Message);
1256       if (!Complete)
1257          // Still more retrieving to do
1258          return;
1259
1260       if (SigFile == "")
1261       {
1262          // There was no signature file, so we are finished.  Download
1263          // the indexes without verification.
1264          QueueIndexes(false);
1265       }
1266       else
1267       {
1268          // There was a signature file, so pass it to gpgv for
1269          // verification
1270
1271          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1272             std::cerr << "Metaindex acquired, queueing gpg verification ("
1273                       << SigFile << "," << DestFile << ")\n";
1274          AuthPass = true;
1275          Desc.URI = "gpgv:" + SigFile;
1276          QueueURI(Desc);
1277          Mode = "gpgv";
1278          return;
1279       }
1280    }
1281
1282    if (Complete == true)
1283    {
1284       string FinalFile = _config->FindDir("Dir::State::lists");
1285       FinalFile += URItoFileName(RealURI);
1286       if (SigFile == DestFile)
1287          SigFile = FinalFile;
1288       Rename(DestFile,FinalFile);
1289       chmod(FinalFile.c_str(),0644);
1290       DestFile = FinalFile;
1291    }
1292 }
1293                                                                         /*}}}*/
1294 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1295 {
1296    // We have just finished downloading a Release file (it is not
1297    // verified yet)
1298
1299    string FileName = LookupTag(Message,"Filename");
1300    if (FileName.empty() == true)
1301    {
1302       Status = StatError;
1303       ErrorText = "Method gave a blank filename";
1304       return;
1305    }
1306
1307    if (FileName != DestFile)
1308    {
1309       Local = true;
1310       Desc.URI = "copy:" + FileName;
1311       QueueURI(Desc);
1312       return;
1313    }
1314
1315    // make sure to verify against the right file on I-M-S hit
1316    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1317    if(IMSHit)
1318    {
1319       string FinalFile = _config->FindDir("Dir::State::lists");
1320       FinalFile += URItoFileName(RealURI);
1321       if (SigFile == DestFile)
1322          SigFile = FinalFile;
1323       DestFile = FinalFile;
1324    }
1325    Complete = true;
1326 }
1327                                                                         /*}}}*/
1328 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1329 {
1330    // At this point, the gpgv method has succeeded, so there is a
1331    // valid signature from a key in the trusted keyring.  We
1332    // perform additional verification of its contents, and use them
1333    // to verify the indexes we are about to download
1334
1335    if (!MetaIndexParser->Load(DestFile))
1336    {
1337       Status = StatAuthError;
1338       ErrorText = MetaIndexParser->ErrorText;
1339       return;
1340    }
1341
1342    if (!VerifyVendor(Message))
1343    {
1344       return;
1345    }
1346
1347    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1348       std::cerr << "Signature verification succeeded: "
1349                 << DestFile << std::endl;
1350
1351    // Download further indexes with verification
1352    QueueIndexes(true);
1353
1354    // is it a clearsigned MetaIndex file?
1355    if (DestFile == SigFile)
1356       return;
1357
1358    // Done, move signature file into position
1359    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1360       URItoFileName(RealURI) + ".gpg";
1361    Rename(SigFile,VerifiedSigFile);
1362    chmod(VerifiedSigFile.c_str(),0644);
1363 }
1364                                                                         /*}}}*/
1365 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1366 {
1367 #if 0
1368    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1369     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1370    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1371    {
1372       Status = StatError;
1373       ErrorText = MetaIndexParser->ErrorText;
1374       return;
1375    }
1376 #endif
1377    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1378         Target != IndexTargets->end();
1379         Target++)
1380    {
1381       HashString ExpectedIndexHash;
1382       if (verify)
1383       {
1384          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1385          if (Record == NULL)
1386          {
1387             if ((*Target)->IsOptional() == false)
1388             {
1389                Status = StatAuthError;
1390                strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1391                return;
1392             }
1393          }
1394          else
1395          {
1396             ExpectedIndexHash = Record->Hash;
1397             if (_config->FindB("Debug::pkgAcquire::Auth", false))
1398             {
1399                std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1400                std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1401             }
1402             if (ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1403             {
1404                Status = StatAuthError;
1405                strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1406                return;
1407             }
1408          }
1409       }
1410
1411       if ((*Target)->IsOptional() == true)
1412       {
1413          if ((*Target)->IsSubIndex() == true)
1414             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1415                                 (*Target)->ShortDesc, ExpectedIndexHash);
1416          else
1417             new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1418          continue;
1419       }
1420
1421       /* Queue Packages file (either diff or full packages files, depending
1422          on the users option) - we also check if the PDiff Index file is listed
1423          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1424          instead, but passing the required info to it is to much hassle */
1425       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1426           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1427          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1428                              (*Target)->ShortDesc, ExpectedIndexHash);
1429       else
1430          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1431    }
1432 }
1433                                                                         /*}}}*/
1434 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1435 {
1436    string::size_type pos;
1437
1438    // check for missing sigs (that where not fatal because otherwise we had
1439    // bombed earlier)
1440    string missingkeys;
1441    string msg = _("There is no public key available for the "
1442                   "following key IDs:\n");
1443    pos = Message.find("NO_PUBKEY ");
1444    if (pos != std::string::npos)
1445    {
1446       string::size_type start = pos+strlen("NO_PUBKEY ");
1447       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1448       missingkeys += (Fingerprint);
1449    }
1450    if(!missingkeys.empty())
1451       _error->Warning("%s", string(msg+missingkeys).c_str());
1452
1453    string Transformed = MetaIndexParser->GetExpectedDist();
1454
1455    if (Transformed == "../project/experimental")
1456    {
1457       Transformed = "experimental";
1458    }
1459
1460    pos = Transformed.rfind('/');
1461    if (pos != string::npos)
1462    {
1463       Transformed = Transformed.substr(0, pos);
1464    }
1465
1466    if (Transformed == ".")
1467    {
1468       Transformed = "";
1469    }
1470
1471    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1472        MetaIndexParser->GetValidUntil() > 0) {
1473       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1474       if (invalid_since > 0)
1475          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1476          // the time since then the file is invalid - formated in the same way as in
1477          // the download progress display (e.g. 7d 3h 42min 1s)
1478          return _error->Error(
1479             _("Release file for %s is expired (invalid since %s). "
1480               "Updates for this repository will not be applied."),
1481             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1482    }
1483
1484    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1485    {
1486       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1487       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1488       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1489    }
1490
1491    if (MetaIndexParser->CheckDist(Transformed) == false)
1492    {
1493       // This might become fatal one day
1494 //       Status = StatAuthError;
1495 //       ErrorText = "Conflicting distribution; expected "
1496 //          + MetaIndexParser->GetExpectedDist() + " but got "
1497 //          + MetaIndexParser->GetDist();
1498 //       return false;
1499       if (!Transformed.empty())
1500       {
1501          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1502                          Desc.Description.c_str(),
1503                          Transformed.c_str(),
1504                          MetaIndexParser->GetDist().c_str());
1505       }
1506    }
1507
1508    return true;
1509 }
1510                                                                         /*}}}*/
1511 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1512 // ---------------------------------------------------------------------
1513 /* */
1514 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1515 {
1516    if (AuthPass == true)
1517    {
1518       // gpgv method failed, if we have a good signature
1519       string LastGoodSigFile = _config->FindDir("Dir::State::lists");
1520       if (DestFile == SigFile)
1521          LastGoodSigFile.append(URItoFileName(RealURI));
1522       else
1523          LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify");
1524
1525       if(FileExists(LastGoodSigFile))
1526       {
1527          if (DestFile != SigFile)
1528          {
1529             string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1530                                         URItoFileName(RealURI) + ".gpg";
1531             Rename(LastGoodSigFile,VerifiedSigFile);
1532          }
1533          Status = StatTransientNetworkError;
1534          _error->Warning(_("A error occurred during the signature "
1535                            "verification. The repository is not updated "
1536                            "and the previous index files will be used. "
1537                            "GPG error: %s: %s\n"),
1538                          Desc.Description.c_str(),
1539                          LookupTag(Message,"Message").c_str());
1540          RunScripts("APT::Update::Auth-Failure");
1541          return;
1542       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1543          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1544          _error->Error(_("GPG error: %s: %s"),
1545                          Desc.Description.c_str(),
1546                          LookupTag(Message,"Message").c_str());
1547          return;
1548       } else {
1549          _error->Warning(_("GPG error: %s: %s"),
1550                          Desc.Description.c_str(),
1551                          LookupTag(Message,"Message").c_str());
1552       }
1553       // gpgv method failed
1554       ReportMirrorFailure("GPGFailure");
1555    }
1556
1557    /* Always move the meta index, even if gpgv failed. This ensures
1558     * that PackageFile objects are correctly filled in */
1559    if (FileExists(DestFile)) {
1560       string FinalFile = _config->FindDir("Dir::State::lists");
1561       FinalFile += URItoFileName(RealURI);
1562       /* InRelease files become Release files, otherwise
1563        * they would be considered as trusted later on */
1564       if (SigFile == DestFile) {
1565          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1566                                        "Release");
1567          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1568                                        "Release");
1569          SigFile = FinalFile;
1570       }
1571       Rename(DestFile,FinalFile);
1572       chmod(FinalFile.c_str(),0644);
1573
1574       DestFile = FinalFile;
1575    }
1576
1577    // No Release file was present, or verification failed, so fall
1578    // back to queueing Packages files without verification
1579    QueueIndexes(false);
1580 }
1581                                                                         /*}}}*/
1582 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1583                 string const &URI, string const &URIDesc, string const &ShortDesc,
1584                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1585                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1586                 const vector<struct IndexTarget*>* IndexTargets,
1587                 indexRecords* MetaIndexParser) :
1588         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1589         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1590         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1591 {
1592    SigFile = DestFile;
1593 }
1594                                                                         /*}}}*/
1595 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1596 // ---------------------------------------------------------------------
1597 // FIXME: this can go away once the InRelease file is used widely
1598 string pkgAcqMetaClearSig::Custom600Headers()
1599 {
1600    string Final = _config->FindDir("Dir::State::lists");
1601    Final += URItoFileName(RealURI);
1602
1603    struct stat Buf;
1604    if (stat(Final.c_str(),&Buf) != 0)
1605       return "\nIndex-File: true\nFail-Ignore: true\n";
1606
1607    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1608 }
1609                                                                         /*}}}*/
1610 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1611 {
1612    if (AuthPass == false)
1613    {
1614       new pkgAcqMetaSig(Owner,
1615                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1616                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1617                         IndexTargets, MetaIndexParser);
1618       if (Cnf->LocalOnly == true ||
1619           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1620          Dequeue();
1621    }
1622    else
1623       pkgAcqMetaIndex::Failed(Message, Cnf);
1624 }
1625                                                                         /*}}}*/
1626 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1627 // ---------------------------------------------------------------------
1628 /* This just sets up the initial fetch environment and queues the first
1629    possibilitiy */
1630 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1631                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1632                              string &StoreFilename) :
1633                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1634                StoreFilename(StoreFilename), Vf(Version.FileList()),
1635                Trusted(false)
1636 {
1637    Retries = _config->FindI("Acquire::Retries",0);
1638
1639    if (Version.Arch() == 0)
1640    {
1641       _error->Error(_("I wasn't able to locate a file for the %s package. "
1642                       "This might mean you need to manually fix this package. "
1643                       "(due to missing arch)"),
1644                     Version.ParentPkg().Name());
1645       return;
1646    }
1647
1648    /* We need to find a filename to determine the extension. We make the
1649       assumption here that all the available sources for this version share
1650       the same extension.. */
1651    // Skip not source sources, they do not have file fields.
1652    for (; Vf.end() == false; Vf++)
1653    {
1654       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1655          continue;
1656       break;
1657    }
1658
1659    // Does not really matter here.. we are going to fail out below
1660    if (Vf.end() != true)
1661    {
1662       // If this fails to get a file name we will bomb out below.
1663       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1664       if (_error->PendingError() == true)
1665          return;
1666
1667       // Generate the final file name as: package_version_arch.foo
1668       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1669                       QuoteString(Version.VerStr(),"_:") + '_' +
1670                       QuoteString(Version.Arch(),"_:.") +
1671                       "." + flExtension(Parse.FileName());
1672    }
1673
1674    // check if we have one trusted source for the package. if so, switch
1675    // to "TrustedOnly" mode
1676    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1677    {
1678       pkgIndexFile *Index;
1679       if (Sources->FindIndex(i.File(),Index) == false)
1680          continue;
1681       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1682       {
1683          std::cerr << "Checking index: " << Index->Describe()
1684                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1685       }
1686       if (Index->IsTrusted()) {
1687          Trusted = true;
1688          break;
1689       }
1690    }
1691
1692    // "allow-unauthenticated" restores apts old fetching behaviour
1693    // that means that e.g. unauthenticated file:// uris are higher
1694    // priority than authenticated http:// uris
1695    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1696       Trusted = false;
1697
1698    // Select a source
1699    if (QueueNext() == false && _error->PendingError() == false)
1700       _error->Error(_("I wasn't able to locate a file for the %s package. "
1701                     "This might mean you need to manually fix this package."),
1702                     Version.ParentPkg().Name());
1703 }
1704                                                                         /*}}}*/
1705 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1706 // ---------------------------------------------------------------------
1707 /* This queues the next available file version for download. It checks if
1708    the archive is already available in the cache and stashs the MD5 for
1709    checking later. */
1710 bool pkgAcqArchive::QueueNext()
1711 {
1712    string const ForceHash = _config->Find("Acquire::ForceHash");
1713    for (; Vf.end() == false; Vf++)
1714    {
1715       // Ignore not source sources
1716       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1717          continue;
1718
1719       // Try to cross match against the source list
1720       pkgIndexFile *Index;
1721       if (Sources->FindIndex(Vf.File(),Index) == false)
1722             continue;
1723
1724       // only try to get a trusted package from another source if that source
1725       // is also trusted
1726       if(Trusted && !Index->IsTrusted())
1727          continue;
1728
1729       // Grab the text package record
1730       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1731       if (_error->PendingError() == true)
1732          return false;
1733
1734       string PkgFile = Parse.FileName();
1735       if (ForceHash.empty() == false)
1736       {
1737          if(stringcasecmp(ForceHash, "sha512") == 0)
1738             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1739          if(stringcasecmp(ForceHash, "sha256") == 0)
1740             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1741          else if (stringcasecmp(ForceHash, "sha1") == 0)
1742             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1743          else
1744             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1745       }
1746       else
1747       {
1748          string Hash;
1749          if ((Hash = Parse.SHA512Hash()).empty() == false)
1750             ExpectedHash = HashString("SHA512", Hash);
1751          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1752             ExpectedHash = HashString("SHA256", Hash);
1753          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1754             ExpectedHash = HashString("SHA1", Hash);
1755          else
1756             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1757       }
1758       if (PkgFile.empty() == true)
1759          return _error->Error(_("The package index files are corrupted. No Filename: "
1760                               "field for package %s."),
1761                               Version.ParentPkg().Name());
1762
1763       Desc.URI = Index->ArchiveURI(PkgFile);
1764       Desc.Description = Index->ArchiveInfo(Version);
1765       Desc.Owner = this;
1766       Desc.ShortDesc = Version.ParentPkg().Name();
1767
1768       // See if we already have the file. (Legacy filenames)
1769       FileSize = Version->Size;
1770       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1771       struct stat Buf;
1772       if (stat(FinalFile.c_str(),&Buf) == 0)
1773       {
1774          // Make sure the size matches
1775          if ((unsigned long long)Buf.st_size == Version->Size)
1776          {
1777             Complete = true;
1778             Local = true;
1779             Status = StatDone;
1780             StoreFilename = DestFile = FinalFile;
1781             return true;
1782          }
1783
1784          /* Hmm, we have a file and its size does not match, this means it is
1785             an old style mismatched arch */
1786          unlink(FinalFile.c_str());
1787       }
1788
1789       // Check it again using the new style output filenames
1790       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1791       if (stat(FinalFile.c_str(),&Buf) == 0)
1792       {
1793          // Make sure the size matches
1794          if ((unsigned long long)Buf.st_size == Version->Size)
1795          {
1796             Complete = true;
1797             Local = true;
1798             Status = StatDone;
1799             StoreFilename = DestFile = FinalFile;
1800             return true;
1801          }
1802
1803          /* Hmm, we have a file and its size does not match, this shouldnt
1804             happen.. */
1805          unlink(FinalFile.c_str());
1806       }
1807
1808       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1809
1810       // Check the destination file
1811       if (stat(DestFile.c_str(),&Buf) == 0)
1812       {
1813          // Hmm, the partial file is too big, erase it
1814          if ((unsigned long long)Buf.st_size > Version->Size)
1815             unlink(DestFile.c_str());
1816          else
1817             PartialSize = Buf.st_size;
1818       }
1819
1820       // Create the item
1821       Local = false;
1822       Desc.URI = Index->ArchiveURI(PkgFile);
1823       Desc.Description = Index->ArchiveInfo(Version);
1824       Desc.Owner = this;
1825       Desc.ShortDesc = Version.ParentPkg().Name();
1826       QueueURI(Desc);
1827
1828       Vf++;
1829       return true;
1830    }
1831    return false;
1832 }
1833                                                                         /*}}}*/
1834 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1835 // ---------------------------------------------------------------------
1836 /* */
1837 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
1838                          pkgAcquire::MethodConfig *Cfg)
1839 {
1840    Item::Done(Message,Size,CalcHash,Cfg);
1841
1842    // Check the size
1843    if (Size != Version->Size)
1844    {
1845       Status = StatError;
1846       ErrorText = _("Size mismatch");
1847       return;
1848    }
1849
1850    // Check the hash
1851    if(ExpectedHash.toStr() != CalcHash)
1852    {
1853       Status = StatError;
1854       ErrorText = _("Hash Sum mismatch");
1855       if(FileExists(DestFile))
1856          Rename(DestFile,DestFile + ".FAILED");
1857       return;
1858    }
1859
1860    // Grab the output filename
1861    string FileName = LookupTag(Message,"Filename");
1862    if (FileName.empty() == true)
1863    {
1864       Status = StatError;
1865       ErrorText = "Method gave a blank filename";
1866       return;
1867    }
1868
1869    Complete = true;
1870
1871    // Reference filename
1872    if (FileName != DestFile)
1873    {
1874       StoreFilename = DestFile = FileName;
1875       Local = true;
1876       return;
1877    }
1878
1879    // Done, move it into position
1880    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1881    FinalFile += flNotDir(StoreFilename);
1882    Rename(DestFile,FinalFile);
1883
1884    StoreFilename = DestFile = FinalFile;
1885    Complete = true;
1886 }
1887                                                                         /*}}}*/
1888 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1889 // ---------------------------------------------------------------------
1890 /* Here we try other sources */
1891 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1892 {
1893    ErrorText = LookupTag(Message,"Message");
1894
1895    /* We don't really want to retry on failed media swaps, this prevents
1896       that. An interesting observation is that permanent failures are not
1897       recorded. */
1898    if (Cnf->Removable == true &&
1899        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1900    {
1901       // Vf = Version.FileList();
1902       while (Vf.end() == false) Vf++;
1903       StoreFilename = string();
1904       Item::Failed(Message,Cnf);
1905       return;
1906    }
1907
1908    if (QueueNext() == false)
1909    {
1910       // This is the retry counter
1911       if (Retries != 0 &&
1912           Cnf->LocalOnly == false &&
1913           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1914       {
1915          Retries--;
1916          Vf = Version.FileList();
1917          if (QueueNext() == true)
1918             return;
1919       }
1920
1921       StoreFilename = string();
1922       Item::Failed(Message,Cnf);
1923    }
1924 }
1925                                                                         /*}}}*/
1926 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1927 // ---------------------------------------------------------------------
1928 bool pkgAcqArchive::IsTrusted()
1929 {
1930    return Trusted;
1931 }
1932                                                                         /*}}}*/
1933 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1934 // ---------------------------------------------------------------------
1935 /* */
1936 void pkgAcqArchive::Finished()
1937 {
1938    if (Status == pkgAcquire::Item::StatDone &&
1939        Complete == true)
1940       return;
1941    StoreFilename = string();
1942 }
1943                                                                         /*}}}*/
1944 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1945 // ---------------------------------------------------------------------
1946 /* The file is added to the queue */
1947 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1948                        unsigned long long Size,string Dsc,string ShortDesc,
1949                        const string &DestDir, const string &DestFilename,
1950                        bool IsIndexFile) :
1951                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1952 {
1953    Retries = _config->FindI("Acquire::Retries",0);
1954
1955    if(!DestFilename.empty())
1956       DestFile = DestFilename;
1957    else if(!DestDir.empty())
1958       DestFile = DestDir + "/" + flNotDir(URI);
1959    else
1960       DestFile = flNotDir(URI);
1961
1962    // Create the item
1963    Desc.URI = URI;
1964    Desc.Description = Dsc;
1965    Desc.Owner = this;
1966
1967    // Set the short description to the archive component
1968    Desc.ShortDesc = ShortDesc;
1969
1970    // Get the transfer sizes
1971    FileSize = Size;
1972    struct stat Buf;
1973    if (stat(DestFile.c_str(),&Buf) == 0)
1974    {
1975       // Hmm, the partial file is too big, erase it
1976       if ((unsigned long long)Buf.st_size > Size)
1977          unlink(DestFile.c_str());
1978       else
1979          PartialSize = Buf.st_size;
1980    }
1981
1982    QueueURI(Desc);
1983 }
1984                                                                         /*}}}*/
1985 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1986 // ---------------------------------------------------------------------
1987 /* */
1988 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
1989                       pkgAcquire::MethodConfig *Cnf)
1990 {
1991    Item::Done(Message,Size,CalcHash,Cnf);
1992
1993    // Check the hash
1994    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1995    {
1996       Status = StatError;
1997       ErrorText = _("Hash Sum mismatch");
1998       Rename(DestFile,DestFile + ".FAILED");
1999       return;
2000    }
2001
2002    string FileName = LookupTag(Message,"Filename");
2003    if (FileName.empty() == true)
2004    {
2005       Status = StatError;
2006       ErrorText = "Method gave a blank filename";
2007       return;
2008    }
2009
2010    Complete = true;
2011
2012    // The files timestamp matches
2013    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2014       return;
2015
2016    // We have to copy it into place
2017    if (FileName != DestFile)
2018    {
2019       Local = true;
2020       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2021           Cnf->Removable == true)
2022       {
2023          Desc.URI = "copy:" + FileName;
2024          QueueURI(Desc);
2025          return;
2026       }
2027
2028       // Erase the file if it is a symlink so we can overwrite it
2029       struct stat St;
2030       if (lstat(DestFile.c_str(),&St) == 0)
2031       {
2032          if (S_ISLNK(St.st_mode) != 0)
2033             unlink(DestFile.c_str());
2034       }
2035
2036       // Symlink the file
2037       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2038       {
2039          ErrorText = "Link to " + DestFile + " failure ";
2040          Status = StatError;
2041          Complete = false;
2042       }
2043    }
2044 }
2045                                                                         /*}}}*/
2046 // AcqFile::Failed - Failure handler                                    /*{{{*/
2047 // ---------------------------------------------------------------------
2048 /* Here we try other sources */
2049 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2050 {
2051    ErrorText = LookupTag(Message,"Message");
2052
2053    // This is the retry counter
2054    if (Retries != 0 &&
2055        Cnf->LocalOnly == false &&
2056        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2057    {
2058       Retries--;
2059       QueueURI(Desc);
2060       return;
2061    }
2062
2063    Item::Failed(Message,Cnf);
2064 }
2065                                                                         /*}}}*/
2066 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2067 // ---------------------------------------------------------------------
2068 /* The only header we use is the last-modified header. */
2069 string pkgAcqFile::Custom600Headers()
2070 {
2071    if (IsIndexFile)
2072       return "\nIndex-File: true";
2073    return "";
2074 }
2075                                                                         /*}}}*/
2076 bool IndexTarget::IsOptional() const {
2077    if (strncmp(ShortDesc.c_str(), "Translation", 11) != 0)
2078       return false;
2079    return true;
2080 }
2081 bool IndexTarget::IsSubIndex() const {
2082    if (ShortDesc != "TranslationIndex")
2083       return false;
2084    return true;
2085 }