apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.40 1999/10/31 06:32:27 jgg Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #ifdef __GNUG__
  17 #pragma implementation "apt-pkg/acquire-item.h"
  18 #endif
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24
  25 #include <sys/stat.h>
  26 #include <unistd.h>
  27 #include <errno.h>
  28 #include <string.h>
  29 #include <stdio.h>
  30                                                                         /*}}}*/
  31
  32 // Acquire::Item::Item - Constructor                                    /*{{{*/
  33 // ---------------------------------------------------------------------
  34 /* */
  35 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  36                        PartialSize(0), Mode(0), ID(0), Complete(false),
  37                        Local(false), QueueCounter(0)
  38 {
  39    Owner->Add(this);
  40    Status = StatIdle;
  41 }
  42                                                                         /*}}}*/
  43 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 pkgAcquire::Item::~Item()
  47 {
  48    Owner->Remove(this);
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* We return to an idle state if there are still other queues that could
  54    fetch this object */
  55 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  56 {
  57    Status = StatIdle;
  58    ErrorText = LookupTag(Message,"Message");
  59    if (QueueCounter <= 1)
  60    {
  61       /* This indicates that the file is not available right now but might
  62          be sometime later. If we do a retry cycle then this should be
  63          retried [CDROMs] */
  64       if (Cnf->LocalOnly == true &&
  65           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  66       {
  67          Status = StatIdle;
  68          Dequeue();
  69          return;
  70       }
  71
  72       Status = StatError;
  73       Dequeue();
  74    }
  75 }
  76                                                                         /*}}}*/
  77 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  78 // ---------------------------------------------------------------------
  79 /* Stash status and the file size. Note that setting Complete means
  80    sub-phases of the acquire process such as decompresion are operating */
  81 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  82 {
  83    Status = StatFetching;
  84    if (FileSize == 0 && Complete == false)
  85       FileSize = Size;
  86 }
  87                                                                         /*}}}*/
  88 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 void pkgAcquire::Item::Done(string Message,unsigned long Size,string)
  92 {
  93    // We just downloaded something..
  94    string FileName = LookupTag(Message,"Filename");
  95    if (Complete == false && FileName == DestFile)
  96    {
  97       if (Owner->Log != 0)
  98          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
  99    }
 100
 101    if (FileSize == 0)
 102       FileSize= Size;
 103
 104    Status = StatDone;
 105    ErrorText = string();
 106    Owner->Dequeue(this);
 107 }
 108                                                                         /*}}}*/
 109 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 110 // ---------------------------------------------------------------------
 111 /* This helper function is used by alot of item methods as thier final
 112    step */
 113 void pkgAcquire::Item::Rename(string From,string To)
 114 {
 115    if (rename(From.c_str(),To.c_str()) != 0)
 116    {
 117       char S[300];
 118       sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
 119               From.c_str(),To.c_str());
 120       Status = StatError;
 121       ErrorText = S;
 122    }
 123 }
 124                                                                         /*}}}*/
 125
 126 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* The package file is added to the queue and a second class is
 129    instantiated to fetch the revision file */
 130 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
 131              Item(Owner), Location(Location)
 132 {
 133    Decompression = false;
 134    Erase = false;
 135
 136    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 137    DestFile += URItoFileName(Location->PackagesURI());
 138
 139    // Create the item
 140    Desc.URI = Location->PackagesURI() + ".gz";
 141    Desc.Description = Location->PackagesInfo();
 142    Desc.Owner = this;
 143
 144    // Set the short description to the archive component
 145    if (Location->Dist[Location->Dist.size() - 1] == '/')
 146       Desc.ShortDesc = Location->Dist;
 147    else
 148       Desc.ShortDesc = Location->Dist + '/' + Location->Section;
 149
 150    QueueURI(Desc);
 151
 152    // Create the Release fetch class
 153    new pkgAcqIndexRel(Owner,Location);
 154 }
 155                                                                         /*}}}*/
 156 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* The only header we use is the last-modified header. */
 159 string pkgAcqIndex::Custom600Headers()
 160 {
 161    string Final = _config->FindDir("Dir::State::lists");
 162    Final += URItoFileName(Location->PackagesURI());
 163
 164    struct stat Buf;
 165    if (stat(Final.c_str(),&Buf) != 0)
 166       return "\nIndex-File: true";
 167
 168    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 169 }
 170                                                                         /*}}}*/
 171 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 172 // ---------------------------------------------------------------------
 173 /* This goes through a number of states.. On the initial fetch the
 174    method could possibly return an alternate filename which points
 175    to the uncompressed version of the file. If this is so the file
 176    is copied into the partial directory. In all other cases the file
 177    is decompressed with a gzip uri. */
 178 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
 179 {
 180    Item::Done(Message,Size,MD5);
 181
 182    if (Decompression == true)
 183    {
 184       // Done, move it into position
 185       string FinalFile = _config->FindDir("Dir::State::lists");
 186       FinalFile += URItoFileName(Location->PackagesURI());
 187       Rename(DestFile,FinalFile);
 188
 189       /* We restore the original name to DestFile so that the clean operation
 190          will work OK */
 191       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 192       DestFile += URItoFileName(Location->PackagesURI());
 193
 194       // Remove the compressed version.
 195       if (Erase == true)
 196          unlink(DestFile.c_str());
 197       return;
 198    }
 199
 200    Erase = false;
 201    Complete = true;
 202
 203    // Handle the unzipd case
 204    string FileName = LookupTag(Message,"Alt-Filename");
 205    if (FileName.empty() == false)
 206    {
 207       // The files timestamp matches
 208       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 209          return;
 210
 211       Decompression = true;
 212       Local = true;
 213       DestFile += ".decomp";
 214       Desc.URI = "copy:" + FileName;
 215       QueueURI(Desc);
 216       Mode = "copy";
 217       return;
 218    }
 219
 220    FileName = LookupTag(Message,"Filename");
 221    if (FileName.empty() == true)
 222    {
 223       Status = StatError;
 224       ErrorText = "Method gave a blank filename";
 225    }
 226
 227    // The files timestamp matches
 228    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 229       return;
 230
 231    if (FileName == DestFile)
 232       Erase = true;
 233    else
 234       Local = true;
 235
 236    Decompression = true;
 237    DestFile += ".decomp";
 238    Desc.URI = "gzip:" + FileName,Location->PackagesInfo();
 239    QueueURI(Desc);
 240    Mode = "gzip";
 241 }
 242                                                                         /*}}}*/
 243
 244 // AcqIndexRel::pkgAcqIndexRel - Constructor                            /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* The Release file is added to the queue */
 247 pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
 248                                const pkgSourceList::Item *Location) :
 249                 Item(Owner), Location(Location)
 250 {
 251    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 252    DestFile += URItoFileName(Location->ReleaseURI());
 253
 254    // Create the item
 255    Desc.URI = Location->ReleaseURI();
 256    Desc.Description = Location->ReleaseInfo();
 257    Desc.Owner = this;
 258
 259    // Set the short description to the archive component
 260    if (Location->Dist[Location->Dist.size() - 1] == '/')
 261       Desc.ShortDesc = Location->Dist;
 262    else
 263       Desc.ShortDesc = Location->Dist + '/' + Location->Section;
 264
 265    QueueURI(Desc);
 266 }
 267                                                                         /*}}}*/
 268 // AcqIndexRel::Custom600Headers - Insert custom request headers        /*{{{*/
 269 // ---------------------------------------------------------------------
 270 /* The only header we use is the last-modified header. */
 271 string pkgAcqIndexRel::Custom600Headers()
 272 {
 273    string Final = _config->FindDir("Dir::State::lists");
 274    Final += URItoFileName(Location->ReleaseURI());
 275
 276    struct stat Buf;
 277    if (stat(Final.c_str(),&Buf) != 0)
 278       return "\nIndex-File: true";
 279
 280    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 281 }
 282                                                                         /*}}}*/
 283 // AcqIndexRel::Done - Item downloaded OK                               /*{{{*/
 284 // ---------------------------------------------------------------------
 285 /* The release file was not placed into the download directory then
 286    a copy URI is generated and it is copied there otherwise the file
 287    in the partial directory is moved into .. and the URI is finished. */
 288 void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
 289 {
 290    Item::Done(Message,Size,MD5);
 291
 292    string FileName = LookupTag(Message,"Filename");
 293    if (FileName.empty() == true)
 294    {
 295       Status = StatError;
 296       ErrorText = "Method gave a blank filename";
 297       return;
 298    }
 299
 300    Complete = true;
 301
 302    // The files timestamp matches
 303    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 304       return;
 305
 306    // We have to copy it into place
 307    if (FileName != DestFile)
 308    {
 309       Local = true;
 310       Desc.URI = "copy:" + FileName;
 311       QueueURI(Desc);
 312       return;
 313    }
 314
 315    // Done, move it into position
 316    string FinalFile = _config->FindDir("Dir::State::lists");
 317    FinalFile += URItoFileName(Location->ReleaseURI());
 318    Rename(DestFile,FinalFile);
 319 }
 320                                                                         /*}}}*/
 321 // AcqIndexRel::Failed - Silence failure messages for missing rel files /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 void pkgAcqIndexRel::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 325 {
 326    if (Cnf->LocalOnly == true ||
 327        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 328    {
 329       // Ignore this
 330       Status = StatDone;
 331       Complete = false;
 332       Dequeue();
 333       return;
 334    }
 335
 336    Item::Failed(Message,Cnf);
 337 }
 338                                                                         /*}}}*/
 339
 340 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This just sets up the initial fetch environment and queues the first
 343    possibilitiy */
 344 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
 345                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
 346                              string &StoreFilename) :
 347                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
 348                StoreFilename(StoreFilename), Vf(Version.FileList())
 349 {
 350    Retries = _config->FindI("Acquire::Retries",0);
 351
 352    if (Version.Arch() == 0)
 353    {
 354       _error->Error("I wasn't able to locate file for the %s package. "
 355                     "This might mean you need to manually fix this package. (due to missing arch)",
 356                     Version.ParentPkg().Name());
 357       return;
 358    }
 359
 360    // Generate the final file name as: package_version_arch.deb
 361    StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
 362                    QuoteString(Version.VerStr(),"_:") + '_' +
 363                    QuoteString(Version.Arch(),"_:.") + ".deb";
 364
 365    // Select a source
 366    if (QueueNext() == false && _error->PendingError() == false)
 367       _error->Error("I wasn't able to locate file for the %s package. "
 368                     "This might mean you need to manually fix this package.",
 369                     Version.ParentPkg().Name());
 370 }
 371                                                                         /*}}}*/
 372 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
 373 // ---------------------------------------------------------------------
 374 /* This queues the next available file version for download. It checks if
 375    the archive is already available in the cache and stashs the MD5 for
 376    checking later. */
 377 bool pkgAcqArchive::QueueNext()
 378 {
 379    for (; Vf.end() == false; Vf++)
 380    {
 381       // Ignore not source sources
 382       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
 383          continue;
 384
 385       // Try to cross match against the source list
 386       string PkgFile = flNotDir(Vf.File().FileName());
 387       pkgSourceList::const_iterator Location;
 388       for (Location = Sources->begin(); Location != Sources->end(); Location++)
 389          if (PkgFile == URItoFileName(Location->PackagesURI()))
 390             break;
 391
 392       if (Location == Sources->end())
 393          continue;
 394
 395       // Grab the text package record
 396       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
 397       if (_error->PendingError() == true)
 398          return false;
 399
 400       PkgFile = Parse.FileName();
 401       MD5 = Parse.MD5Hash();
 402       if (PkgFile.empty() == true)
 403          return _error->Error("The package index files are corrupted. No Filename: "
 404                               "field for package %s."
 405                               ,Version.ParentPkg().Name());
 406
 407       // See if we already have the file. (Legacy filenames)
 408       FileSize = Version->Size;
 409       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
 410       struct stat Buf;
 411       if (stat(FinalFile.c_str(),&Buf) == 0)
 412       {
 413          // Make sure the size matches
 414          if ((unsigned)Buf.st_size == Version->Size)
 415          {
 416             Complete = true;
 417             Local = true;
 418             Status = StatDone;
 419             StoreFilename = DestFile = FinalFile;
 420             return true;
 421          }
 422
 423          /* Hmm, we have a file and its size does not match, this means it is
 424             an old style mismatched arch */
 425          unlink(FinalFile.c_str());
 426       }
 427
 428       // Check it again using the new style output filenames
 429       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
 430       if (stat(FinalFile.c_str(),&Buf) == 0)
 431       {
 432          // Make sure the size matches
 433          if ((unsigned)Buf.st_size == Version->Size)
 434          {
 435             Complete = true;
 436             Local = true;
 437             Status = StatDone;
 438             StoreFilename = DestFile = FinalFile;
 439             return true;
 440          }
 441
 442          /* Hmm, we have a file and its size does not match, this shouldnt
 443             happen.. */
 444          unlink(FinalFile.c_str());
 445       }
 446
 447       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
 448
 449       // Check the destination file
 450       if (stat(DestFile.c_str(),&Buf) == 0)
 451       {
 452          // Hmm, the partial file is too big, erase it
 453          if ((unsigned)Buf.st_size > Version->Size)
 454             unlink(DestFile.c_str());
 455          else
 456             PartialSize = Buf.st_size;
 457       }
 458
 459       // Create the item
 460       Desc.URI = Location->ArchiveURI(PkgFile);
 461       Desc.Description = Location->ArchiveInfo(Version);
 462       Desc.Owner = this;
 463       Desc.ShortDesc = Version.ParentPkg().Name();
 464       QueueURI(Desc);
 465
 466       Vf++;
 467       return true;
 468    }
 469    return false;
 470 }
 471                                                                         /*}}}*/
 472 // AcqArchive::Done - Finished fetching                                 /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash)
 476 {
 477    Item::Done(Message,Size,Md5Hash);
 478
 479    // Check the size
 480    if (Size != Version->Size)
 481    {
 482       Status = StatError;
 483       ErrorText = "Size mismatch";
 484       return;
 485    }
 486
 487    // Check the md5
 488    if (Md5Hash.empty() == false && MD5.empty() == false)
 489    {
 490       if (Md5Hash != MD5)
 491       {
 492          Status = StatError;
 493          ErrorText = "MD5Sum mismatch";
 494          Rename(DestFile,DestFile + ".FAILED");
 495          return;
 496       }
 497    }
 498
 499    // Grab the output filename
 500    string FileName = LookupTag(Message,"Filename");
 501    if (FileName.empty() == true)
 502    {
 503       Status = StatError;
 504       ErrorText = "Method gave a blank filename";
 505       return;
 506    }
 507
 508    Complete = true;
 509
 510    // Reference filename
 511    if (FileName != DestFile)
 512    {
 513       StoreFilename = DestFile = FileName;
 514       Local = true;
 515       return;
 516    }
 517
 518    // Done, move it into position
 519    string FinalFile = _config->FindDir("Dir::Cache::Archives");
 520    FinalFile += flNotDir(StoreFilename);
 521    Rename(DestFile,FinalFile);
 522
 523    StoreFilename = DestFile = FinalFile;
 524    Complete = true;
 525 }
 526                                                                         /*}}}*/
 527 // AcqArchive::Failed - Failure handler                                 /*{{{*/
 528 // ---------------------------------------------------------------------
 529 /* Here we try other sources */
 530 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 531 {
 532    ErrorText = LookupTag(Message,"Message");
 533    if (QueueNext() == false)
 534    {
 535       // This is the retry counter
 536       if (Retries != 0 &&
 537           Cnf->LocalOnly == false &&
 538           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 539       {
 540          Retries--;
 541          Vf = Version.FileList();
 542          if (QueueNext() == true)
 543             return;
 544       }
 545
 546       StoreFilename = string();
 547       Item::Failed(Message,Cnf);
 548    }
 549 }
 550                                                                         /*}}}*/
 551 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
 552 // ---------------------------------------------------------------------
 553 /* */
 554 void pkgAcqArchive::Finished()
 555 {
 556    if (Status == pkgAcquire::Item::StatDone &&
 557        Complete == true)
 558       return;
 559    StoreFilename = string();
 560 }
 561                                                                         /*}}}*/
 562
 563 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
 564 // ---------------------------------------------------------------------
 565 /* The file is added to the queue */
 566 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
 567                        unsigned long Size,string Dsc,string ShortDesc) :
 568                        Item(Owner), Md5Hash(MD5)
 569 {
 570    Retries = _config->FindI("Acquire::Retries",0);
 571
 572    DestFile = flNotDir(URI);
 573
 574    // Create the item
 575    Desc.URI = URI;
 576    Desc.Description = Dsc;
 577    Desc.Owner = this;
 578
 579    // Set the short description to the archive component
 580    Desc.ShortDesc = ShortDesc;
 581
 582    // Get the transfer sizes
 583    FileSize = Size;
 584    struct stat Buf;
 585    if (stat(DestFile.c_str(),&Buf) == 0)
 586    {
 587       // Hmm, the partial file is too big, erase it
 588       if ((unsigned)Buf.st_size > Size)
 589          unlink(DestFile.c_str());
 590       else
 591          PartialSize = Buf.st_size;
 592    }
 593
 594    QueueURI(Desc);
 595 }
 596                                                                         /*}}}*/
 597 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* */
 600 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5)
 601 {
 602    // Check the md5
 603    if (Md5Hash.empty() == false && MD5.empty() == false)
 604    {
 605       if (Md5Hash != MD5)
 606       {
 607          Status = StatError;
 608          ErrorText = "MD5Sum mismatch";
 609          Rename(DestFile,DestFile + ".FAILED");
 610          return;
 611       }
 612    }
 613
 614    Item::Done(Message,Size,MD5);
 615
 616    string FileName = LookupTag(Message,"Filename");
 617    if (FileName.empty() == true)
 618    {
 619       Status = StatError;
 620       ErrorText = "Method gave a blank filename";
 621       return;
 622    }
 623
 624    Complete = true;
 625
 626    // The files timestamp matches
 627    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 628       return;
 629
 630    // We have to copy it into place
 631    if (FileName != DestFile)
 632    {
 633       Local = true;
 634       if (_config->FindB("Acquire::Source-Symlinks",true) == false)
 635       {
 636          Desc.URI = "copy:" + FileName;
 637          QueueURI(Desc);
 638          return;
 639       }
 640
 641       // Erase the file if it is a symlink so we can overwrite it
 642       struct stat St;
 643       if (lstat(DestFile.c_str(),&St) == 0)
 644       {
 645          if (S_ISLNK(St.st_mode) != 0)
 646             unlink(DestFile.c_str());
 647       }
 648
 649       // Symlink the file
 650       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
 651       {
 652          ErrorText = "Link to " + DestFile + " failure ";
 653          Status = StatError;
 654          Complete = false;
 655       }
 656    }
 657 }
 658                                                                         /*}}}*/
 659 // AcqFile::Failed - Failure handler                                    /*{{{*/
 660 // ---------------------------------------------------------------------
 661 /* Here we try other sources */
 662 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 663 {
 664    ErrorText = LookupTag(Message,"Message");
 665
 666    // This is the retry counter
 667    if (Retries != 0 &&
 668        Cnf->LocalOnly == false &&
 669        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 670    {
 671       Retries--;
 672       QueueURI(Desc);
 673       return;
 674    }
 675
 676    Item::Failed(Message,Cnf);
 677 }
 678                                                                         /*}}}*/