apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.33 1999/07/10 05:32:25 jgg Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #ifdef __GNUG__
  17 #pragma implementation "apt-pkg/acquire-item.h"
  18 #endif
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24
  25 #include <sys/stat.h>
  26 #include <unistd.h>
  27 #include <errno.h>
  28 #include <string.h>
  29 #include <stdio.h>
  30                                                                         /*}}}*/
  31
  32 // Acquire::Item::Item - Constructor                                    /*{{{*/
  33 // ---------------------------------------------------------------------
  34 /* */
  35 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  36                        PartialSize(0), Mode(0), ID(0), Complete(false),
  37                        Local(false), QueueCounter(0)
  38 {
  39    Owner->Add(this);
  40    Status = StatIdle;
  41 }
  42                                                                         /*}}}*/
  43 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 pkgAcquire::Item::~Item()
  47 {
  48    Owner->Remove(this);
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* We return to an idle state if there are still other queues that could
  54    fetch this object */
  55 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  56 {
  57    Status = StatIdle;
  58    ErrorText = LookupTag(Message,"Message");
  59    if (QueueCounter <= 1)
  60    {
  61       /* This indicates that the file is not available right now but might
  62          be sometime later. If we do a retry cycle then this should be
  63          retried [CDROMs] */
  64       if (Cnf->LocalOnly == true &&
  65           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  66       {
  67          Status = StatIdle;
  68          Dequeue();
  69          return;
  70       }
  71
  72       Status = StatError;
  73       Dequeue();
  74    }
  75 }
  76                                                                         /*}}}*/
  77 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  78 // ---------------------------------------------------------------------
  79 /* Stash status and the file size. Note that setting Complete means
  80    sub-phases of the acquire process such as decompresion are operating */
  81 void pkgAcquire::Item::Start(string Message,unsigned long Size)
  82 {
  83    Status = StatFetching;
  84    if (FileSize == 0 && Complete == false)
  85       FileSize = Size;
  86 }
  87                                                                         /*}}}*/
  88 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 void pkgAcquire::Item::Done(string Message,unsigned long Size,string)
  92 {
  93    // We just downloaded something..
  94    string FileName = LookupTag(Message,"Filename");
  95    if (Complete == false && FileName == DestFile)
  96    {
  97       if (Owner->Log != 0)
  98          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
  99    }
 100
 101    if (FileSize == 0)
 102       FileSize= Size;
 103
 104    Status = StatDone;
 105    ErrorText = string();
 106    Owner->Dequeue(this);
 107 }
 108                                                                         /*}}}*/
 109 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 110 // ---------------------------------------------------------------------
 111 /* This helper function is used by alot of item methods as thier final
 112    step */
 113 void pkgAcquire::Item::Rename(string From,string To)
 114 {
 115    if (rename(From.c_str(),To.c_str()) != 0)
 116    {
 117       char S[300];
 118       sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
 119               From.c_str(),To.c_str());
 120       Status = StatError;
 121       ErrorText = S;
 122    }
 123 }
 124                                                                         /*}}}*/
 125
 126 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* The package file is added to the queue and a second class is
 129    instantiated to fetch the revision file */
 130 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
 131              Item(Owner), Location(Location)
 132 {
 133    Decompression = false;
 134    Erase = false;
 135
 136    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 137    DestFile += URItoFileName(Location->PackagesURI());
 138
 139    // Create the item
 140    Desc.URI = Location->PackagesURI() + ".gz";
 141    Desc.Description = Location->PackagesInfo();
 142    Desc.Owner = this;
 143
 144    // Set the short description to the archive component
 145    if (Location->Dist[Location->Dist.size() - 1] == '/')
 146       Desc.ShortDesc = Location->Dist;
 147    else
 148       Desc.ShortDesc = Location->Dist + '/' + Location->Section;
 149
 150    QueueURI(Desc);
 151
 152    // Create the Release fetch class
 153    new pkgAcqIndexRel(Owner,Location);
 154 }
 155                                                                         /*}}}*/
 156 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* The only header we use is the last-modified header. */
 159 string pkgAcqIndex::Custom600Headers()
 160 {
 161    string Final = _config->FindDir("Dir::State::lists");
 162    Final += URItoFileName(Location->PackagesURI());
 163
 164    struct stat Buf;
 165    if (stat(Final.c_str(),&Buf) != 0)
 166       return "\nIndex-File: true";
 167
 168    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 169 }
 170                                                                         /*}}}*/
 171 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 172 // ---------------------------------------------------------------------
 173 /* This goes through a number of states.. On the initial fetch the
 174    method could possibly return an alternate filename which points
 175    to the uncompressed version of the file. If this is so the file
 176    is copied into the partial directory. In all other cases the file
 177    is decompressed with a gzip uri. */
 178 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
 179 {
 180    Item::Done(Message,Size,MD5);
 181
 182    if (Decompression == true)
 183    {
 184       // Done, move it into position
 185       string FinalFile = _config->FindDir("Dir::State::lists");
 186       FinalFile += URItoFileName(Location->PackagesURI());
 187       Rename(DestFile,FinalFile);
 188
 189       /* We restore the original name to DestFile so that the clean operation
 190          will work OK */
 191       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 192       DestFile += URItoFileName(Location->PackagesURI());
 193
 194       // Remove the compressed version.
 195       if (Erase == true)
 196          unlink(DestFile.c_str());
 197       return;
 198    }
 199
 200    Erase = false;
 201    Complete = true;
 202
 203    // Handle the unzipd case
 204    string FileName = LookupTag(Message,"Alt-Filename");
 205    if (FileName.empty() == false)
 206    {
 207       // The files timestamp matches
 208       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 209          return;
 210
 211       Decompression = true;
 212       Local = true;
 213       DestFile += ".decomp";
 214       Desc.URI = "copy:" + FileName;
 215       QueueURI(Desc);
 216       Mode = "copy";
 217       return;
 218    }
 219
 220    FileName = LookupTag(Message,"Filename");
 221    if (FileName.empty() == true)
 222    {
 223       Status = StatError;
 224       ErrorText = "Method gave a blank filename";
 225    }
 226
 227    // The files timestamp matches
 228    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 229       return;
 230
 231    if (FileName == DestFile)
 232       Erase = true;
 233    else
 234       Local = true;
 235
 236    Decompression = true;
 237    DestFile += ".decomp";
 238    Desc.URI = "gzip:" + FileName,Location->PackagesInfo();
 239    QueueURI(Desc);
 240    Mode = "gzip";
 241 }
 242                                                                         /*}}}*/
 243
 244 // AcqIndexRel::pkgAcqIndexRel - Constructor                            /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* The Release file is added to the queue */
 247 pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
 248                                const pkgSourceList::Item *Location) :
 249                 Item(Owner), Location(Location)
 250 {
 251    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 252    DestFile += URItoFileName(Location->ReleaseURI());
 253
 254    // Create the item
 255    Desc.URI = Location->ReleaseURI();
 256    Desc.Description = Location->ReleaseInfo();
 257    Desc.Owner = this;
 258
 259    // Set the short description to the archive component
 260    if (Location->Dist[Location->Dist.size() - 1] == '/')
 261       Desc.ShortDesc = Location->Dist;
 262    else
 263       Desc.ShortDesc = Location->Dist + '/' + Location->Section;
 264
 265    QueueURI(Desc);
 266 }
 267                                                                         /*}}}*/
 268 // AcqIndexRel::Custom600Headers - Insert custom request headers        /*{{{*/
 269 // ---------------------------------------------------------------------
 270 /* The only header we use is the last-modified header. */
 271 string pkgAcqIndexRel::Custom600Headers()
 272 {
 273    string Final = _config->FindDir("Dir::State::lists");
 274    Final += URItoFileName(Location->ReleaseURI());
 275
 276    struct stat Buf;
 277    if (stat(Final.c_str(),&Buf) != 0)
 278       return "\nIndex-File: true";
 279
 280    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 281 }
 282                                                                         /*}}}*/
 283 // AcqIndexRel::Done - Item downloaded OK                               /*{{{*/
 284 // ---------------------------------------------------------------------
 285 /* The release file was not placed into the download directory then
 286    a copy URI is generated and it is copied there otherwise the file
 287    in the partial directory is moved into .. and the URI is finished. */
 288 void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
 289 {
 290    Item::Done(Message,Size,MD5);
 291
 292    string FileName = LookupTag(Message,"Filename");
 293    if (FileName.empty() == true)
 294    {
 295       Status = StatError;
 296       ErrorText = "Method gave a blank filename";
 297       return;
 298    }
 299
 300    Complete = true;
 301
 302    // The files timestamp matches
 303    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 304       return;
 305
 306    // We have to copy it into place
 307    if (FileName != DestFile)
 308    {
 309       Local = true;
 310       Desc.URI = "copy:" + FileName;
 311       QueueURI(Desc);
 312       return;
 313    }
 314
 315    // Done, move it into position
 316    string FinalFile = _config->FindDir("Dir::State::lists");
 317    FinalFile += URItoFileName(Location->ReleaseURI());
 318    Rename(DestFile,FinalFile);
 319 }
 320                                                                         /*}}}*/
 321 // AcqIndexRel::Failed - Silence failure messages for missing rel files /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 void pkgAcqIndexRel::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 325 {
 326    if (Cnf->LocalOnly == true ||
 327        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 328    {
 329       // Ignore this
 330       Status = StatDone;
 331       Complete = false;
 332       Dequeue();
 333       return;
 334    }
 335
 336    Item::Failed(Message,Cnf);
 337 }
 338                                                                         /*}}}*/
 339
 340 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This just sets up the initial fetch environment and queues the first
 343    possibilitiy */
 344 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
 345                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
 346                              string &StoreFilename) :
 347                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
 348                StoreFilename(StoreFilename), Vf(Version.FileList())
 349 {
 350    Retries = _config->FindI("Acquire::Retries",0);
 351
 352    if (Version.Arch() == 0)
 353       _error->Error("I wasn't able to locate file for the %s package. "
 354                     "This might mean you need to manually fix this package. (due to missing arch)",
 355                     Version.ParentPkg().Name());
 356
 357    // Generate the final file name as: package_version_arch.deb
 358    StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
 359                    QuoteString(Version.VerStr(),"_:") + '_' +
 360                    QuoteString(Version.Arch(),"_:.") + ".deb";
 361
 362    // Select a source
 363    if (QueueNext() == false && _error->PendingError() == false)
 364       _error->Error("I wasn't able to locate file for the %s package. "
 365                     "This might mean you need to manually fix this package.",
 366                     Version.ParentPkg().Name());
 367 }
 368                                                                         /*}}}*/
 369 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
 370 // ---------------------------------------------------------------------
 371 /* This queues the next available file version for download. It checks if
 372    the archive is already available in the cache and stashs the MD5 for
 373    checking later. */
 374 bool pkgAcqArchive::QueueNext()
 375 {
 376    for (; Vf.end() == false; Vf++)
 377    {
 378       // Ignore not source sources
 379       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
 380          continue;
 381
 382       // Try to cross match against the source list
 383       string PkgFile = flNotDir(Vf.File().FileName());
 384       pkgSourceList::const_iterator Location;
 385       for (Location = Sources->begin(); Location != Sources->end(); Location++)
 386          if (PkgFile == URItoFileName(Location->PackagesURI()))
 387             break;
 388
 389       if (Location == Sources->end())
 390          continue;
 391
 392       // Grab the text package record
 393       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
 394       if (_error->PendingError() == true)
 395          return false;
 396
 397       PkgFile = Parse.FileName();
 398       MD5 = Parse.MD5Hash();
 399       if (PkgFile.empty() == true)
 400          return _error->Error("The package index files are corrupted. No Filename: "
 401                               "field for package %s."
 402                               ,Version.ParentPkg().Name());
 403
 404       // See if we already have the file. (Legacy filenames)
 405       FileSize = Version->Size;
 406       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
 407       struct stat Buf;
 408       if (stat(FinalFile.c_str(),&Buf) == 0)
 409       {
 410          // Make sure the size matches
 411          if ((unsigned)Buf.st_size == Version->Size)
 412          {
 413             Complete = true;
 414             Local = true;
 415             Status = StatDone;
 416             StoreFilename = DestFile = FinalFile;
 417             return true;
 418          }
 419
 420          /* Hmm, we have a file and its size does not match, this means it is
 421             an old style mismatched arch */
 422          unlink(FinalFile.c_str());
 423       }
 424
 425       // Check it again using the new style output filenames
 426       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
 427       if (stat(FinalFile.c_str(),&Buf) == 0)
 428       {
 429          // Make sure the size matches
 430          if ((unsigned)Buf.st_size == Version->Size)
 431          {
 432             Complete = true;
 433             Local = true;
 434             Status = StatDone;
 435             StoreFilename = DestFile = FinalFile;
 436             return true;
 437          }
 438
 439          /* Hmm, we have a file and its size does not match, this shouldnt
 440             happen.. */
 441          unlink(FinalFile.c_str());
 442       }
 443
 444       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
 445
 446       // Check the destination file
 447       if (stat(DestFile.c_str(),&Buf) == 0)
 448       {
 449          // Hmm, the partial file is too big, erase it
 450          if ((unsigned)Buf.st_size > Version->Size)
 451             unlink(DestFile.c_str());
 452          else
 453             PartialSize = Buf.st_size;
 454       }
 455
 456       // Create the item
 457       Desc.URI = Location->ArchiveURI(PkgFile);
 458       Desc.Description = Location->ArchiveInfo(Version);
 459       Desc.Owner = this;
 460       Desc.ShortDesc = Version.ParentPkg().Name();
 461       QueueURI(Desc);
 462
 463       Vf++;
 464       return true;
 465    }
 466    return false;
 467 }
 468                                                                         /*}}}*/
 469 // AcqArchive::Done - Finished fetching                                 /*{{{*/
 470 // ---------------------------------------------------------------------
 471 /* */
 472 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash)
 473 {
 474    Item::Done(Message,Size,Md5Hash);
 475
 476    // Check the size
 477    if (Size != Version->Size)
 478    {
 479       _error->Error("Size mismatch for package %s",Version.ParentPkg().Name());
 480       return;
 481    }
 482
 483    // Check the md5
 484    if (Md5Hash.empty() == false && MD5.empty() == false)
 485    {
 486       if (Md5Hash != MD5)
 487       {
 488          _error->Error("MD5Sum mismatch for package %s",Version.ParentPkg().Name());
 489          Rename(DestFile,DestFile + ".FAILED");
 490          return;
 491       }
 492    }
 493
 494    // Grab the output filename
 495    string FileName = LookupTag(Message,"Filename");
 496    if (FileName.empty() == true)
 497    {
 498       Status = StatError;
 499       ErrorText = "Method gave a blank filename";
 500       return;
 501    }
 502
 503    Complete = true;
 504
 505    // Reference filename
 506    if (FileName != DestFile)
 507    {
 508       StoreFilename = DestFile = FileName;
 509       Local = true;
 510       return;
 511    }
 512
 513    // Done, move it into position
 514    string FinalFile = _config->FindDir("Dir::Cache::Archives");
 515    FinalFile += flNotDir(StoreFilename);
 516    Rename(DestFile,FinalFile);
 517
 518    StoreFilename = DestFile = FinalFile;
 519    Complete = true;
 520 }
 521                                                                         /*}}}*/
 522 // AcqArchive::Failed - Failure handler                                 /*{{{*/
 523 // ---------------------------------------------------------------------
 524 /* Here we try other sources */
 525 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 526 {
 527    ErrorText = LookupTag(Message,"Message");
 528    if (QueueNext() == false)
 529    {
 530       // This is the retry counter
 531       if (Retries != 0 &&
 532           Cnf->LocalOnly == false &&
 533           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 534       {
 535          Retries--;
 536          Vf = Version.FileList();
 537          if (QueueNext() == true)
 538             return;
 539       }
 540
 541       StoreFilename = string();
 542       Item::Failed(Message,Cnf);
 543    }
 544 }
 545                                                                         /*}}}*/
 546 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* */
 549 void pkgAcqArchive::Finished()
 550 {
 551    if (Status == pkgAcquire::Item::StatDone &&
 552        Complete == true)
 553       return;
 554    StoreFilename = string();
 555 }
 556                                                                         /*}}}*/
 557
 558 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
 559 // ---------------------------------------------------------------------
 560 /* The file is added to the queue */
 561 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
 562                        unsigned long Size,string Dsc,string ShortDesc) :
 563                        Item(Owner), MD5(MD5)
 564 {
 565    DestFile = flNotDir(URI);
 566
 567    // Create the item
 568    Desc.URI = URI;
 569    Desc.Description = Dsc;
 570    Desc.Owner = this;
 571
 572    // Set the short description to the archive component
 573    Desc.ShortDesc = ShortDesc;
 574
 575    // Get the transfer sizes
 576    FileSize = Size;
 577    struct stat Buf;
 578    if (stat(DestFile.c_str(),&Buf) == 0)
 579    {
 580       // Hmm, the partial file is too big, erase it
 581       if ((unsigned)Buf.st_size > Size)
 582          unlink(DestFile.c_str());
 583       else
 584          PartialSize = Buf.st_size;
 585    }
 586
 587    QueueURI(Desc);
 588 }
 589                                                                         /*}}}*/
 590 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
 591 // ---------------------------------------------------------------------
 592 /* */
 593 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5)
 594 {
 595    Item::Done(Message,Size,MD5);
 596
 597    string FileName = LookupTag(Message,"Filename");
 598    if (FileName.empty() == true)
 599    {
 600       Status = StatError;
 601       ErrorText = "Method gave a blank filename";
 602       return;
 603    }
 604
 605    Complete = true;
 606
 607    // The files timestamp matches
 608    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 609       return;
 610
 611    // We have to copy it into place
 612    if (FileName != DestFile)
 613    {
 614       Local = true;
 615       Desc.URI = "copy:" + FileName;
 616       QueueURI(Desc);
 617       return;
 618    }
 619 }
 620                                                                         /*}}}*/