apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.32 1999/06/06 06:58:36 jgg Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #ifdef __GNUG__
  17 #pragma implementation "apt-pkg/acquire-item.h"
  18 #endif
  19 #include <apt-pkg/acquire-item.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24
  25 #include <sys/stat.h>
  26 #include <unistd.h>
  27 #include <errno.h>
  28 #include <string.h>
  29 #include <stdio.h>
  30                                                                         /*}}}*/
  31
  32 // Acquire::Item::Item - Constructor                                    /*{{{*/
  33 // ---------------------------------------------------------------------
  34 /* */
  35 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  36                        PartialSize(0), Mode(0), ID(0), Complete(false),
  37                        Local(false), QueueCounter(0)
  38 {
  39    Owner->Add(this);
  40    Status = StatIdle;
  41 }
  42                                                                         /*}}}*/
  43 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 pkgAcquire::Item::~Item()
  47 {
  48    Owner->Remove(this);
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* We return to an idle state if there are still other queues that could
  54    fetch this object */
  55 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  56 {
  57    Status = StatIdle;
  58    ErrorText = LookupTag(Message,"Message");
  59    if (QueueCounter <= 1)
  60    {
  61       /* This indicates that the file is not available right now but might
  62          be sometime later. If we do a retry cycle then this should be
  63          retried [CDROMs] */
  64       if (Cnf->LocalOnly == true &&
  65           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  66       {
  67          Status = StatIdle;
  68          Dequeue();
  69          return;
  70       }
  71
  72       Status = StatError;
  73       Dequeue();
  74    }
  75 }
  76                                                                         /*}}}*/
  77 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  78 // ---------------------------------------------------------------------
  79 /* Stash status and the file size. Note that setting Complete means
  80    sub-phases of the acquire process such as decompresion are operating */
  81 void pkgAcquire::Item::Start(string Message,unsigned long Size)
  82 {
  83    Status = StatFetching;
  84    if (FileSize == 0 && Complete == false)
  85       FileSize = Size;
  86 }
  87                                                                         /*}}}*/
  88 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 void pkgAcquire::Item::Done(string Message,unsigned long Size,string)
  92 {
  93    // We just downloaded something..
  94    string FileName = LookupTag(Message,"Filename");
  95    if (Complete == false && FileName == DestFile)
  96    {
  97       if (Owner->Log != 0)
  98          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
  99    }
 100
 101    if (FileSize == 0)
 102       FileSize= Size;
 103
 104    Status = StatDone;
 105    ErrorText = string();
 106    Owner->Dequeue(this);
 107 }
 108                                                                         /*}}}*/
 109 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 110 // ---------------------------------------------------------------------
 111 /* This helper function is used by alot of item methods as thier final
 112    step */
 113 void pkgAcquire::Item::Rename(string From,string To)
 114 {
 115    if (rename(From.c_str(),To.c_str()) != 0)
 116    {
 117       char S[300];
 118       sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
 119               From.c_str(),To.c_str());
 120       Status = StatError;
 121       ErrorText = S;
 122    }
 123 }
 124                                                                         /*}}}*/
 125
 126 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* The package file is added to the queue and a second class is
 129    instantiated to fetch the revision file */
 130 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
 131              Item(Owner), Location(Location)
 132 {
 133    Decompression = false;
 134    Erase = false;
 135
 136    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 137    DestFile += URItoFileName(Location->PackagesURI());
 138
 139    // Create the item
 140    Desc.URI = Location->PackagesURI() + ".gz";
 141    Desc.Description = Location->PackagesInfo();
 142    Desc.Owner = this;
 143
 144    // Set the short description to the archive component
 145    if (Location->Dist[Location->Dist.size() - 1] == '/')
 146       Desc.ShortDesc = Location->Dist;
 147    else
 148       Desc.ShortDesc = Location->Dist + '/' + Location->Section;
 149
 150    QueueURI(Desc);
 151
 152    // Create the Release fetch class
 153    new pkgAcqIndexRel(Owner,Location);
 154 }
 155                                                                         /*}}}*/
 156 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* The only header we use is the last-modified header. */
 159 string pkgAcqIndex::Custom600Headers()
 160 {
 161    string Final = _config->FindDir("Dir::State::lists");
 162    Final += URItoFileName(Location->PackagesURI());
 163
 164    struct stat Buf;
 165    if (stat(Final.c_str(),&Buf) != 0)
 166       return "\nIndex-File: true";
 167
 168    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 169 }
 170                                                                         /*}}}*/
 171 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 172 // ---------------------------------------------------------------------
 173 /* This goes through a number of states.. On the initial fetch the
 174    method could possibly return an alternate filename which points
 175    to the uncompressed version of the file. If this is so the file
 176    is copied into the partial directory. In all other cases the file
 177    is decompressed with a gzip uri. */
 178 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
 179 {
 180    Item::Done(Message,Size,MD5);
 181
 182    if (Decompression == true)
 183    {
 184       // Done, move it into position
 185       string FinalFile = _config->FindDir("Dir::State::lists");
 186       FinalFile += URItoFileName(Location->PackagesURI());
 187       Rename(DestFile,FinalFile);
 188
 189       /* We restore the original name to DestFile so that the clean operation
 190          will work OK */
 191       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 192       DestFile += URItoFileName(Location->PackagesURI());
 193
 194       // Remove the compressed version.
 195       if (Erase == true)
 196          unlink(DestFile.c_str());
 197       return;
 198    }
 199
 200    Erase = false;
 201    Complete = true;
 202
 203    // Handle the unzipd case
 204    string FileName = LookupTag(Message,"Alt-Filename");
 205    if (FileName.empty() == false)
 206    {
 207       // The files timestamp matches
 208       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 209          return;
 210
 211       Decompression = true;
 212       Local = true;
 213       DestFile += ".decomp";
 214       Desc.URI = "copy:" + FileName;
 215       QueueURI(Desc);
 216       Mode = "copy";
 217       return;
 218    }
 219
 220    FileName = LookupTag(Message,"Filename");
 221    if (FileName.empty() == true)
 222    {
 223       Status = StatError;
 224       ErrorText = "Method gave a blank filename";
 225    }
 226
 227    // The files timestamp matches
 228    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 229       return;
 230
 231    if (FileName == DestFile)
 232       Erase = true;
 233    else
 234       Local = true;
 235
 236    Decompression = true;
 237    DestFile += ".decomp";
 238    Desc.URI = "gzip:" + FileName,Location->PackagesInfo();
 239    QueueURI(Desc);
 240    Mode = "gzip";
 241 }
 242                                                                         /*}}}*/
 243
 244 // AcqIndexRel::pkgAcqIndexRel - Constructor                            /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* The Release file is added to the queue */
 247 pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
 248                                const pkgSourceList::Item *Location) :
 249                 Item(Owner), Location(Location)
 250 {
 251    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 252    DestFile += URItoFileName(Location->ReleaseURI());
 253
 254    // Create the item
 255    Desc.URI = Location->ReleaseURI();
 256    Desc.Description = Location->ReleaseInfo();
 257    Desc.Owner = this;
 258
 259    // Set the short description to the archive component
 260    if (Location->Dist[Location->Dist.size() - 1] == '/')
 261       Desc.ShortDesc = Location->Dist;
 262    else
 263       Desc.ShortDesc = Location->Dist + '/' + Location->Section;
 264
 265    QueueURI(Desc);
 266 }
 267                                                                         /*}}}*/
 268 // AcqIndexRel::Custom600Headers - Insert custom request headers        /*{{{*/
 269 // ---------------------------------------------------------------------
 270 /* The only header we use is the last-modified header. */
 271 string pkgAcqIndexRel::Custom600Headers()
 272 {
 273    string Final = _config->FindDir("Dir::State::lists");
 274    Final += URItoFileName(Location->ReleaseURI());
 275
 276    struct stat Buf;
 277    if (stat(Final.c_str(),&Buf) != 0)
 278       return "\nIndex-File: true";
 279
 280    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 281 }
 282                                                                         /*}}}*/
 283 // AcqIndexRel::Done - Item downloaded OK                               /*{{{*/
 284 // ---------------------------------------------------------------------
 285 /* The release file was not placed into the download directory then
 286    a copy URI is generated and it is copied there otherwise the file
 287    in the partial directory is moved into .. and the URI is finished. */
 288 void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
 289 {
 290    Item::Done(Message,Size,MD5);
 291
 292    string FileName = LookupTag(Message,"Filename");
 293    if (FileName.empty() == true)
 294    {
 295       Status = StatError;
 296       ErrorText = "Method gave a blank filename";
 297       return;
 298    }
 299
 300    Complete = true;
 301
 302    // The files timestamp matches
 303    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 304       return;
 305
 306    // We have to copy it into place
 307    if (FileName != DestFile)
 308    {
 309       Local = true;
 310       Desc.URI = "copy:" + FileName;
 311       QueueURI(Desc);
 312       return;
 313    }
 314
 315    // Done, move it into position
 316    string FinalFile = _config->FindDir("Dir::State::lists");
 317    FinalFile += URItoFileName(Location->ReleaseURI());
 318    Rename(DestFile,FinalFile);
 319 }
 320                                                                         /*}}}*/
 321 // AcqIndexRel::Failed - Silence failure messages for missing rel files /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 void pkgAcqIndexRel::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 325 {
 326    // This is the retry counter
 327    if (Cnf->LocalOnly == true ||
 328        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 329    {
 330       // Ignore this
 331       Status = StatDone;
 332       Complete = false;
 333       Dequeue();
 334       return;
 335    }
 336
 337    Item::Failed(Message,Cnf);
 338 }
 339                                                                         /*}}}*/
 340
 341 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* This just sets up the initial fetch environment and queues the first
 344    possibilitiy */
 345 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
 346                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
 347                              string &StoreFilename) :
 348                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
 349                StoreFilename(StoreFilename), Vf(Version.FileList())
 350 {
 351    Retries = _config->FindI("Acquire::Retries",0);
 352
 353    if (Version.Arch() == 0)
 354       _error->Error("I wasn't able to locate file for the %s package. "
 355                     "This might mean you need to manually fix this package. (due to missing arch)",
 356                     Version.ParentPkg().Name());
 357
 358    // Generate the final file name as: package_version_arch.deb
 359    StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
 360                    QuoteString(Version.VerStr(),"_:") + '_' +
 361                    QuoteString(Version.Arch(),"_:.") + ".deb";
 362
 363    // Select a source
 364    if (QueueNext() == false && _error->PendingError() == false)
 365       _error->Error("I wasn't able to locate file for the %s package. "
 366                     "This might mean you need to manually fix this package.",
 367                     Version.ParentPkg().Name());
 368 }
 369                                                                         /*}}}*/
 370 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
 371 // ---------------------------------------------------------------------
 372 /* This queues the next available file version for download. It checks if
 373    the archive is already available in the cache and stashs the MD5 for
 374    checking later. */
 375 bool pkgAcqArchive::QueueNext()
 376 {
 377    for (; Vf.end() == false; Vf++)
 378    {
 379       // Ignore not source sources
 380       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
 381          continue;
 382
 383       // Try to cross match against the source list
 384       string PkgFile = flNotDir(Vf.File().FileName());
 385       pkgSourceList::const_iterator Location;
 386       for (Location = Sources->begin(); Location != Sources->end(); Location++)
 387          if (PkgFile == URItoFileName(Location->PackagesURI()))
 388             break;
 389
 390       if (Location == Sources->end())
 391          continue;
 392
 393       // Grab the text package record
 394       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
 395       if (_error->PendingError() == true)
 396          return false;
 397
 398       PkgFile = Parse.FileName();
 399       MD5 = Parse.MD5Hash();
 400       if (PkgFile.empty() == true)
 401          return _error->Error("The package index files are corrupted. No Filename: "
 402                               "field for package %s."
 403                               ,Version.ParentPkg().Name());
 404
 405       // See if we already have the file. (Legacy filenames)
 406       FileSize = Version->Size;
 407       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
 408       struct stat Buf;
 409       if (stat(FinalFile.c_str(),&Buf) == 0)
 410       {
 411          // Make sure the size matches
 412          if ((unsigned)Buf.st_size == Version->Size)
 413          {
 414             Complete = true;
 415             Local = true;
 416             Status = StatDone;
 417             StoreFilename = DestFile = FinalFile;
 418             return true;
 419          }
 420
 421          /* Hmm, we have a file and its size does not match, this means it is
 422             an old style mismatched arch */
 423          unlink(FinalFile.c_str());
 424       }
 425
 426       // Check it again using the new style output filenames
 427       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
 428       if (stat(FinalFile.c_str(),&Buf) == 0)
 429       {
 430          // Make sure the size matches
 431          if ((unsigned)Buf.st_size == Version->Size)
 432          {
 433             Complete = true;
 434             Local = true;
 435             Status = StatDone;
 436             StoreFilename = DestFile = FinalFile;
 437             return true;
 438          }
 439
 440          /* Hmm, we have a file and its size does not match, this shouldnt
 441             happen.. */
 442          unlink(FinalFile.c_str());
 443       }
 444
 445       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
 446
 447       // Check the destination file
 448       if (stat(DestFile.c_str(),&Buf) == 0)
 449       {
 450          // Hmm, the partial file is too big, erase it
 451          if ((unsigned)Buf.st_size > Version->Size)
 452             unlink(DestFile.c_str());
 453          else
 454             PartialSize = Buf.st_size;
 455       }
 456
 457       // Create the item
 458       Desc.URI = Location->ArchiveURI(PkgFile);
 459       Desc.Description = Location->ArchiveInfo(Version);
 460       Desc.Owner = this;
 461       Desc.ShortDesc = Version.ParentPkg().Name();
 462       QueueURI(Desc);
 463
 464       Vf++;
 465       return true;
 466    }
 467    return false;
 468 }
 469                                                                         /*}}}*/
 470 // AcqArchive::Done - Finished fetching                                 /*{{{*/
 471 // ---------------------------------------------------------------------
 472 /* */
 473 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash)
 474 {
 475    Item::Done(Message,Size,Md5Hash);
 476
 477    // Check the size
 478    if (Size != Version->Size)
 479    {
 480       _error->Error("Size mismatch for package %s",Version.ParentPkg().Name());
 481       return;
 482    }
 483
 484    // Check the md5
 485    if (Md5Hash.empty() == false && MD5.empty() == false)
 486    {
 487       if (Md5Hash != MD5)
 488       {
 489          _error->Error("MD5Sum mismatch for package %s",Version.ParentPkg().Name());
 490          Rename(DestFile,DestFile + ".FAILED");
 491          return;
 492       }
 493    }
 494
 495    // Grab the output filename
 496    string FileName = LookupTag(Message,"Filename");
 497    if (FileName.empty() == true)
 498    {
 499       Status = StatError;
 500       ErrorText = "Method gave a blank filename";
 501       return;
 502    }
 503
 504    Complete = true;
 505
 506    // Reference filename
 507    if (FileName != DestFile)
 508    {
 509       StoreFilename = DestFile = FileName;
 510       Local = true;
 511       return;
 512    }
 513
 514    // Done, move it into position
 515    string FinalFile = _config->FindDir("Dir::Cache::Archives");
 516    FinalFile += flNotDir(StoreFilename);
 517    Rename(DestFile,FinalFile);
 518
 519    StoreFilename = DestFile = FinalFile;
 520    Complete = true;
 521 }
 522                                                                         /*}}}*/
 523 // AcqArchive::Failed - Failure handler                                 /*{{{*/
 524 // ---------------------------------------------------------------------
 525 /* Here we try other sources */
 526 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 527 {
 528    ErrorText = LookupTag(Message,"Message");
 529    if (QueueNext() == false)
 530    {
 531       // This is the retry counter
 532       if (Retries != 0 &&
 533           Cnf->LocalOnly == false &&
 534           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 535       {
 536          Retries--;
 537          Vf = Version.FileList();
 538          if (QueueNext() == true)
 539             return;
 540       }
 541
 542       StoreFilename = string();
 543       Item::Failed(Message,Cnf);
 544    }
 545 }
 546                                                                         /*}}}*/
 547 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
 548 // ---------------------------------------------------------------------
 549 /* */
 550 void pkgAcqArchive::Finished()
 551 {
 552    if (Status == pkgAcquire::Item::StatDone &&
 553        Complete == true)
 554       return;
 555    StoreFilename = string();
 556 }
 557                                                                         /*}}}*/
 558
 559 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
 560 // ---------------------------------------------------------------------
 561 /* The file is added to the queue */
 562 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
 563                        unsigned long Size,string Dsc,string ShortDesc) :
 564                        Item(Owner), MD5(MD5)
 565 {
 566    DestFile = flNotDir(URI);
 567
 568    // Create the item
 569    Desc.URI = URI;
 570    Desc.Description = Dsc;
 571    Desc.Owner = this;
 572
 573    // Set the short description to the archive component
 574    Desc.ShortDesc = ShortDesc;
 575
 576    // Get the transfer sizes
 577    FileSize = Size;
 578    struct stat Buf;
 579    if (stat(DestFile.c_str(),&Buf) == 0)
 580    {
 581       // Hmm, the partial file is too big, erase it
 582       if ((unsigned)Buf.st_size > Size)
 583          unlink(DestFile.c_str());
 584       else
 585          PartialSize = Buf.st_size;
 586    }
 587
 588    QueueURI(Desc);
 589 }
 590                                                                         /*}}}*/
 591 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* */
 594 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5)
 595 {
 596    Item::Done(Message,Size,MD5);
 597
 598    string FileName = LookupTag(Message,"Filename");
 599    if (FileName.empty() == true)
 600    {
 601       Status = StatError;
 602       ErrorText = "Method gave a blank filename";
 603       return;
 604    }
 605
 606    Complete = true;
 607
 608    // The files timestamp matches
 609    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 610       return;
 611
 612    // We have to copy it into place
 613    if (FileName != DestFile)
 614    {
 615       Local = true;
 616       Desc.URI = "copy:" + FileName;
 617       QueueURI(Desc);
 618       return;
 619    }
 620 }
 621                                                                         /*}}}*/