apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.36 1999/04/18 07:25:32 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25 #include <system.h>
  26
  27 #include <sys/stat.h>
  28 #include <unistd.h>
  29                                                                         /*}}}*/
  30
  31 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  32 // ---------------------------------------------------------------------
  33 /* We set the diry flag and make sure that is written to the disk */
  34 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  35                     Map(Map), Cache(Map), Progress(Prog)
  36 {
  37    if (_error->PendingError() == true)
  38       return;
  39
  40    if (Map.Size() == 0)
  41    {
  42       Map.RawAllocate(sizeof(pkgCache::Header));
  43       *Cache.HeaderP = pkgCache::Header();
  44    }
  45    Cache.HeaderP->Dirty = true;
  46    Map.Sync(0,sizeof(pkgCache::Header));
  47    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  48    memset(UniqHash,0,sizeof(UniqHash));
  49 }
  50                                                                         /*}}}*/
  51 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* We sync the data then unset the dirty flag in two steps so as to
  54    advoid a problem during a crash */
  55 pkgCacheGenerator::~pkgCacheGenerator()
  56 {
  57    if (_error->PendingError() == true)
  58       return;
  59    if (Map.Sync() == false)
  60       return;
  61
  62    Cache.HeaderP->Dirty = false;
  63    Map.Sync(0,sizeof(pkgCache::Header));
  64 }
  65                                                                         /*}}}*/
  66 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* This provides the generation of the entries in the cache. Each loop
  69    goes through a single package record from the underlying parse engine. */
  70 bool pkgCacheGenerator::MergeList(ListParser &List)
  71 {
  72    List.Owner = this;
  73
  74    unsigned int Counter = 0;
  75    while (List.Step() == true)
  76    {
  77       // Get a pointer to the package structure
  78       string PackageName = List.Package();
  79       if (PackageName.empty() == true)
  80          return false;
  81
  82       pkgCache::PkgIterator Pkg;
  83       if (NewPackage(Pkg,PackageName) == false)
  84          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  85       Counter++;
  86       if (Counter % 100 == 0)
  87          Progress.Progress(List.Offset());
  88
  89       /* Get a pointer to the version structure. We know the list is sorted
  90          so we use that fact in the search. Insertion of new versions is
  91          done with correct sorting */
  92       string Version = List.Version();
  93       if (Version.empty() == true)
  94       {
  95          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  96             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  97          continue;
  98       }
  99
 100       pkgCache::VerIterator Ver = Pkg.VersionList();
 101       __apt_ptrloc *Last = &Pkg->VersionList;
 102       int Res = 1;
 103       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 104       {
 105          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 106                                  Ver.VerStr() + strlen(Ver.VerStr()));
 107          if (Res >= 0)
 108             break;
 109       }
 110
 111       /* We already have a version for this item, record that we
 112          saw it */
 113       if (Res == 0)
 114       {
 115          if (List.UsePackage(Pkg,Ver) == false)
 116             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 117
 118          if (NewFileVer(Ver,List) == false)
 119             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 120
 121          continue;
 122       }
 123
 124       // Add a new version
 125       *Last = NewVersion(Ver,Version,*Last);
 126       Ver->ParentPkg = Pkg.Index();
 127       if (List.NewVersion(Ver) == false)
 128          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 129
 130       if (List.UsePackage(Pkg,Ver) == false)
 131          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 132
 133       if (NewFileVer(Ver,List) == false)
 134          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 135    }
 136
 137    return true;
 138 }
 139                                                                         /*}}}*/
 140 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 141 // ---------------------------------------------------------------------
 142 /* This creates a new package structure and adds it to the hash table */
 143 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 144 {
 145    Pkg = Cache.FindPkg(Name);
 146    if (Pkg.end() == false)
 147       return true;
 148
 149    // Get a structure
 150    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 151    if (Package == 0)
 152       return false;
 153
 154    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 155
 156    // Insert it into the hash table
 157    unsigned long Hash = Cache.Hash(Name);
 158    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 159    Cache.HeaderP->HashTable[Hash] = Package;
 160
 161    // Set the name and the ID
 162    Pkg->Name = Map.WriteString(Name);
 163    if (Pkg->Name == 0)
 164       return false;
 165    Pkg->ID = Cache.HeaderP->PackageCount++;
 166
 167    return true;
 168 }
 169                                                                         /*}}}*/
 170 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 171 // ---------------------------------------------------------------------
 172 /* */
 173 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 174                                    ListParser &List)
 175 {
 176    // Get a structure
 177    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 178    if (VerFile == 0)
 179       return 0;
 180
 181    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 182    VF->File = CurrentFile - Cache.PkgFileP;
 183
 184    // Link it to the end of the list
 185    __apt_ptrloc *Last = &Ver->FileList;
 186    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 187       Last = &V->NextFile;
 188    VF->NextFile = *Last;
 189    *Last = VF.Index();
 190
 191    VF->Offset = List.Offset();
 192    VF->Size = List.Size();
 193    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 194       Cache.HeaderP->MaxVerFileSize = VF->Size;
 195    Cache.HeaderP->VerFileCount++;
 196
 197    return true;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This puts a version structure in the linked list */
 203 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 204                                             string VerStr,
 205                                             unsigned long Next)
 206 {
 207    // Get a structure
 208    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 209    if (Version == 0)
 210       return 0;
 211
 212    // Fill it in
 213    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 214    Ver->NextVer = Next;
 215    Ver->ID = Cache.HeaderP->VersionCount++;
 216    Ver->VerStr = Map.WriteString(VerStr);
 217    if (Ver->VerStr == 0)
 218       return 0;
 219
 220    return Version;
 221 }
 222                                                                         /*}}}*/
 223 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 224 // ---------------------------------------------------------------------
 225 /* This creates a dependency element in the tree. It is linked to the
 226    version and to the package that it is pointing to. */
 227 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 228                                                string PackageName,
 229                                                string Version,
 230                                                unsigned int Op,
 231                                                unsigned int Type)
 232 {
 233    pkgCache &Cache = Owner->Cache;
 234
 235    // Get a structure
 236    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 237    if (Dependency == 0)
 238       return false;
 239
 240    // Fill it in
 241    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 242    Dep->ParentVer = Ver.Index();
 243    Dep->Type = Type;
 244    Dep->CompareOp = Op;
 245    Dep->ID = Cache.HeaderP->DependsCount++;
 246
 247    // Locate the target package
 248    pkgCache::PkgIterator Pkg;
 249    if (Owner->NewPackage(Pkg,PackageName) == false)
 250       return false;
 251
 252    // Probe the reverse dependency list for a version string that matches
 253    if (Version.empty() == false)
 254    {
 255 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 256          if (I->Version != 0 && I.TargetVer() == Version)
 257             Dep->Version = I->Version;*/
 258       if (Dep->Version == 0)
 259          if ((Dep->Version = WriteString(Version)) == 0)
 260             return false;
 261    }
 262
 263    // Link it to the package
 264    Dep->Package = Pkg.Index();
 265    Dep->NextRevDepends = Pkg->RevDepends;
 266    Pkg->RevDepends = Dep.Index();
 267
 268    /* Link it to the version (at the end of the list)
 269       Caching the old end point speeds up generation substantially */
 270    if (OldDepVer != Ver)
 271    {
 272       OldDepLast = &Ver->DependsList;
 273       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 274          OldDepLast = &D->NextDepends;
 275       OldDepVer = Ver;
 276    }
 277
 278    Dep->NextDepends = *OldDepLast;
 279    *OldDepLast = Dep.Index();
 280    OldDepLast = &Dep->NextDepends;
 281
 282    return true;
 283 }
 284                                                                         /*}}}*/
 285 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 286 // ---------------------------------------------------------------------
 287 /* */
 288 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 289                                                 string PackageName,
 290                                                 string Version)
 291 {
 292    pkgCache &Cache = Owner->Cache;
 293
 294    // We do not add self referencing provides
 295    if (Ver.ParentPkg().Name() == PackageName)
 296       return true;
 297
 298    // Get a structure
 299    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 300    if (Provides == 0)
 301       return false;
 302    Cache.HeaderP->ProvidesCount++;
 303
 304    // Fill it in
 305    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 306    Prv->Version = Ver.Index();
 307    Prv->NextPkgProv = Ver->ProvidesList;
 308    Ver->ProvidesList = Prv.Index();
 309    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 310       return false;
 311
 312    // Locate the target package
 313    pkgCache::PkgIterator Pkg;
 314    if (Owner->NewPackage(Pkg,PackageName) == false)
 315       return false;
 316
 317    // Link it to the package
 318    Prv->ParentPkg = Pkg.Index();
 319    Prv->NextProvides = Pkg->ProvidesList;
 320    Pkg->ProvidesList = Prv.Index();
 321
 322    return true;
 323 }
 324                                                                         /*}}}*/
 325 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 326 // ---------------------------------------------------------------------
 327 /* This is used to select which file is to be associated with all newly
 328    added versions. */
 329 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 330 {
 331    struct stat Buf;
 332    if (stat(File.c_str(),&Buf) == -1)
 333       return _error->Errno("stat","Couldn't stat ",File.c_str());
 334
 335    // Get some space for the structure
 336    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 337    if (CurrentFile == Cache.PkgFileP)
 338       return false;
 339
 340    // Fill it in
 341    CurrentFile->FileName = Map.WriteString(File);
 342    CurrentFile->Size = Buf.st_size;
 343    CurrentFile->mtime = Buf.st_mtime;
 344    CurrentFile->NextFile = Cache.HeaderP->FileList;
 345    CurrentFile->Flags = Flags;
 346    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 347    PkgFileName = File;
 348    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 349    Cache.HeaderP->PackageFileCount++;
 350
 351    if (CurrentFile->FileName == 0)
 352       return false;
 353
 354    Progress.SubProgress(Buf.st_size);
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* This is used to create handles to strings. Given the same text it
 361    always returns the same number */
 362 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 363                                                  unsigned int Size)
 364 {
 365    /* We use a very small transient hash table here, this speeds up generation
 366       by a fair amount on slower machines */
 367    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 368    if (Bucket != 0 &&
 369        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 370       return Bucket->String;
 371
 372    // Search for an insertion point
 373    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 374    int Res = 1;
 375    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 376    for (; I != Cache.StringItemP; Last = &I->NextItem,
 377         I = Cache.StringItemP + I->NextItem)
 378    {
 379       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 380       if (Res >= 0)
 381          break;
 382    }
 383
 384    // Match
 385    if (Res == 0)
 386    {
 387       Bucket = I;
 388       return I->String;
 389    }
 390
 391    // Get a structure
 392    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 393    if (Item == 0)
 394       return 0;
 395
 396    // Fill in the structure
 397    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 398    ItemP->NextItem = I - Cache.StringItemP;
 399    *Last = Item;
 400    ItemP->String = Map.WriteString(S,Size);
 401    if (ItemP->String == 0)
 402       return 0;
 403
 404    Bucket = ItemP;
 405    return ItemP->String;
 406 }
 407                                                                         /*}}}*/
 408
 409 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 410 // ---------------------------------------------------------------------
 411 /* The source cache is checked against the source list and the files
 412    on disk, any difference results in a false. */
 413 bool pkgSrcCacheCheck(pkgSourceList &List)
 414 {
 415    if (_error->PendingError() == true)
 416       return false;
 417
 418    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 419    string ListDir = _config->FindDir("Dir::State::lists");
 420
 421    // Count the number of missing files
 422    int Missing = 0;
 423    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 424    {
 425       // Only cache deb source types.
 426       if (I->Type != pkgSourceList::Item::Deb)
 427       {
 428          Missing++;
 429          continue;
 430       }
 431
 432       string File = ListDir + URItoFileName(I->PackagesURI());
 433       struct stat Buf;
 434       if (stat(File.c_str(),&Buf) != 0)
 435       {
 436          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 437                           I->PackagesInfo().c_str(),File.c_str());
 438          Missing++;
 439       }
 440    }
 441
 442    // Open the source package cache
 443    if (FileExists(CacheFile) == false)
 444       return false;
 445
 446    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 447    if (_error->PendingError() == true)
 448    {
 449       _error->Discard();
 450       return false;
 451    }
 452
 453    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 454    if (_error->PendingError() == true || Map.Size() == 0)
 455    {
 456       _error->Discard();
 457       return false;
 458    }
 459
 460    pkgCache Cache(Map);
 461    if (_error->PendingError() == true)
 462    {
 463       _error->Discard();
 464       return false;
 465    }
 466
 467    // They are certianly out of sync
 468    if (Cache.Head().PackageFileCount != List.size() - Missing)
 469       return false;
 470
 471    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 472    {
 473       // Search for a match in the source list
 474       bool Bad = true;
 475       for (pkgSourceList::const_iterator I = List.begin();
 476            I != List.end(); I++)
 477       {
 478          // Only cache deb source types.
 479          if (I->Type != pkgSourceList::Item::Deb)
 480             continue;
 481
 482          string File = ListDir + URItoFileName(I->PackagesURI());
 483          if (F.FileName() == File)
 484          {
 485             Bad = false;
 486             break;
 487          }
 488       }
 489
 490       // Check if the file matches what was cached
 491       Bad |= !F.IsOk();
 492       if (Bad == true)
 493          return false;
 494    }
 495
 496    return true;
 497 }
 498                                                                         /*}}}*/
 499 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 500 // ---------------------------------------------------------------------
 501 /* This does a simple check of all files used to compose the cache */
 502 bool pkgPkgCacheCheck(string CacheFile)
 503 {
 504    if (_error->PendingError() == true)
 505       return false;
 506
 507    // Open the source package cache
 508    if (FileExists(CacheFile) == false)
 509       return false;
 510
 511    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 512    if (_error->PendingError() == true)
 513    {
 514       _error->Discard();
 515       return false;
 516    }
 517
 518    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 519    if (_error->PendingError() == true || Map.Size() == 0)
 520    {
 521       _error->Discard();
 522       return false;
 523    }
 524
 525    pkgCache Cache(Map);
 526    if (_error->PendingError() == true)
 527    {
 528       _error->Discard();
 529       return false;
 530    }
 531
 532    // Status files that must be in the cache
 533    string Status[3];
 534    Status[0] = _config->FindFile("Dir::State::xstatus");
 535    Status[1]= _config->FindFile("Dir::State::userstatus");
 536    Status[2] = _config->FindFile("Dir::State::status");
 537
 538    // Cheack each file
 539    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 540    {
 541       if (F.IsOk() == false)
 542          return false;
 543
 544       // See if this is one of the status files
 545       for (int I = 0; I != 3; I++)
 546          if (F.FileName() == Status[I])
 547             Status[I] = string();
 548    }
 549
 550    // Make sure all the status files are loaded.
 551    for (int I = 0; I != 3; I++)
 552    {
 553       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 554          return false;
 555    }
 556
 557    return true;
 558 }
 559                                                                         /*}}}*/
 560 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 561 // ---------------------------------------------------------------------
 562 /* This adds the size of all the status files to the size counter */
 563 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 564 {
 565    // Grab the file names
 566    string xstatus = _config->FindFile("Dir::State::xstatus");
 567    string userstatus = _config->FindFile("Dir::State::userstatus");
 568    string status = _config->FindFile("Dir::State::status");
 569
 570    // Grab the sizes
 571    struct stat Buf;
 572    if (stat(xstatus.c_str(),&Buf) == 0)
 573       TotalSize += Buf.st_size;
 574    if (stat(userstatus.c_str(),&Buf) == 0)
 575       TotalSize += Buf.st_size;
 576    if (stat(status.c_str(),&Buf) != 0)
 577       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 578    TotalSize += Buf.st_size;
 579
 580    return true;
 581 }
 582                                                                         /*}}}*/
 583 // MergeStatus - Add the status files to the cache                      /*{{{*/
 584 // ---------------------------------------------------------------------
 585 /* This adds the status files to the map */
 586 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 587                            unsigned long &CurrentSize,unsigned long TotalSize)
 588 {
 589    // Grab the file names
 590    string Status[3];
 591    Status[0] = _config->FindFile("Dir::State::xstatus");
 592    Status[1]= _config->FindFile("Dir::State::userstatus");
 593    Status[2] = _config->FindFile("Dir::State::status");
 594
 595    for (int I = 0; I != 3; I++)
 596    {
 597       // Check if the file exists and it is not the primary status file.
 598       string File = Status[I];
 599       if (I != 2 && FileExists(File) == false)
 600          continue;
 601
 602       FileFd Pkg(File,FileFd::ReadOnly);
 603       debListParser Parser(Pkg);
 604       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 605       if (_error->PendingError() == true)
 606          return _error->Error("Problem opening %s",File.c_str());
 607       CurrentSize += Pkg.Size();
 608
 609       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 610       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 611          return _error->Error("Problem with SelectFile %s",File.c_str());
 612
 613       if (Gen.MergeList(Parser) == false)
 614          return _error->Error("Problem with MergeList %s",File.c_str());
 615       Progress.Progress(Pkg.Size());
 616    }
 617
 618    return true;
 619 }
 620                                                                         /*}}}*/
 621 // GenerateSrcCache - Write the source package lists to the map         /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* This puts the source package cache into the given generator. */
 624 bool pkgGenerateSrcCache(pkgSourceList &List,OpProgress &Progress,
 625                          pkgCacheGenerator &Gen,
 626                          unsigned long &CurrentSize,unsigned long &TotalSize)
 627 {
 628    string ListDir = _config->FindDir("Dir::State::lists");
 629
 630    // Prepare the progress indicator
 631    TotalSize = 0;
 632    struct stat Buf;
 633    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 634    {
 635       string File = ListDir + URItoFileName(I->PackagesURI());
 636       if (stat(File.c_str(),&Buf) != 0)
 637          continue;
 638       TotalSize += Buf.st_size;
 639    }
 640
 641    if (pkgAddSourcesSize(TotalSize) == false)
 642       return false;
 643
 644    // Generate the pkg source cache
 645    CurrentSize = 0;
 646    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 647    {
 648       // Only cache deb source types.
 649       if (I->Type != pkgSourceList::Item::Deb)
 650          continue;
 651
 652       string File = ListDir + URItoFileName(I->PackagesURI());
 653
 654       if (FileExists(File) == false)
 655          continue;
 656
 657       FileFd Pkg(File,FileFd::ReadOnly);
 658       debListParser Parser(Pkg);
 659       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 660       if (_error->PendingError() == true)
 661          return _error->Error("Problem opening %s",File.c_str());
 662       CurrentSize += Pkg.Size();
 663
 664       Progress.SubProgress(0,I->PackagesInfo());
 665       if (Gen.SelectFile(File) == false)
 666          return _error->Error("Problem with SelectFile %s",File.c_str());
 667
 668       if (Gen.MergeList(Parser) == false)
 669          return _error->Error("Problem with MergeList %s",File.c_str());
 670
 671       // Check the release file
 672       string RFile = ListDir + URItoFileName(I->ReleaseURI());
 673       if (FileExists(RFile) == true)
 674       {
 675          FileFd Rel(RFile,FileFd::ReadOnly);
 676          if (_error->PendingError() == true)
 677             return false;
 678          Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 679       }
 680    }
 681
 682    return true;
 683 }
 684                                                                         /*}}}*/
 685 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 686 // ---------------------------------------------------------------------
 687 /* This copies the package source cache and then merges the status and
 688    xstatus files into it. */
 689 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 690 {
 691    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 692
 693    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 694    bool SrcOk = pkgSrcCacheCheck(List);
 695    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 696
 697    // Rebuild the source and package caches
 698    if (SrcOk == false)
 699    {
 700       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 701       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 702       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 703       DynamicMMap Map(CacheF,MMap::Public);
 704       if (_error->PendingError() == true)
 705          return false;
 706
 707       pkgCacheGenerator Gen(Map,Progress);
 708       unsigned long CurrentSize = 0;
 709       unsigned long TotalSize = 0;
 710       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 711          return false;
 712
 713       // Write the src cache
 714       Gen.GetCache().HeaderP->Dirty = false;
 715       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 716          return _error->Error("IO Error saving source cache");
 717       Gen.GetCache().HeaderP->Dirty = true;
 718
 719       // Merge in the source caches
 720       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 721    }
 722
 723    if (PkgOk == true)
 724    {
 725       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 726       return true;
 727    }
 728
 729    // We use the source cache to generate the package cache
 730    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 731
 732    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 733    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 734    DynamicMMap Map(CacheF,MMap::Public);
 735    if (_error->PendingError() == true)
 736       return false;
 737
 738    // Preload the map with the source cache
 739    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 740                     SCacheF.Size()) == false)
 741       return false;
 742
 743    pkgCacheGenerator Gen(Map,Progress);
 744
 745    // Compute the progress
 746    unsigned long TotalSize = 0;
 747    if (pkgAddSourcesSize(TotalSize) == false)
 748       return false;
 749
 750    unsigned long CurrentSize = 0;
 751    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 752 }
 753                                                                         /*}}}*/
 754 // MakeStatusCacheMem - Returns a map for the status cache              /*{{{*/
 755 // ---------------------------------------------------------------------
 756 /* This creates a map object for the status cache. If the process has write
 757    access to the caches then it is the same as MakeStatusCache, otherwise it
 758    creates a memory block and puts the cache in there. */
 759 MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress)
 760 {
 761    /* If the cache file is writeable this is just a wrapper for
 762       MakeStatusCache */
 763    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 764    bool Writeable = access(CacheFile.c_str(),W_OK) == 0;
 765    if (Writeable == true)
 766    {
 767       if (pkgMakeStatusCache(List,Progress) == false)
 768          return 0;
 769
 770       // Open the cache file
 771       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 772       if (_error->PendingError() == true)
 773          return 0;
 774
 775       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 776       if (_error->PendingError() == true)
 777       {
 778          delete Map;
 779          return 0;
 780       }
 781       return Map;
 782    }
 783
 784    // Mostly from MakeStatusCache..
 785    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 786
 787    bool SrcOk = pkgSrcCacheCheck(List);
 788    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 789
 790    // Rebuild the source and package caches
 791    if (SrcOk == false)
 792    {
 793       DynamicMMap *Map = new DynamicMMap(MMap::Public);
 794       if (_error->PendingError() == true)
 795       {
 796          delete Map;
 797          return 0;
 798       }
 799
 800       pkgCacheGenerator Gen(*Map,Progress);
 801       unsigned long CurrentSize = 0;
 802       unsigned long TotalSize = 0;
 803       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 804       {
 805          delete Map;
 806          return 0;
 807       }
 808
 809       // Merge in the source caches
 810       if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 811       {
 812          delete Map;
 813          return 0;
 814       }
 815
 816       return Map;
 817    }
 818
 819    if (PkgOk == true)
 820    {
 821       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 822
 823       // Open the cache file
 824       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 825       if (_error->PendingError() == true)
 826          return 0;
 827
 828       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 829       if (_error->PendingError() == true)
 830       {
 831          delete Map;
 832          return 0;
 833       }
 834       return Map;
 835    }
 836
 837    // We use the source cache to generate the package cache
 838    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 839    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 840    DynamicMMap *Map = new DynamicMMap(MMap::Public);
 841    if (_error->PendingError() == true)
 842    {
 843       delete Map;
 844       return 0;
 845    }
 846
 847    // Preload the map with the source cache
 848    if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 849                     SCacheF.Size()) == false)
 850    {
 851       delete Map;
 852       return 0;
 853    }
 854
 855    pkgCacheGenerator Gen(*Map,Progress);
 856
 857    // Compute the progress
 858    unsigned long TotalSize = 0;
 859    if (pkgAddSourcesSize(TotalSize) == false)
 860    {
 861       delete Map;
 862       return 0;
 863    }
 864
 865    unsigned long CurrentSize = 0;
 866    if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 867    {
 868       delete Map;
 869       return 0;
 870    }
 871
 872    return Map;
 873 }
 874                                                                         /*}}}*/