apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.37 1999/04/19 02:35:38 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25 #include <system.h>
  26
  27 #include <sys/stat.h>
  28 #include <unistd.h>
  29 #include <errno.h>
  30                                                                         /*}}}*/
  31
  32 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  33 // ---------------------------------------------------------------------
  34 /* We set the diry flag and make sure that is written to the disk */
  35 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  36                     Map(Map), Cache(Map), Progress(Prog)
  37 {
  38    if (_error->PendingError() == true)
  39       return;
  40
  41    if (Map.Size() == 0)
  42    {
  43       Map.RawAllocate(sizeof(pkgCache::Header));
  44       *Cache.HeaderP = pkgCache::Header();
  45    }
  46    Cache.HeaderP->Dirty = true;
  47    Map.Sync(0,sizeof(pkgCache::Header));
  48    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  49    memset(UniqHash,0,sizeof(UniqHash));
  50 }
  51                                                                         /*}}}*/
  52 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We sync the data then unset the dirty flag in two steps so as to
  55    advoid a problem during a crash */
  56 pkgCacheGenerator::~pkgCacheGenerator()
  57 {
  58    if (_error->PendingError() == true)
  59       return;
  60    if (Map.Sync() == false)
  61       return;
  62
  63    Cache.HeaderP->Dirty = false;
  64    Map.Sync(0,sizeof(pkgCache::Header));
  65 }
  66                                                                         /*}}}*/
  67 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* This provides the generation of the entries in the cache. Each loop
  70    goes through a single package record from the underlying parse engine. */
  71 bool pkgCacheGenerator::MergeList(ListParser &List)
  72 {
  73    List.Owner = this;
  74
  75    unsigned int Counter = 0;
  76    while (List.Step() == true)
  77    {
  78       // Get a pointer to the package structure
  79       string PackageName = List.Package();
  80       if (PackageName.empty() == true)
  81          return false;
  82
  83       pkgCache::PkgIterator Pkg;
  84       if (NewPackage(Pkg,PackageName) == false)
  85          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  86       Counter++;
  87       if (Counter % 100 == 0)
  88          Progress.Progress(List.Offset());
  89
  90       /* Get a pointer to the version structure. We know the list is sorted
  91          so we use that fact in the search. Insertion of new versions is
  92          done with correct sorting */
  93       string Version = List.Version();
  94       if (Version.empty() == true)
  95       {
  96          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  97             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  98          continue;
  99       }
 100
 101       pkgCache::VerIterator Ver = Pkg.VersionList();
 102       __apt_ptrloc *Last = &Pkg->VersionList;
 103       int Res = 1;
 104       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 105       {
 106          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 107                                  Ver.VerStr() + strlen(Ver.VerStr()));
 108          if (Res >= 0)
 109             break;
 110       }
 111
 112       /* We already have a version for this item, record that we
 113          saw it */
 114       if (Res == 0)
 115       {
 116          if (List.UsePackage(Pkg,Ver) == false)
 117             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 118
 119          if (NewFileVer(Ver,List) == false)
 120             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 121
 122          continue;
 123       }
 124
 125       // Add a new version
 126       *Last = NewVersion(Ver,Version,*Last);
 127       Ver->ParentPkg = Pkg.Index();
 128       if (List.NewVersion(Ver) == false)
 129          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 130
 131       if (List.UsePackage(Pkg,Ver) == false)
 132          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 133
 134       if (NewFileVer(Ver,List) == false)
 135          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 136    }
 137
 138    return true;
 139 }
 140                                                                         /*}}}*/
 141 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 142 // ---------------------------------------------------------------------
 143 /* This creates a new package structure and adds it to the hash table */
 144 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 145 {
 146    Pkg = Cache.FindPkg(Name);
 147    if (Pkg.end() == false)
 148       return true;
 149
 150    // Get a structure
 151    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 152    if (Package == 0)
 153       return false;
 154
 155    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 156
 157    // Insert it into the hash table
 158    unsigned long Hash = Cache.Hash(Name);
 159    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 160    Cache.HeaderP->HashTable[Hash] = Package;
 161
 162    // Set the name and the ID
 163    Pkg->Name = Map.WriteString(Name);
 164    if (Pkg->Name == 0)
 165       return false;
 166    Pkg->ID = Cache.HeaderP->PackageCount++;
 167
 168    return true;
 169 }
 170                                                                         /*}}}*/
 171 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 172 // ---------------------------------------------------------------------
 173 /* */
 174 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 175                                    ListParser &List)
 176 {
 177    // Get a structure
 178    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 179    if (VerFile == 0)
 180       return 0;
 181
 182    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 183    VF->File = CurrentFile - Cache.PkgFileP;
 184
 185    // Link it to the end of the list
 186    __apt_ptrloc *Last = &Ver->FileList;
 187    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 188       Last = &V->NextFile;
 189    VF->NextFile = *Last;
 190    *Last = VF.Index();
 191
 192    VF->Offset = List.Offset();
 193    VF->Size = List.Size();
 194    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 195       Cache.HeaderP->MaxVerFileSize = VF->Size;
 196    Cache.HeaderP->VerFileCount++;
 197
 198    return true;
 199 }
 200                                                                         /*}}}*/
 201 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 202 // ---------------------------------------------------------------------
 203 /* This puts a version structure in the linked list */
 204 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 205                                             string VerStr,
 206                                             unsigned long Next)
 207 {
 208    // Get a structure
 209    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 210    if (Version == 0)
 211       return 0;
 212
 213    // Fill it in
 214    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 215    Ver->NextVer = Next;
 216    Ver->ID = Cache.HeaderP->VersionCount++;
 217    Ver->VerStr = Map.WriteString(VerStr);
 218    if (Ver->VerStr == 0)
 219       return 0;
 220
 221    return Version;
 222 }
 223                                                                         /*}}}*/
 224 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 225 // ---------------------------------------------------------------------
 226 /* This creates a dependency element in the tree. It is linked to the
 227    version and to the package that it is pointing to. */
 228 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 229                                                string PackageName,
 230                                                string Version,
 231                                                unsigned int Op,
 232                                                unsigned int Type)
 233 {
 234    pkgCache &Cache = Owner->Cache;
 235
 236    // Get a structure
 237    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 238    if (Dependency == 0)
 239       return false;
 240
 241    // Fill it in
 242    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 243    Dep->ParentVer = Ver.Index();
 244    Dep->Type = Type;
 245    Dep->CompareOp = Op;
 246    Dep->ID = Cache.HeaderP->DependsCount++;
 247
 248    // Locate the target package
 249    pkgCache::PkgIterator Pkg;
 250    if (Owner->NewPackage(Pkg,PackageName) == false)
 251       return false;
 252
 253    // Probe the reverse dependency list for a version string that matches
 254    if (Version.empty() == false)
 255    {
 256 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 257          if (I->Version != 0 && I.TargetVer() == Version)
 258             Dep->Version = I->Version;*/
 259       if (Dep->Version == 0)
 260          if ((Dep->Version = WriteString(Version)) == 0)
 261             return false;
 262    }
 263
 264    // Link it to the package
 265    Dep->Package = Pkg.Index();
 266    Dep->NextRevDepends = Pkg->RevDepends;
 267    Pkg->RevDepends = Dep.Index();
 268
 269    /* Link it to the version (at the end of the list)
 270       Caching the old end point speeds up generation substantially */
 271    if (OldDepVer != Ver)
 272    {
 273       OldDepLast = &Ver->DependsList;
 274       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 275          OldDepLast = &D->NextDepends;
 276       OldDepVer = Ver;
 277    }
 278
 279    Dep->NextDepends = *OldDepLast;
 280    *OldDepLast = Dep.Index();
 281    OldDepLast = &Dep->NextDepends;
 282
 283    return true;
 284 }
 285                                                                         /*}}}*/
 286 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 287 // ---------------------------------------------------------------------
 288 /* */
 289 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 290                                                 string PackageName,
 291                                                 string Version)
 292 {
 293    pkgCache &Cache = Owner->Cache;
 294
 295    // We do not add self referencing provides
 296    if (Ver.ParentPkg().Name() == PackageName)
 297       return true;
 298
 299    // Get a structure
 300    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 301    if (Provides == 0)
 302       return false;
 303    Cache.HeaderP->ProvidesCount++;
 304
 305    // Fill it in
 306    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 307    Prv->Version = Ver.Index();
 308    Prv->NextPkgProv = Ver->ProvidesList;
 309    Ver->ProvidesList = Prv.Index();
 310    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 311       return false;
 312
 313    // Locate the target package
 314    pkgCache::PkgIterator Pkg;
 315    if (Owner->NewPackage(Pkg,PackageName) == false)
 316       return false;
 317
 318    // Link it to the package
 319    Prv->ParentPkg = Pkg.Index();
 320    Prv->NextProvides = Pkg->ProvidesList;
 321    Pkg->ProvidesList = Prv.Index();
 322
 323    return true;
 324 }
 325                                                                         /*}}}*/
 326 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* This is used to select which file is to be associated with all newly
 329    added versions. */
 330 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 331 {
 332    struct stat Buf;
 333    if (stat(File.c_str(),&Buf) == -1)
 334       return _error->Errno("stat","Couldn't stat ",File.c_str());
 335
 336    // Get some space for the structure
 337    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 338    if (CurrentFile == Cache.PkgFileP)
 339       return false;
 340
 341    // Fill it in
 342    CurrentFile->FileName = Map.WriteString(File);
 343    CurrentFile->Size = Buf.st_size;
 344    CurrentFile->mtime = Buf.st_mtime;
 345    CurrentFile->NextFile = Cache.HeaderP->FileList;
 346    CurrentFile->Flags = Flags;
 347    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 348    PkgFileName = File;
 349    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 350    Cache.HeaderP->PackageFileCount++;
 351
 352    if (CurrentFile->FileName == 0)
 353       return false;
 354
 355    Progress.SubProgress(Buf.st_size);
 356    return true;
 357 }
 358                                                                         /*}}}*/
 359 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* This is used to create handles to strings. Given the same text it
 362    always returns the same number */
 363 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 364                                                  unsigned int Size)
 365 {
 366    /* We use a very small transient hash table here, this speeds up generation
 367       by a fair amount on slower machines */
 368    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 369    if (Bucket != 0 &&
 370        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 371       return Bucket->String;
 372
 373    // Search for an insertion point
 374    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 375    int Res = 1;
 376    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 377    for (; I != Cache.StringItemP; Last = &I->NextItem,
 378         I = Cache.StringItemP + I->NextItem)
 379    {
 380       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 381       if (Res >= 0)
 382          break;
 383    }
 384
 385    // Match
 386    if (Res == 0)
 387    {
 388       Bucket = I;
 389       return I->String;
 390    }
 391
 392    // Get a structure
 393    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 394    if (Item == 0)
 395       return 0;
 396
 397    // Fill in the structure
 398    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 399    ItemP->NextItem = I - Cache.StringItemP;
 400    *Last = Item;
 401    ItemP->String = Map.WriteString(S,Size);
 402    if (ItemP->String == 0)
 403       return 0;
 404
 405    Bucket = ItemP;
 406    return ItemP->String;
 407 }
 408                                                                         /*}}}*/
 409
 410 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 411 // ---------------------------------------------------------------------
 412 /* The source cache is checked against the source list and the files
 413    on disk, any difference results in a false. */
 414 bool pkgSrcCacheCheck(pkgSourceList &List)
 415 {
 416    if (_error->PendingError() == true)
 417       return false;
 418
 419    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 420    string ListDir = _config->FindDir("Dir::State::lists");
 421
 422    // Count the number of missing files
 423    int Missing = 0;
 424    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 425    {
 426       // Only cache deb source types.
 427       if (I->Type != pkgSourceList::Item::Deb)
 428       {
 429          Missing++;
 430          continue;
 431       }
 432
 433       string File = ListDir + URItoFileName(I->PackagesURI());
 434       struct stat Buf;
 435       if (stat(File.c_str(),&Buf) != 0)
 436       {
 437          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 438                           I->PackagesInfo().c_str(),File.c_str());
 439          Missing++;
 440       }
 441    }
 442
 443    // Open the source package cache
 444    if (FileExists(CacheFile) == false)
 445       return false;
 446
 447    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 448    if (_error->PendingError() == true)
 449    {
 450       _error->Discard();
 451       return false;
 452    }
 453
 454    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 455    if (_error->PendingError() == true || Map.Size() == 0)
 456    {
 457       _error->Discard();
 458       return false;
 459    }
 460
 461    pkgCache Cache(Map);
 462    if (_error->PendingError() == true)
 463    {
 464       _error->Discard();
 465       return false;
 466    }
 467
 468    // They are certianly out of sync
 469    if (Cache.Head().PackageFileCount != List.size() - Missing)
 470       return false;
 471
 472    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 473    {
 474       // Search for a match in the source list
 475       bool Bad = true;
 476       for (pkgSourceList::const_iterator I = List.begin();
 477            I != List.end(); I++)
 478       {
 479          // Only cache deb source types.
 480          if (I->Type != pkgSourceList::Item::Deb)
 481             continue;
 482
 483          string File = ListDir + URItoFileName(I->PackagesURI());
 484          if (F.FileName() == File)
 485          {
 486             Bad = false;
 487             break;
 488          }
 489       }
 490
 491       // Check if the file matches what was cached
 492       Bad |= !F.IsOk();
 493       if (Bad == true)
 494          return false;
 495    }
 496
 497    return true;
 498 }
 499                                                                         /*}}}*/
 500 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* This does a simple check of all files used to compose the cache */
 503 bool pkgPkgCacheCheck(string CacheFile)
 504 {
 505    if (_error->PendingError() == true)
 506       return false;
 507
 508    // Open the source package cache
 509    if (FileExists(CacheFile) == false)
 510       return false;
 511
 512    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 513    if (_error->PendingError() == true)
 514    {
 515       _error->Discard();
 516       return false;
 517    }
 518
 519    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 520    if (_error->PendingError() == true || Map.Size() == 0)
 521    {
 522       _error->Discard();
 523       return false;
 524    }
 525
 526    pkgCache Cache(Map);
 527    if (_error->PendingError() == true)
 528    {
 529       _error->Discard();
 530       return false;
 531    }
 532
 533    // Status files that must be in the cache
 534    string Status[3];
 535    Status[0] = _config->FindFile("Dir::State::xstatus");
 536    Status[1]= _config->FindFile("Dir::State::userstatus");
 537    Status[2] = _config->FindFile("Dir::State::status");
 538
 539    // Cheack each file
 540    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 541    {
 542       if (F.IsOk() == false)
 543          return false;
 544
 545       // See if this is one of the status files
 546       for (int I = 0; I != 3; I++)
 547          if (F.FileName() == Status[I])
 548             Status[I] = string();
 549    }
 550
 551    // Make sure all the status files are loaded.
 552    for (int I = 0; I != 3; I++)
 553    {
 554       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 555          return false;
 556    }
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 562 // ---------------------------------------------------------------------
 563 /* This adds the size of all the status files to the size counter */
 564 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 565 {
 566    // Grab the file names
 567    string xstatus = _config->FindFile("Dir::State::xstatus");
 568    string userstatus = _config->FindFile("Dir::State::userstatus");
 569    string status = _config->FindFile("Dir::State::status");
 570
 571    // Grab the sizes
 572    struct stat Buf;
 573    if (stat(xstatus.c_str(),&Buf) == 0)
 574       TotalSize += Buf.st_size;
 575    if (stat(userstatus.c_str(),&Buf) == 0)
 576       TotalSize += Buf.st_size;
 577    if (stat(status.c_str(),&Buf) != 0)
 578       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 579    TotalSize += Buf.st_size;
 580
 581    return true;
 582 }
 583                                                                         /*}}}*/
 584 // MergeStatus - Add the status files to the cache                      /*{{{*/
 585 // ---------------------------------------------------------------------
 586 /* This adds the status files to the map */
 587 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 588                            unsigned long &CurrentSize,unsigned long TotalSize)
 589 {
 590    // Grab the file names
 591    string Status[3];
 592    Status[0] = _config->FindFile("Dir::State::xstatus");
 593    Status[1]= _config->FindFile("Dir::State::userstatus");
 594    Status[2] = _config->FindFile("Dir::State::status");
 595
 596    for (int I = 0; I != 3; I++)
 597    {
 598       // Check if the file exists and it is not the primary status file.
 599       string File = Status[I];
 600       if (I != 2 && FileExists(File) == false)
 601          continue;
 602
 603       FileFd Pkg(File,FileFd::ReadOnly);
 604       debListParser Parser(Pkg);
 605       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 606       if (_error->PendingError() == true)
 607          return _error->Error("Problem opening %s",File.c_str());
 608       CurrentSize += Pkg.Size();
 609
 610       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 611       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 612          return _error->Error("Problem with SelectFile %s",File.c_str());
 613
 614       if (Gen.MergeList(Parser) == false)
 615          return _error->Error("Problem with MergeList %s",File.c_str());
 616       Progress.Progress(Pkg.Size());
 617    }
 618
 619    return true;
 620 }
 621                                                                         /*}}}*/
 622 // GenerateSrcCache - Write the source package lists to the map         /*{{{*/
 623 // ---------------------------------------------------------------------
 624 /* This puts the source package cache into the given generator. */
 625 bool pkgGenerateSrcCache(pkgSourceList &List,OpProgress &Progress,
 626                          pkgCacheGenerator &Gen,
 627                          unsigned long &CurrentSize,unsigned long &TotalSize)
 628 {
 629    string ListDir = _config->FindDir("Dir::State::lists");
 630
 631    // Prepare the progress indicator
 632    TotalSize = 0;
 633    struct stat Buf;
 634    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 635    {
 636       string File = ListDir + URItoFileName(I->PackagesURI());
 637       if (stat(File.c_str(),&Buf) != 0)
 638          continue;
 639       TotalSize += Buf.st_size;
 640    }
 641
 642    if (pkgAddSourcesSize(TotalSize) == false)
 643       return false;
 644
 645    // Generate the pkg source cache
 646    CurrentSize = 0;
 647    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 648    {
 649       // Only cache deb source types.
 650       if (I->Type != pkgSourceList::Item::Deb)
 651          continue;
 652
 653       string File = ListDir + URItoFileName(I->PackagesURI());
 654
 655       if (FileExists(File) == false)
 656          continue;
 657
 658       FileFd Pkg(File,FileFd::ReadOnly);
 659       debListParser Parser(Pkg);
 660       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 661       if (_error->PendingError() == true)
 662          return _error->Error("Problem opening %s",File.c_str());
 663       CurrentSize += Pkg.Size();
 664
 665       Progress.SubProgress(0,I->PackagesInfo());
 666       if (Gen.SelectFile(File) == false)
 667          return _error->Error("Problem with SelectFile %s",File.c_str());
 668
 669       if (Gen.MergeList(Parser) == false)
 670          return _error->Error("Problem with MergeList %s",File.c_str());
 671
 672       // Check the release file
 673       string RFile = ListDir + URItoFileName(I->ReleaseURI());
 674       if (FileExists(RFile) == true)
 675       {
 676          FileFd Rel(RFile,FileFd::ReadOnly);
 677          if (_error->PendingError() == true)
 678             return false;
 679          Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 680       }
 681    }
 682
 683    return true;
 684 }
 685                                                                         /*}}}*/
 686 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 687 // ---------------------------------------------------------------------
 688 /* This copies the package source cache and then merges the status and
 689    xstatus files into it. */
 690 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 691 {
 692    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 693
 694    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 695    bool SrcOk = pkgSrcCacheCheck(List);
 696    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 697
 698    // Rebuild the source and package caches
 699    if (SrcOk == false)
 700    {
 701       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 702       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 703       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 704       DynamicMMap Map(CacheF,MMap::Public);
 705       if (_error->PendingError() == true)
 706          return false;
 707
 708       pkgCacheGenerator Gen(Map,Progress);
 709       unsigned long CurrentSize = 0;
 710       unsigned long TotalSize = 0;
 711       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 712          return false;
 713
 714       // Write the src cache
 715       Gen.GetCache().HeaderP->Dirty = false;
 716       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 717          return _error->Error("IO Error saving source cache");
 718       Gen.GetCache().HeaderP->Dirty = true;
 719
 720       // Merge in the source caches
 721       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 722    }
 723
 724    if (PkgOk == true)
 725    {
 726       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 727       return true;
 728    }
 729
 730    // We use the source cache to generate the package cache
 731    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 732
 733    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 734    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 735    DynamicMMap Map(CacheF,MMap::Public);
 736    if (_error->PendingError() == true)
 737       return false;
 738
 739    // Preload the map with the source cache
 740    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 741                     SCacheF.Size()) == false)
 742       return false;
 743
 744    pkgCacheGenerator Gen(Map,Progress);
 745
 746    // Compute the progress
 747    unsigned long TotalSize = 0;
 748    if (pkgAddSourcesSize(TotalSize) == false)
 749       return false;
 750
 751    unsigned long CurrentSize = 0;
 752    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 753 }
 754                                                                         /*}}}*/
 755 // MakeStatusCacheMem - Returns a map for the status cache              /*{{{*/
 756 // ---------------------------------------------------------------------
 757 /* This creates a map object for the status cache. If the process has write
 758    access to the caches then it is the same as MakeStatusCache, otherwise it
 759    creates a memory block and puts the cache in there. */
 760 MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress)
 761 {
 762    /* If the cache file is writeable this is just a wrapper for
 763       MakeStatusCache */
 764    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 765    bool Writeable = (access(CacheFile.c_str(),W_OK) == 0) ||
 766                     (errno == ENOENT);
 767
 768    if (Writeable == true)
 769    {
 770       if (pkgMakeStatusCache(List,Progress) == false)
 771          return 0;
 772
 773       // Open the cache file
 774       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 775       if (_error->PendingError() == true)
 776          return 0;
 777
 778       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 779       if (_error->PendingError() == true)
 780       {
 781          delete Map;
 782          return 0;
 783       }
 784       return Map;
 785    }
 786
 787    // Mostly from MakeStatusCache..
 788    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 789
 790    bool SrcOk = pkgSrcCacheCheck(List);
 791    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 792
 793    // Rebuild the source and package caches
 794    if (SrcOk == false)
 795    {
 796       DynamicMMap *Map = new DynamicMMap(MMap::Public);
 797       if (_error->PendingError() == true)
 798       {
 799          delete Map;
 800          return 0;
 801       }
 802
 803       pkgCacheGenerator Gen(*Map,Progress);
 804       unsigned long CurrentSize = 0;
 805       unsigned long TotalSize = 0;
 806       if (pkgGenerateSrcCache(List,Progress,Gen,CurrentSize,TotalSize) == false)
 807       {
 808          delete Map;
 809          return 0;
 810       }
 811
 812       // Merge in the source caches
 813       if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 814       {
 815          delete Map;
 816          return 0;
 817       }
 818
 819       return Map;
 820    }
 821
 822    if (PkgOk == true)
 823    {
 824       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 825
 826       // Open the cache file
 827       FileFd File(_config->FindFile("Dir::Cache::pkgcache"),FileFd::ReadOnly);
 828       if (_error->PendingError() == true)
 829          return 0;
 830
 831       MMap *Map = new MMap(File,MMap::Public | MMap::ReadOnly);
 832       if (_error->PendingError() == true)
 833       {
 834          delete Map;
 835          return 0;
 836       }
 837       return Map;
 838    }
 839
 840    // We use the source cache to generate the package cache
 841    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 842    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 843    DynamicMMap *Map = new DynamicMMap(MMap::Public);
 844    if (_error->PendingError() == true)
 845    {
 846       delete Map;
 847       return 0;
 848    }
 849
 850    // Preload the map with the source cache
 851    if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 852                     SCacheF.Size()) == false)
 853    {
 854       delete Map;
 855       return 0;
 856    }
 857
 858    pkgCacheGenerator Gen(*Map,Progress);
 859
 860    // Compute the progress
 861    unsigned long TotalSize = 0;
 862    if (pkgAddSourcesSize(TotalSize) == false)
 863    {
 864       delete Map;
 865       return 0;
 866    }
 867
 868    unsigned long CurrentSize = 0;
 869    if (pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize) == false)
 870    {
 871       delete Map;
 872       return 0;
 873    }
 874
 875    return Map;
 876 }
 877                                                                         /*}}}*/