apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53 2003/02/02 02:44:20 doogie Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37 #include <system.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       Map.RawAllocate(sizeof(pkgCache::Header));
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 129             return _error->Error(_("Error occured while processing %s (UsePackage1)"),
 130                                  PackageName.c_str());
 131          continue;
 132       }
 133
 134       pkgCache::VerIterator Ver = Pkg.VersionList();
 135       map_ptrloc *Last = &Pkg->VersionList;
 136       int Res = 1;
 137       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 138       {
 139          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 140          if (Res >= 0)
 141             break;
 142       }
 143
 144       /* We already have a version for this item, record that we
 145          saw it */
 146       unsigned long Hash = List.VersionHash();
 147       if (Res == 0 && Ver->Hash == Hash)
 148       {
 149          if (List.UsePackage(Pkg,Ver) == false)
 150             return _error->Error(_("Error occured while processing %s (UsePackage2)"),
 151                                  PackageName.c_str());
 152
 153          if (NewFileVer(Ver,List) == false)
 154             return _error->Error(_("Error occured while processing %s (NewFileVer1)"),
 155                                  PackageName.c_str());
 156
 157          // Read only a single record and return
 158          if (OutVer != 0)
 159          {
 160             *OutVer = Ver;
 161             FoundFileDeps |= List.HasFileDeps();
 162             return true;
 163          }
 164
 165          continue;
 166       }
 167
 168       // Skip to the end of the same version set.
 169       if (Res == 0)
 170       {
 171          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 172          {
 173             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 174             if (Res != 0)
 175                break;
 176          }
 177       }
 178
 179       // Add a new version
 180       *Last = NewVersion(Ver,Version,*Last);
 181       Ver->ParentPkg = Pkg.Index();
 182       Ver->Hash = Hash;
 183       if (List.NewVersion(Ver) == false)
 184          return _error->Error(_("Error occured while processing %s (NewVersion1)"),
 185                               PackageName.c_str());
 186
 187       if (List.UsePackage(Pkg,Ver) == false)
 188          return _error->Error(_("Error occured while processing %s (UsePackage3)"),
 189                               PackageName.c_str());
 190
 191       if (NewFileVer(Ver,List) == false)
 192          return _error->Error(_("Error occured while processing %s (NewVersion2)"),
 193                               PackageName.c_str());
 194
 195       // Read only a single record and return
 196       if (OutVer != 0)
 197       {
 198          *OutVer = Ver;
 199          FoundFileDeps |= List.HasFileDeps();
 200          return true;
 201       }
 202    }
 203
 204    FoundFileDeps |= List.HasFileDeps();
 205
 206    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 207       return _error->Error(_("Wow, you exceeded the number of package "
 208                              "names this APT is capable of."));
 209    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 210       return _error->Error(_("Wow, you exceeded the number of versions "
 211                              "this APT is capable of."));
 212    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 213       return _error->Error(_("Wow, you exceeded the number of dependencies "
 214                              "this APT is capable of."));
 215    return true;
 216 }
 217                                                                         /*}}}*/
 218 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 219 // ---------------------------------------------------------------------
 220 /* If we found any file depends while parsing the main list we need to
 221    resolve them. Since it is undesired to load the entire list of files
 222    into the cache as virtual packages we do a two stage effort. MergeList
 223    identifies the file depends and this creates Provdies for them by
 224    re-parsing all the indexs. */
 225 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 226 {
 227    List.Owner = this;
 228
 229    unsigned int Counter = 0;
 230    while (List.Step() == true)
 231    {
 232       string PackageName = List.Package();
 233       if (PackageName.empty() == true)
 234          return false;
 235       string Version = List.Version();
 236       if (Version.empty() == true)
 237          continue;
 238
 239       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 240       if (Pkg.end() == true)
 241          return _error->Error(_("Error occured while processing %s (FindPkg)"),
 242                                 PackageName.c_str());
 243       Counter++;
 244       if (Counter % 100 == 0 && Progress != 0)
 245          Progress->Progress(List.Offset());
 246
 247       unsigned long Hash = List.VersionHash();
 248       pkgCache::VerIterator Ver = Pkg.VersionList();
 249       for (; Ver.end() == false; Ver++)
 250       {
 251          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 252          {
 253             if (List.CollectFileProvides(Cache,Ver) == false)
 254                return _error->Error(_("Error occured while processing %s (CollectFileProvides)"),PackageName.c_str());
 255             break;
 256          }
 257       }
 258
 259       if (Ver.end() == true)
 260          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 261    }
 262
 263    return true;
 264 }
 265                                                                         /*}}}*/
 266 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 267 // ---------------------------------------------------------------------
 268 /* This creates a new package structure and adds it to the hash table */
 269 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 270 {
 271    Pkg = Cache.FindPkg(Name);
 272    if (Pkg.end() == false)
 273       return true;
 274
 275    // Get a structure
 276    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 277    if (Package == 0)
 278       return false;
 279
 280    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 281
 282    // Insert it into the hash table
 283    unsigned long Hash = Cache.Hash(Name);
 284    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 285    Cache.HeaderP->HashTable[Hash] = Package;
 286
 287    // Set the name and the ID
 288    Pkg->Name = Map.WriteString(Name);
 289    if (Pkg->Name == 0)
 290       return false;
 291    Pkg->ID = Cache.HeaderP->PackageCount++;
 292
 293    return true;
 294 }
 295                                                                         /*}}}*/
 296 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 297 // ---------------------------------------------------------------------
 298 /* */
 299 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 300                                    ListParser &List)
 301 {
 302    if (CurrentFile == 0)
 303       return true;
 304
 305    // Get a structure
 306    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 307    if (VerFile == 0)
 308       return 0;
 309
 310    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 311    VF->File = CurrentFile - Cache.PkgFileP;
 312
 313    // Link it to the end of the list
 314    map_ptrloc *Last = &Ver->FileList;
 315    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 316       Last = &V->NextFile;
 317    VF->NextFile = *Last;
 318    *Last = VF.Index();
 319
 320    VF->Offset = List.Offset();
 321    VF->Size = List.Size();
 322    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 323       Cache.HeaderP->MaxVerFileSize = VF->Size;
 324    Cache.HeaderP->VerFileCount++;
 325
 326    return true;
 327 }
 328                                                                         /*}}}*/
 329 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 330 // ---------------------------------------------------------------------
 331 /* This puts a version structure in the linked list */
 332 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 333                                             string VerStr,
 334                                             unsigned long Next)
 335 {
 336    // Get a structure
 337    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 338    if (Version == 0)
 339       return 0;
 340
 341    // Fill it in
 342    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 343    Ver->NextVer = Next;
 344    Ver->ID = Cache.HeaderP->VersionCount++;
 345    Ver->VerStr = Map.WriteString(VerStr);
 346    if (Ver->VerStr == 0)
 347       return 0;
 348
 349    return Version;
 350 }
 351                                                                         /*}}}*/
 352 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 353 // ---------------------------------------------------------------------
 354 /* This creates a dependency element in the tree. It is linked to the
 355    version and to the package that it is pointing to. */
 356 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 357                                                string PackageName,
 358                                                string Version,
 359                                                unsigned int Op,
 360                                                unsigned int Type)
 361 {
 362    pkgCache &Cache = Owner->Cache;
 363
 364    // Get a structure
 365    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 366    if (Dependency == 0)
 367       return false;
 368
 369    // Fill it in
 370    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 371    Dep->ParentVer = Ver.Index();
 372    Dep->Type = Type;
 373    Dep->CompareOp = Op;
 374    Dep->ID = Cache.HeaderP->DependsCount++;
 375
 376    // Locate the target package
 377    pkgCache::PkgIterator Pkg;
 378    if (Owner->NewPackage(Pkg,PackageName) == false)
 379       return false;
 380
 381    // Probe the reverse dependency list for a version string that matches
 382    if (Version.empty() == false)
 383    {
 384 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 385          if (I->Version != 0 && I.TargetVer() == Version)
 386             Dep->Version = I->Version;*/
 387       if (Dep->Version == 0)
 388          if ((Dep->Version = WriteString(Version)) == 0)
 389             return false;
 390    }
 391
 392    // Link it to the package
 393    Dep->Package = Pkg.Index();
 394    Dep->NextRevDepends = Pkg->RevDepends;
 395    Pkg->RevDepends = Dep.Index();
 396
 397    /* Link it to the version (at the end of the list)
 398       Caching the old end point speeds up generation substantially */
 399    if (OldDepVer != Ver)
 400    {
 401       OldDepLast = &Ver->DependsList;
 402       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 403          OldDepLast = &D->NextDepends;
 404       OldDepVer = Ver;
 405    }
 406
 407    // Is it a file dependency?
 408    if (PackageName[0] == '/')
 409       FoundFileDeps = true;
 410
 411    Dep->NextDepends = *OldDepLast;
 412    *OldDepLast = Dep.Index();
 413    OldDepLast = &Dep->NextDepends;
 414
 415    return true;
 416 }
 417                                                                         /*}}}*/
 418 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 419 // ---------------------------------------------------------------------
 420 /* */
 421 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 422                                                 string PackageName,
 423                                                 string Version)
 424 {
 425    pkgCache &Cache = Owner->Cache;
 426
 427    // We do not add self referencing provides
 428    if (Ver.ParentPkg().Name() == PackageName)
 429       return true;
 430
 431    // Get a structure
 432    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 433    if (Provides == 0)
 434       return false;
 435    Cache.HeaderP->ProvidesCount++;
 436
 437    // Fill it in
 438    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 439    Prv->Version = Ver.Index();
 440    Prv->NextPkgProv = Ver->ProvidesList;
 441    Ver->ProvidesList = Prv.Index();
 442    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 443       return false;
 444
 445    // Locate the target package
 446    pkgCache::PkgIterator Pkg;
 447    if (Owner->NewPackage(Pkg,PackageName) == false)
 448       return false;
 449
 450    // Link it to the package
 451    Prv->ParentPkg = Pkg.Index();
 452    Prv->NextProvides = Pkg->ProvidesList;
 453    Pkg->ProvidesList = Prv.Index();
 454
 455    return true;
 456 }
 457                                                                         /*}}}*/
 458 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* This is used to select which file is to be associated with all newly
 461    added versions. The caller is responsible for setting the IMS fields. */
 462 bool pkgCacheGenerator::SelectFile(string File,string Site,
 463                                    const pkgIndexFile &Index,
 464                                    unsigned long Flags)
 465 {
 466    // Get some space for the structure
 467    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 468    if (CurrentFile == Cache.PkgFileP)
 469       return false;
 470
 471    // Fill it in
 472    CurrentFile->FileName = Map.WriteString(File);
 473    CurrentFile->Site = WriteUniqString(Site);
 474    CurrentFile->NextFile = Cache.HeaderP->FileList;
 475    CurrentFile->Flags = Flags;
 476    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 477    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 478    PkgFileName = File;
 479    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 480    Cache.HeaderP->PackageFileCount++;
 481
 482    if (CurrentFile->FileName == 0)
 483       return false;
 484
 485    if (Progress != 0)
 486       Progress->SubProgress(Index.Size());
 487    return true;
 488 }
 489                                                                         /*}}}*/
 490 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 491 // ---------------------------------------------------------------------
 492 /* This is used to create handles to strings. Given the same text it
 493    always returns the same number */
 494 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 495                                                  unsigned int Size)
 496 {
 497    /* We use a very small transient hash table here, this speeds up generation
 498       by a fair amount on slower machines */
 499    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 500    if (Bucket != 0 &&
 501        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 502       return Bucket->String;
 503
 504    // Search for an insertion point
 505    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 506    int Res = 1;
 507    map_ptrloc *Last = &Cache.HeaderP->StringList;
 508    for (; I != Cache.StringItemP; Last = &I->NextItem,
 509         I = Cache.StringItemP + I->NextItem)
 510    {
 511       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 512       if (Res >= 0)
 513          break;
 514    }
 515
 516    // Match
 517    if (Res == 0)
 518    {
 519       Bucket = I;
 520       return I->String;
 521    }
 522
 523    // Get a structure
 524    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 525    if (Item == 0)
 526       return 0;
 527
 528    // Fill in the structure
 529    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 530    ItemP->NextItem = I - Cache.StringItemP;
 531    *Last = Item;
 532    ItemP->String = Map.WriteString(S,Size);
 533    if (ItemP->String == 0)
 534       return 0;
 535
 536    Bucket = ItemP;
 537    return ItemP->String;
 538 }
 539                                                                         /*}}}*/
 540
 541 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 542 // ---------------------------------------------------------------------
 543 /* This just verifies that each file in the list of index files exists,
 544    has matching attributes with the cache and the cache does not have
 545    any extra files. */
 546 static bool CheckValidity(string CacheFile, FileIterator Start,
 547                           FileIterator End,MMap **OutMap = 0)
 548 {
 549    // No file, certainly invalid
 550    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 551       return false;
 552
 553    // Map it
 554    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 555    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 556    pkgCache Cache(Map);
 557    if (_error->PendingError() == true || Map->Size() == 0)
 558    {
 559       _error->Discard();
 560       return false;
 561    }
 562
 563    /* Now we check every index file, see if it is in the cache,
 564       verify the IMS data and check that it is on the disk too.. */
 565    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 566    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 567    for (; Start != End; Start++)
 568    {
 569       if ((*Start)->HasPackages() == false)
 570          continue;
 571
 572       if ((*Start)->Exists() == false)
 573       {
 574          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 575                           (*Start)->Describe().c_str());
 576          continue;
 577       }
 578
 579       // FindInCache is also expected to do an IMS check.
 580       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 581       if (File.end() == true)
 582          return false;
 583
 584       Visited[File->ID] = true;
 585    }
 586
 587    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 588       if (Visited[I] == false)
 589          return false;
 590
 591    if (_error->PendingError() == true)
 592    {
 593       _error->Discard();
 594       return false;
 595    }
 596
 597    if (OutMap != 0)
 598       *OutMap = Map.UnGuard();
 599    return true;
 600 }
 601                                                                         /*}}}*/
 602 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* Size is kind of an abstract notion that is only used for the progress
 605    meter */
 606 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 607 {
 608    unsigned long TotalSize = 0;
 609    for (; Start != End; Start++)
 610    {
 611       if ((*Start)->HasPackages() == false)
 612          continue;
 613       TotalSize += (*Start)->Size();
 614    }
 615    return TotalSize;
 616 }
 617                                                                         /*}}}*/
 618 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 619 // ---------------------------------------------------------------------
 620 /* */
 621 static bool BuildCache(pkgCacheGenerator &Gen,
 622                        OpProgress &Progress,
 623                        unsigned long &CurrentSize,unsigned long TotalSize,
 624                        FileIterator Start, FileIterator End)
 625 {
 626    FileIterator I;
 627    for (I = Start; I != End; I++)
 628    {
 629       if ((*I)->HasPackages() == false)
 630          continue;
 631
 632       if ((*I)->Exists() == false)
 633          continue;
 634
 635       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 636       {
 637          _error->Warning("Duplicate sources.list entry %s",
 638                          (*I)->Describe().c_str());
 639          continue;
 640       }
 641
 642       unsigned long Size = (*I)->Size();
 643       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading Package Lists"));
 644       CurrentSize += Size;
 645
 646       if ((*I)->Merge(Gen,Progress) == false)
 647          return false;
 648    }
 649
 650    if (Gen.HasFileDeps() == true)
 651    {
 652       Progress.Done();
 653       TotalSize = ComputeSize(Start, End);
 654       CurrentSize = 0;
 655       for (I = Start; I != End; I++)
 656       {
 657          unsigned long Size = (*I)->Size();
 658          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 659          CurrentSize += Size;
 660          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 661             return false;
 662       }
 663    }
 664
 665    return true;
 666 }
 667                                                                         /*}}}*/
 668 // MakeStatusCache - Construct the status cache                         /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This makes sure that the status cache (the cache that has all
 671    index files from the sources list and all local ones) is ready
 672    to be mmaped. If OutMap is not zero then a MMap object representing
 673    the cache will be stored there. This is pretty much mandetory if you
 674    are using AllowMem. AllowMem lets the function be run as non-root
 675    where it builds the cache 'fast' into a memory buffer. */
 676 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 677                         MMap **OutMap,bool AllowMem)
 678 {
 679    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 680
 681    vector<pkgIndexFile *> Files(List.begin(),List.end());
 682    unsigned long EndOfSource = Files.size();
 683    if (_system->AddStatusFiles(Files) == false)
 684       return false;
 685
 686    // Decide if we can write to the files..
 687    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 688    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 689
 690    // Decide if we can write to the cache
 691    bool Writeable = false;
 692    if (CacheFile.empty() == false)
 693       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 694    else
 695       if (SrcCacheFile.empty() == false)
 696          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 697
 698    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 699       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 700
 701    Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
 702
 703    // Cache is OK, Fin.
 704    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 705    {
 706       Progress.OverallProgress(1,1,1,_("Reading Package Lists"));
 707       return true;
 708    }
 709
 710    /* At this point we know we need to reconstruct the package cache,
 711       begin. */
 712    SPtr<FileFd> CacheF;
 713    SPtr<DynamicMMap> Map;
 714    if (Writeable == true && CacheFile.empty() == false)
 715    {
 716       unlink(CacheFile.c_str());
 717       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 718       fchmod(CacheF->Fd(),0644);
 719       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 720       if (_error->PendingError() == true)
 721          return false;
 722    }
 723    else
 724    {
 725       // Just build it in memory..
 726       Map = new DynamicMMap(MMap::Public,MapSize);
 727    }
 728
 729    // Lets try the source cache.
 730    unsigned long CurrentSize = 0;
 731    unsigned long TotalSize = 0;
 732    if (CheckValidity(SrcCacheFile,Files.begin(),
 733                      Files.begin()+EndOfSource) == true)
 734    {
 735       // Preload the map with the source cache
 736       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 737       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 738                        SCacheF.Size()) == false)
 739          return false;
 740
 741       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 742
 743       // Build the status cache
 744       pkgCacheGenerator Gen(Map.Get(),&Progress);
 745       if (_error->PendingError() == true)
 746          return false;
 747       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 748                      Files.begin()+EndOfSource,Files.end()) == false)
 749          return false;
 750    }
 751    else
 752    {
 753       TotalSize = ComputeSize(Files.begin(),Files.end());
 754
 755       // Build the source cache
 756       pkgCacheGenerator Gen(Map.Get(),&Progress);
 757       if (_error->PendingError() == true)
 758          return false;
 759       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 760                      Files.begin(),Files.begin()+EndOfSource) == false)
 761          return false;
 762
 763       // Write it back
 764       if (Writeable == true && SrcCacheFile.empty() == false)
 765       {
 766          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 767          if (_error->PendingError() == true)
 768             return false;
 769
 770          fchmod(SCacheF.Fd(),0644);
 771
 772          // Write out the main data
 773          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 774             return _error->Error(_("IO Error saving source cache"));
 775          SCacheF.Sync();
 776
 777          // Write out the proper header
 778          Gen.GetCache().HeaderP->Dirty = false;
 779          if (SCacheF.Seek(0) == false ||
 780              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 781             return _error->Error(_("IO Error saving source cache"));
 782          Gen.GetCache().HeaderP->Dirty = true;
 783          SCacheF.Sync();
 784       }
 785
 786       // Build the status cache
 787       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 788                      Files.begin()+EndOfSource,Files.end()) == false)
 789          return false;
 790    }
 791
 792    if (_error->PendingError() == true)
 793       return false;
 794    if (OutMap != 0)
 795    {
 796       if (CacheF != 0)
 797       {
 798          delete Map.UnGuard();
 799          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 800       }
 801       else
 802       {
 803          *OutMap = Map.UnGuard();
 804       }
 805    }
 806
 807    return true;
 808 }
 809                                                                         /*}}}*/
 810 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 811 // ---------------------------------------------------------------------
 812 /* */
 813 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 814 {
 815    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 816    vector<pkgIndexFile *> Files;
 817    unsigned long EndOfSource = Files.size();
 818    if (_system->AddStatusFiles(Files) == false)
 819       return false;
 820
 821    SPtr<DynamicMMap> Map;
 822    Map = new DynamicMMap(MMap::Public,MapSize);
 823    unsigned long CurrentSize = 0;
 824    unsigned long TotalSize = 0;
 825
 826    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 827
 828    // Build the status cache
 829    Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
 830    pkgCacheGenerator Gen(Map.Get(),&Progress);
 831    if (_error->PendingError() == true)
 832       return false;
 833    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 834                   Files.begin()+EndOfSource,Files.end()) == false)
 835       return false;
 836
 837    if (_error->PendingError() == true)
 838       return false;
 839    *OutMap = Map.UnGuard();
 840
 841    return true;
 842 }
 843                                                                         /*}}}*/