apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37 #include <system.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       Map.RawAllocate(sizeof(pkgCache::Header));
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 129             return _error->Error(_("Error occured while processing %s (UsePackage1)"),
 130                                  PackageName.c_str());
 131          continue;
 132       }
 133
 134       pkgCache::VerIterator Ver = Pkg.VersionList();
 135       map_ptrloc *Last = &Pkg->VersionList;
 136       int Res = 1;
 137       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 138       {
 139          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 140          if (Res >= 0)
 141             break;
 142       }
 143
 144       /* We already have a version for this item, record that we
 145          saw it */
 146       unsigned long Hash = List.VersionHash();
 147       if (Res == 0 && Ver->Hash == Hash)
 148       {
 149          if (List.UsePackage(Pkg,Ver) == false)
 150             return _error->Error(_("Error occured while processing %s (UsePackage2)"),
 151                                  PackageName.c_str());
 152
 153          if (NewFileVer(Ver,List) == false)
 154             return _error->Error(_("Error occured while processing %s (NewFileVer1)"),
 155                                  PackageName.c_str());
 156
 157          // Read only a single record and return
 158          if (OutVer != 0)
 159          {
 160             *OutVer = Ver;
 161             FoundFileDeps |= List.HasFileDeps();
 162             return true;
 163          }
 164
 165          continue;
 166       }
 167
 168       // Skip to the end of the same version set.
 169       if (Res == 0)
 170       {
 171          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 172          {
 173             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 174             if (Res != 0)
 175                break;
 176          }
 177       }
 178
 179       // Add a new version
 180       *Last = NewVersion(Ver,Version,*Last);
 181       Ver->ParentPkg = Pkg.Index();
 182       Ver->Hash = Hash;
 183       if (List.NewVersion(Ver) == false)
 184          return _error->Error(_("Error occured while processing %s (NewVersion1)"),
 185                               PackageName.c_str());
 186
 187       if (List.UsePackage(Pkg,Ver) == false)
 188          return _error->Error(_("Error occured while processing %s (UsePackage3)"),
 189                               PackageName.c_str());
 190
 191       if (NewFileVer(Ver,List) == false)
 192          return _error->Error(_("Error occured while processing %s (NewVersion2)"),
 193                               PackageName.c_str());
 194
 195       // Read only a single record and return
 196       if (OutVer != 0)
 197       {
 198          *OutVer = Ver;
 199          FoundFileDeps |= List.HasFileDeps();
 200          return true;
 201       }
 202    }
 203
 204    FoundFileDeps |= List.HasFileDeps();
 205
 206    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 207       return _error->Error(_("Wow, you exceeded the number of package "
 208                              "names this APT is capable of."));
 209    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 210       return _error->Error(_("Wow, you exceeded the number of versions "
 211                              "this APT is capable of."));
 212    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 213       return _error->Error(_("Wow, you exceeded the number of dependencies "
 214                              "this APT is capable of."));
 215    return true;
 216 }
 217                                                                         /*}}}*/
 218 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 219 // ---------------------------------------------------------------------
 220 /* If we found any file depends while parsing the main list we need to
 221    resolve them. Since it is undesired to load the entire list of files
 222    into the cache as virtual packages we do a two stage effort. MergeList
 223    identifies the file depends and this creates Provdies for them by
 224    re-parsing all the indexs. */
 225 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 226 {
 227    List.Owner = this;
 228
 229    unsigned int Counter = 0;
 230    while (List.Step() == true)
 231    {
 232       string PackageName = List.Package();
 233       if (PackageName.empty() == true)
 234          return false;
 235       string Version = List.Version();
 236       if (Version.empty() == true)
 237          continue;
 238
 239       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 240       if (Pkg.end() == true)
 241          return _error->Error(_("Error occured while processing %s (FindPkg)"),
 242                                 PackageName.c_str());
 243       Counter++;
 244       if (Counter % 100 == 0 && Progress != 0)
 245          Progress->Progress(List.Offset());
 246
 247       unsigned long Hash = List.VersionHash();
 248       pkgCache::VerIterator Ver = Pkg.VersionList();
 249       for (; Ver.end() == false; Ver++)
 250       {
 251          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 252          {
 253             if (List.CollectFileProvides(Cache,Ver) == false)
 254                return _error->Error(_("Error occured while processing %s (CollectFileProvides)"),PackageName.c_str());
 255             break;
 256          }
 257       }
 258
 259       if (Ver.end() == true)
 260          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 261    }
 262
 263    return true;
 264 }
 265                                                                         /*}}}*/
 266 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 267 // ---------------------------------------------------------------------
 268 /* This creates a new package structure and adds it to the hash table */
 269 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 270 {
 271    Pkg = Cache.FindPkg(Name);
 272    if (Pkg.end() == false)
 273       return true;
 274
 275    // Get a structure
 276    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 277    if (Package == 0)
 278       return false;
 279
 280    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 281
 282    // Insert it into the hash table
 283    unsigned long Hash = Cache.Hash(Name);
 284    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 285    Cache.HeaderP->HashTable[Hash] = Package;
 286
 287    // Set the name and the ID
 288    Pkg->Name = Map.WriteString(Name);
 289    if (Pkg->Name == 0)
 290       return false;
 291    Pkg->ID = Cache.HeaderP->PackageCount++;
 292
 293    return true;
 294 }
 295                                                                         /*}}}*/
 296 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 297 // ---------------------------------------------------------------------
 298 /* */
 299 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 300                                    ListParser &List)
 301 {
 302    if (CurrentFile == 0)
 303       return true;
 304
 305    // Get a structure
 306    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 307    if (VerFile == 0)
 308       return 0;
 309
 310    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 311    VF->File = CurrentFile - Cache.PkgFileP;
 312
 313    // Link it to the end of the list
 314    map_ptrloc *Last = &Ver->FileList;
 315    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 316       Last = &V->NextFile;
 317    VF->NextFile = *Last;
 318    *Last = VF.Index();
 319
 320    VF->Offset = List.Offset();
 321    VF->Size = List.Size();
 322    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 323       Cache.HeaderP->MaxVerFileSize = VF->Size;
 324    Cache.HeaderP->VerFileCount++;
 325
 326    return true;
 327 }
 328                                                                         /*}}}*/
 329 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 330 // ---------------------------------------------------------------------
 331 /* This puts a version structure in the linked list */
 332 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 333                                             string VerStr,
 334                                             unsigned long Next)
 335 {
 336    // Get a structure
 337    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 338    if (Version == 0)
 339       return 0;
 340
 341    // Fill it in
 342    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 343    Ver->NextVer = Next;
 344    Ver->ID = Cache.HeaderP->VersionCount++;
 345    Ver->VerStr = Map.WriteString(VerStr);
 346    if (Ver->VerStr == 0)
 347       return 0;
 348
 349    return Version;
 350 }
 351                                                                         /*}}}*/
 352 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 353 // ---------------------------------------------------------------------
 354 /* This creates a dependency element in the tree. It is linked to the
 355    version and to the package that it is pointing to. */
 356 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 357                                                string PackageName,
 358                                                string Version,
 359                                                unsigned int Op,
 360                                                unsigned int Type)
 361 {
 362    pkgCache &Cache = Owner->Cache;
 363
 364    // Get a structure
 365    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 366    if (Dependency == 0)
 367       return false;
 368
 369    // Fill it in
 370    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 371    Dep->ParentVer = Ver.Index();
 372    Dep->Type = Type;
 373    Dep->CompareOp = Op;
 374    Dep->ID = Cache.HeaderP->DependsCount++;
 375
 376    // Locate the target package
 377    pkgCache::PkgIterator Pkg;
 378    if (Owner->NewPackage(Pkg,PackageName) == false)
 379       return false;
 380
 381    // Probe the reverse dependency list for a version string that matches
 382    if (Version.empty() == false)
 383    {
 384 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 385          if (I->Version != 0 && I.TargetVer() == Version)
 386             Dep->Version = I->Version;*/
 387       if (Dep->Version == 0)
 388          if ((Dep->Version = WriteString(Version)) == 0)
 389             return false;
 390    }
 391
 392    // Link it to the package
 393    Dep->Package = Pkg.Index();
 394    Dep->NextRevDepends = Pkg->RevDepends;
 395    Pkg->RevDepends = Dep.Index();
 396
 397    /* Link it to the version (at the end of the list)
 398       Caching the old end point speeds up generation substantially */
 399    if (OldDepVer != Ver)
 400    {
 401       OldDepLast = &Ver->DependsList;
 402       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 403          OldDepLast = &D->NextDepends;
 404       OldDepVer = Ver;
 405    }
 406
 407    // Is it a file dependency?
 408    if (PackageName[0] == '/')
 409       FoundFileDeps = true;
 410
 411    Dep->NextDepends = *OldDepLast;
 412    *OldDepLast = Dep.Index();
 413    OldDepLast = &Dep->NextDepends;
 414
 415    return true;
 416 }
 417                                                                         /*}}}*/
 418 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 419 // ---------------------------------------------------------------------
 420 /* */
 421 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 422                                                 string PackageName,
 423                                                 string Version)
 424 {
 425    pkgCache &Cache = Owner->Cache;
 426
 427    // We do not add self referencing provides
 428    if (Ver.ParentPkg().Name() == PackageName)
 429       return true;
 430
 431    // Get a structure
 432    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 433    if (Provides == 0)
 434       return false;
 435    Cache.HeaderP->ProvidesCount++;
 436
 437    // Fill it in
 438    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 439    Prv->Version = Ver.Index();
 440    Prv->NextPkgProv = Ver->ProvidesList;
 441    Ver->ProvidesList = Prv.Index();
 442    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 443       return false;
 444
 445    // Locate the target package
 446    pkgCache::PkgIterator Pkg;
 447    if (Owner->NewPackage(Pkg,PackageName) == false)
 448       return false;
 449
 450    // Link it to the package
 451    Prv->ParentPkg = Pkg.Index();
 452    Prv->NextProvides = Pkg->ProvidesList;
 453    Pkg->ProvidesList = Prv.Index();
 454
 455    return true;
 456 }
 457                                                                         /*}}}*/
 458 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* This is used to select which file is to be associated with all newly
 461    added versions. The caller is responsible for setting the IMS fields. */
 462 bool pkgCacheGenerator::SelectFile(string File,string Site,
 463                                    const pkgIndexFile &Index,
 464                                    unsigned long Flags)
 465 {
 466    // Get some space for the structure
 467    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 468    if (CurrentFile == Cache.PkgFileP)
 469       return false;
 470
 471    // Fill it in
 472    CurrentFile->FileName = Map.WriteString(File);
 473    CurrentFile->Site = WriteUniqString(Site);
 474    CurrentFile->NextFile = Cache.HeaderP->FileList;
 475    CurrentFile->Flags = Flags;
 476    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 477    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 478    PkgFileName = File;
 479    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 480    Cache.HeaderP->PackageFileCount++;
 481
 482    if (CurrentFile->FileName == 0)
 483       return false;
 484
 485    if (Progress != 0)
 486       Progress->SubProgress(Index.Size());
 487    return true;
 488 }
 489                                                                         /*}}}*/
 490 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 491 // ---------------------------------------------------------------------
 492 /* This is used to create handles to strings. Given the same text it
 493    always returns the same number */
 494 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 495                                                  unsigned int Size)
 496 {
 497    /* We use a very small transient hash table here, this speeds up generation
 498       by a fair amount on slower machines */
 499    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 500    if (Bucket != 0 &&
 501        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 502       return Bucket->String;
 503
 504    // Search for an insertion point
 505    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 506    int Res = 1;
 507    map_ptrloc *Last = &Cache.HeaderP->StringList;
 508    for (; I != Cache.StringItemP; Last = &I->NextItem,
 509         I = Cache.StringItemP + I->NextItem)
 510    {
 511       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 512       if (Res >= 0)
 513          break;
 514    }
 515
 516    // Match
 517    if (Res == 0)
 518    {
 519       Bucket = I;
 520       return I->String;
 521    }
 522
 523    // Get a structure
 524    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 525    if (Item == 0)
 526       return 0;
 527
 528    // Fill in the structure
 529    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 530    ItemP->NextItem = I - Cache.StringItemP;
 531    *Last = Item;
 532    ItemP->String = Map.WriteString(S,Size);
 533    if (ItemP->String == 0)
 534       return 0;
 535
 536    Bucket = ItemP;
 537    return ItemP->String;
 538 }
 539                                                                         /*}}}*/
 540
 541 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 542 // ---------------------------------------------------------------------
 543 /* This just verifies that each file in the list of index files exists,
 544    has matching attributes with the cache and the cache does not have
 545    any extra files. */
 546 static bool CheckValidity(string CacheFile, FileIterator Start,
 547                           FileIterator End,MMap **OutMap = 0)
 548 {
 549    // No file, certainly invalid
 550    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 551       return false;
 552
 553    // Map it
 554    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 555    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 556    pkgCache Cache(Map);
 557    if (_error->PendingError() == true || Map->Size() == 0)
 558    {
 559       _error->Discard();
 560       return false;
 561    }
 562
 563    /* Now we check every index file, see if it is in the cache,
 564       verify the IMS data and check that it is on the disk too.. */
 565    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 566    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 567    for (; Start != End; Start++)
 568    {
 569       if ((*Start)->HasPackages() == false)
 570          continue;
 571
 572       if ((*Start)->Exists() == false)
 573       {
 574          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 575                           (*Start)->Describe().c_str());
 576          continue;
 577       }
 578
 579       // FindInCache is also expected to do an IMS check.
 580       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 581       if (File.end() == true)
 582          return false;
 583
 584       Visited[File->ID] = true;
 585    }
 586
 587    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 588       if (Visited[I] == false)
 589          return false;
 590
 591    if (_error->PendingError() == true)
 592    {
 593       _error->Discard();
 594       return false;
 595    }
 596
 597    if (OutMap != 0)
 598       *OutMap = Map.UnGuard();
 599    return true;
 600 }
 601                                                                         /*}}}*/
 602 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* Size is kind of an abstract notion that is only used for the progress
 605    meter */
 606 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 607 {
 608    unsigned long TotalSize = 0;
 609    for (; Start != End; Start++)
 610    {
 611       if ((*Start)->HasPackages() == false)
 612          continue;
 613       TotalSize += (*Start)->Size();
 614    }
 615    return TotalSize;
 616 }
 617                                                                         /*}}}*/
 618 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 619 // ---------------------------------------------------------------------
 620 /* */
 621 static bool BuildCache(pkgCacheGenerator &Gen,
 622                        OpProgress &Progress,
 623                        unsigned long &CurrentSize,unsigned long TotalSize,
 624                        FileIterator Start, FileIterator End)
 625 {
 626    FileIterator I;
 627    for (I = Start; I != End; I++)
 628    {
 629       if ((*I)->HasPackages() == false)
 630          continue;
 631
 632       if ((*I)->Exists() == false)
 633          continue;
 634
 635       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 636       {
 637          _error->Warning("Duplicate sources.list entry %s",
 638                          (*I)->Describe().c_str());
 639          continue;
 640       }
 641
 642       unsigned long Size = (*I)->Size();
 643       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 644       CurrentSize += Size;
 645
 646       if ((*I)->Merge(Gen,Progress) == false)
 647          return false;
 648    }
 649
 650    if (Gen.HasFileDeps() == true)
 651    {
 652       Progress.Done();
 653       TotalSize = ComputeSize(Start, End);
 654       CurrentSize = 0;
 655       for (I = Start; I != End; I++)
 656       {
 657          unsigned long Size = (*I)->Size();
 658          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 659          CurrentSize += Size;
 660          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 661             return false;
 662       }
 663    }
 664
 665    return true;
 666 }
 667                                                                         /*}}}*/
 668 // MakeStatusCache - Construct the status cache                         /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This makes sure that the status cache (the cache that has all
 671    index files from the sources list and all local ones) is ready
 672    to be mmaped. If OutMap is not zero then a MMap object representing
 673    the cache will be stored there. This is pretty much mandetory if you
 674    are using AllowMem. AllowMem lets the function be run as non-root
 675    where it builds the cache 'fast' into a memory buffer. */
 676 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 677                         MMap **OutMap,bool AllowMem)
 678 {
 679    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 680
 681    vector<pkgIndexFile *> Files;
 682    for (vector<metaIndex *>::const_iterator i = List.begin();
 683         i != List.end();
 684         i++)
 685    {
 686       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 687       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 688            j != Indexes->end();
 689            j++)
 690          Files.push_back (*j);
 691    }
 692
 693    unsigned long EndOfSource = Files.size();
 694    if (_system->AddStatusFiles(Files) == false)
 695       return false;
 696
 697    // Decide if we can write to the files..
 698    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 699    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 700
 701    // Decide if we can write to the cache
 702    bool Writeable = false;
 703    if (CacheFile.empty() == false)
 704       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 705    else
 706       if (SrcCacheFile.empty() == false)
 707          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 708
 709    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 710       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 711
 712    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 713
 714    // Cache is OK, Fin.
 715    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 716    {
 717       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 718       return true;
 719    }
 720
 721    /* At this point we know we need to reconstruct the package cache,
 722       begin. */
 723    SPtr<FileFd> CacheF;
 724    SPtr<DynamicMMap> Map;
 725    if (Writeable == true && CacheFile.empty() == false)
 726    {
 727       unlink(CacheFile.c_str());
 728       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 729       fchmod(CacheF->Fd(),0644);
 730       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 731       if (_error->PendingError() == true)
 732          return false;
 733    }
 734    else
 735    {
 736       // Just build it in memory..
 737       Map = new DynamicMMap(MMap::Public,MapSize);
 738    }
 739
 740    // Lets try the source cache.
 741    unsigned long CurrentSize = 0;
 742    unsigned long TotalSize = 0;
 743    if (CheckValidity(SrcCacheFile,Files.begin(),
 744                      Files.begin()+EndOfSource) == true)
 745    {
 746       // Preload the map with the source cache
 747       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 748       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 749                        SCacheF.Size()) == false)
 750          return false;
 751
 752       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 753
 754       // Build the status cache
 755       pkgCacheGenerator Gen(Map.Get(),&Progress);
 756       if (_error->PendingError() == true)
 757          return false;
 758       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 759                      Files.begin()+EndOfSource,Files.end()) == false)
 760          return false;
 761    }
 762    else
 763    {
 764       TotalSize = ComputeSize(Files.begin(),Files.end());
 765
 766       // Build the source cache
 767       pkgCacheGenerator Gen(Map.Get(),&Progress);
 768       if (_error->PendingError() == true)
 769          return false;
 770       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 771                      Files.begin(),Files.begin()+EndOfSource) == false)
 772          return false;
 773
 774       // Write it back
 775       if (Writeable == true && SrcCacheFile.empty() == false)
 776       {
 777          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 778          if (_error->PendingError() == true)
 779             return false;
 780
 781          fchmod(SCacheF.Fd(),0644);
 782
 783          // Write out the main data
 784          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 785             return _error->Error(_("IO Error saving source cache"));
 786          SCacheF.Sync();
 787
 788          // Write out the proper header
 789          Gen.GetCache().HeaderP->Dirty = false;
 790          if (SCacheF.Seek(0) == false ||
 791              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 792             return _error->Error(_("IO Error saving source cache"));
 793          Gen.GetCache().HeaderP->Dirty = true;
 794          SCacheF.Sync();
 795       }
 796
 797       // Build the status cache
 798       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 799                      Files.begin()+EndOfSource,Files.end()) == false)
 800          return false;
 801    }
 802
 803    if (_error->PendingError() == true)
 804       return false;
 805    if (OutMap != 0)
 806    {
 807       if (CacheF != 0)
 808       {
 809          delete Map.UnGuard();
 810          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 811       }
 812       else
 813       {
 814          *OutMap = Map.UnGuard();
 815       }
 816    }
 817
 818    return true;
 819 }
 820                                                                         /*}}}*/
 821 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 822 // ---------------------------------------------------------------------
 823 /* */
 824 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 825 {
 826    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 827    vector<pkgIndexFile *> Files;
 828    unsigned long EndOfSource = Files.size();
 829    if (_system->AddStatusFiles(Files) == false)
 830       return false;
 831
 832    SPtr<DynamicMMap> Map;
 833    Map = new DynamicMMap(MMap::Public,MapSize);
 834    unsigned long CurrentSize = 0;
 835    unsigned long TotalSize = 0;
 836
 837    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 838
 839    // Build the status cache
 840    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 841    pkgCacheGenerator Gen(Map.Get(),&Progress);
 842    if (_error->PendingError() == true)
 843       return false;
 844    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 845                   Files.begin()+EndOfSource,Files.end()) == false)
 846       return false;
 847
 848    if (_error->PendingError() == true)
 849       return false;
 850    *OutMap = Map.UnGuard();
 851
 852    return true;
 853 }
 854                                                                         /*}}}*/