apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apt-pkg/tagfile.h>
  30
  31 #include <apti18n.h>
  32
  33 #include <vector>
  34
  35 #include <sys/stat.h>
  36 #include <unistd.h>
  37 #include <errno.h>
  38 #include <stdio.h>
  39 #include <system.h>
  40                                                                         /*}}}*/
  41 typedef vector<pkgIndexFile *>::iterator FileIterator;
  42
  43 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* We set the diry flag and make sure that is written to the disk */
  46 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  47                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  48                     FoundFileDeps(0)
  49 {
  50    CurrentFile = 0;
  51    memset(UniqHash,0,sizeof(UniqHash));
  52
  53    if (_error->PendingError() == true)
  54       return;
  55
  56    if (Map.Size() == 0)
  57    {
  58       // Setup the map interface..
  59       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  60       Map.RawAllocate(sizeof(pkgCache::Header));
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108
 109    unsigned int Counter = 0;
 110    while (List.Step() == true)
 111    {
 112       // Get a pointer to the package structure
 113       string PackageName = List.Package();
 114       if (PackageName.empty() == true)
 115          return false;
 116
 117       pkgCache::PkgIterator Pkg;
 118       if (NewPackage(Pkg,PackageName) == false)
 119          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 120       Counter++;
 121       if (Counter % 100 == 0 && Progress != 0)
 122          Progress->Progress(List.Offset());
 123
 124       /* Get a pointer to the version structure. We know the list is sorted
 125          so we use that fact in the search. Insertion of new versions is
 126          done with correct sorting */
 127       string Version = List.Version();
 128       if (Version.empty() == true)
 129       {
 130          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 131             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 132                                  PackageName.c_str());
 133          continue;
 134       }
 135
 136       pkgCache::VerIterator Ver = Pkg.VersionList();
 137       map_ptrloc *Last = &Pkg->VersionList;
 138       int Res = 1;
 139       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 140       {
 141          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 142          if (Res >= 0)
 143             break;
 144       }
 145
 146       /* We already have a version for this item, record that we
 147          saw it */
 148       unsigned long Hash = List.VersionHash();
 149       if (Res == 0 && Ver->Hash == Hash)
 150       {
 151          if (List.UsePackage(Pkg,Ver) == false)
 152             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 153                                  PackageName.c_str());
 154
 155          if (NewFileVer(Ver,List) == false)
 156             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 157                                  PackageName.c_str());
 158
 159          // Read only a single record and return
 160          if (OutVer != 0)
 161          {
 162             *OutVer = Ver;
 163             FoundFileDeps |= List.HasFileDeps();
 164             return true;
 165          }
 166
 167          continue;
 168       }
 169
 170       // Skip to the end of the same version set.
 171       if (Res == 0)
 172       {
 173          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 174          {
 175             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 176             if (Res != 0)
 177                break;
 178          }
 179       }
 180
 181       // Add a new version
 182       *Last = NewVersion(Ver,Version,*Last);
 183       Ver->ParentPkg = Pkg.Index();
 184       Ver->Hash = Hash;
 185       if (List.NewVersion(Ver) == false)
 186          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 187                               PackageName.c_str());
 188
 189       if (List.UsePackage(Pkg,Ver) == false)
 190          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 191                               PackageName.c_str());
 192
 193       if (NewFileVer(Ver,List) == false)
 194          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 195                               PackageName.c_str());
 196
 197       // Read only a single record and return
 198       if (OutVer != 0)
 199       {
 200          *OutVer = Ver;
 201          FoundFileDeps |= List.HasFileDeps();
 202          return true;
 203       }
 204    }
 205
 206    FoundFileDeps |= List.HasFileDeps();
 207
 208    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 209       return _error->Error(_("Wow, you exceeded the number of package "
 210                              "names this APT is capable of."));
 211    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 212       return _error->Error(_("Wow, you exceeded the number of versions "
 213                              "this APT is capable of."));
 214    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 215       return _error->Error(_("Wow, you exceeded the number of dependencies "
 216                              "this APT is capable of."));
 217    return true;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 221 // ---------------------------------------------------------------------
 222 /* If we found any file depends while parsing the main list we need to
 223    resolve them. Since it is undesired to load the entire list of files
 224    into the cache as virtual packages we do a two stage effort. MergeList
 225    identifies the file depends and this creates Provdies for them by
 226    re-parsing all the indexs. */
 227 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 228 {
 229    List.Owner = this;
 230
 231    unsigned int Counter = 0;
 232    while (List.Step() == true)
 233    {
 234       string PackageName = List.Package();
 235       if (PackageName.empty() == true)
 236          return false;
 237       string Version = List.Version();
 238       if (Version.empty() == true)
 239          continue;
 240
 241       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 242       if (Pkg.end() == true)
 243          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 244                                 PackageName.c_str());
 245       Counter++;
 246       if (Counter % 100 == 0 && Progress != 0)
 247          Progress->Progress(List.Offset());
 248
 249       unsigned long Hash = List.VersionHash();
 250       pkgCache::VerIterator Ver = Pkg.VersionList();
 251       for (; Ver.end() == false; Ver++)
 252       {
 253          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 254          {
 255             if (List.CollectFileProvides(Cache,Ver) == false)
 256                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 257             break;
 258          }
 259       }
 260
 261       if (Ver.end() == true)
 262          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 263    }
 264
 265    return true;
 266 }
 267                                                                         /*}}}*/
 268 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 269 // ---------------------------------------------------------------------
 270 /* This creates a new package structure and adds it to the hash table */
 271 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 272 {
 273    Pkg = Cache.FindPkg(Name);
 274    if (Pkg.end() == false)
 275       return true;
 276
 277    // Get a structure
 278    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 279    if (Package == 0)
 280       return false;
 281
 282    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 283
 284    // Insert it into the hash table
 285    unsigned long Hash = Cache.Hash(Name);
 286    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 287    Cache.HeaderP->HashTable[Hash] = Package;
 288
 289    // Set the name and the ID
 290    Pkg->Name = Map.WriteString(Name);
 291    if (Pkg->Name == 0)
 292       return false;
 293    Pkg->ID = Cache.HeaderP->PackageCount++;
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 299 // ---------------------------------------------------------------------
 300 /* */
 301 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 302                                    ListParser &List)
 303 {
 304    if (CurrentFile == 0)
 305       return true;
 306
 307    // Get a structure
 308    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 309    if (VerFile == 0)
 310       return 0;
 311
 312    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 313    VF->File = CurrentFile - Cache.PkgFileP;
 314
 315    // Link it to the end of the list
 316    map_ptrloc *Last = &Ver->FileList;
 317    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 318       Last = &V->NextFile;
 319    VF->NextFile = *Last;
 320    *Last = VF.Index();
 321
 322    VF->Offset = List.Offset();
 323    VF->Size = List.Size();
 324    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 325       Cache.HeaderP->MaxVerFileSize = VF->Size;
 326    Cache.HeaderP->VerFileCount++;
 327
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* This puts a version structure in the linked list */
 334 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 335                                             const string &VerStr,
 336                                             unsigned long Next)
 337 {
 338    // Get a structure
 339    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 340    if (Version == 0)
 341       return 0;
 342
 343    // Fill it in
 344    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 345    Ver->NextVer = Next;
 346    Ver->ID = Cache.HeaderP->VersionCount++;
 347    Ver->VerStr = Map.WriteString(VerStr);
 348    if (Ver->VerStr == 0)
 349       return 0;
 350
 351    return Version;
 352 }
 353                                                                         /*}}}*/
 354 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* This creates a dependency element in the tree. It is linked to the
 357    version and to the package that it is pointing to. */
 358 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 359                                                const string &PackageName,
 360                                                const string &Version,
 361                                                unsigned int Op,
 362                                                unsigned int Type)
 363 {
 364    pkgCache &Cache = Owner->Cache;
 365
 366    // Get a structure
 367    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 368    if (Dependency == 0)
 369       return false;
 370
 371    // Fill it in
 372    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 373    Dep->ParentVer = Ver.Index();
 374    Dep->Type = Type;
 375    Dep->CompareOp = Op;
 376    Dep->ID = Cache.HeaderP->DependsCount++;
 377
 378    // Locate the target package
 379    pkgCache::PkgIterator Pkg;
 380    if (Owner->NewPackage(Pkg,PackageName) == false)
 381       return false;
 382
 383    // Probe the reverse dependency list for a version string that matches
 384    if (Version.empty() == false)
 385    {
 386 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 387          if (I->Version != 0 && I.TargetVer() == Version)
 388             Dep->Version = I->Version;*/
 389       if (Dep->Version == 0)
 390          if ((Dep->Version = WriteString(Version)) == 0)
 391             return false;
 392    }
 393
 394    // Link it to the package
 395    Dep->Package = Pkg.Index();
 396    Dep->NextRevDepends = Pkg->RevDepends;
 397    Pkg->RevDepends = Dep.Index();
 398
 399    /* Link it to the version (at the end of the list)
 400       Caching the old end point speeds up generation substantially */
 401    if (OldDepVer != Ver)
 402    {
 403       OldDepLast = &Ver->DependsList;
 404       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 405          OldDepLast = &D->NextDepends;
 406       OldDepVer = Ver;
 407    }
 408
 409    // Is it a file dependency?
 410    if (PackageName[0] == '/')
 411       FoundFileDeps = true;
 412
 413    Dep->NextDepends = *OldDepLast;
 414    *OldDepLast = Dep.Index();
 415    OldDepLast = &Dep->NextDepends;
 416
 417    return true;
 418 }
 419                                                                         /*}}}*/
 420 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 421 // ---------------------------------------------------------------------
 422 /* */
 423 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 424                                                 const string &PackageName,
 425                                                 const string &Version)
 426 {
 427    pkgCache &Cache = Owner->Cache;
 428
 429    // We do not add self referencing provides
 430    if (Ver.ParentPkg().Name() == PackageName)
 431       return true;
 432
 433    // Get a structure
 434    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 435    if (Provides == 0)
 436       return false;
 437    Cache.HeaderP->ProvidesCount++;
 438
 439    // Fill it in
 440    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 441    Prv->Version = Ver.Index();
 442    Prv->NextPkgProv = Ver->ProvidesList;
 443    Ver->ProvidesList = Prv.Index();
 444    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 445       return false;
 446
 447    // Locate the target package
 448    pkgCache::PkgIterator Pkg;
 449    if (Owner->NewPackage(Pkg,PackageName) == false)
 450       return false;
 451
 452    // Link it to the package
 453    Prv->ParentPkg = Pkg.Index();
 454    Prv->NextProvides = Pkg->ProvidesList;
 455    Pkg->ProvidesList = Prv.Index();
 456
 457    return true;
 458 }
 459                                                                         /*}}}*/
 460 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 461 // ---------------------------------------------------------------------
 462 /* This is used to select which file is to be associated with all newly
 463    added versions. The caller is responsible for setting the IMS fields. */
 464 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 465                                    const pkgIndexFile &Index,
 466                                    unsigned long Flags)
 467 {
 468    // Get some space for the structure
 469    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 470    if (CurrentFile == Cache.PkgFileP)
 471       return false;
 472
 473    // Fill it in
 474    CurrentFile->FileName = Map.WriteString(File);
 475    CurrentFile->Site = WriteUniqString(Site);
 476    CurrentFile->NextFile = Cache.HeaderP->FileList;
 477    CurrentFile->Flags = Flags;
 478    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 479    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 480    PkgFileName = File;
 481    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 482    Cache.HeaderP->PackageFileCount++;
 483
 484    if (CurrentFile->FileName == 0)
 485       return false;
 486
 487    if (Progress != 0)
 488       Progress->SubProgress(Index.Size());
 489    return true;
 490 }
 491                                                                         /*}}}*/
 492 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 493 // ---------------------------------------------------------------------
 494 /* This is used to create handles to strings. Given the same text it
 495    always returns the same number */
 496 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 497                                                  unsigned int Size)
 498 {
 499    /* We use a very small transient hash table here, this speeds up generation
 500       by a fair amount on slower machines */
 501    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 502    if (Bucket != 0 &&
 503        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 504       return Bucket->String;
 505
 506    // Search for an insertion point
 507    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 508    int Res = 1;
 509    map_ptrloc *Last = &Cache.HeaderP->StringList;
 510    for (; I != Cache.StringItemP; Last = &I->NextItem,
 511         I = Cache.StringItemP + I->NextItem)
 512    {
 513       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 514       if (Res >= 0)
 515          break;
 516    }
 517
 518    // Match
 519    if (Res == 0)
 520    {
 521       Bucket = I;
 522       return I->String;
 523    }
 524
 525    // Get a structure
 526    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 527    if (Item == 0)
 528       return 0;
 529
 530    // Fill in the structure
 531    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 532    ItemP->NextItem = I - Cache.StringItemP;
 533    *Last = Item;
 534    ItemP->String = Map.WriteString(S,Size);
 535    if (ItemP->String == 0)
 536       return 0;
 537
 538    Bucket = ItemP;
 539    return ItemP->String;
 540 }
 541                                                                         /*}}}*/
 542
 543 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* This just verifies that each file in the list of index files exists,
 546    has matching attributes with the cache and the cache does not have
 547    any extra files. */
 548 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 549                           FileIterator End,MMap **OutMap = 0)
 550 {
 551    // No file, certainly invalid
 552    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 553       return false;
 554
 555    // Map it
 556    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 557    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 558    pkgCache Cache(Map);
 559    if (_error->PendingError() == true || Map->Size() == 0)
 560    {
 561       _error->Discard();
 562       return false;
 563    }
 564
 565    /* Now we check every index file, see if it is in the cache,
 566       verify the IMS data and check that it is on the disk too.. */
 567    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 568    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 569    for (; Start != End; Start++)
 570    {
 571       if ((*Start)->HasPackages() == false)
 572          continue;
 573
 574       if ((*Start)->Exists() == false)
 575       {
 576          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 577                           (*Start)->Describe().c_str());
 578          continue;
 579       }
 580
 581       // FindInCache is also expected to do an IMS check.
 582       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 583       if (File.end() == true)
 584          return false;
 585
 586       Visited[File->ID] = true;
 587    }
 588
 589    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 590       if (Visited[I] == false)
 591          return false;
 592
 593    if (_error->PendingError() == true)
 594    {
 595       _error->Discard();
 596       return false;
 597    }
 598
 599    if (OutMap != 0)
 600       *OutMap = Map.UnGuard();
 601    return true;
 602 }
 603                                                                         /*}}}*/
 604 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 605 // ---------------------------------------------------------------------
 606 /* Size is kind of an abstract notion that is only used for the progress
 607    meter */
 608 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 609 {
 610    unsigned long TotalSize = 0;
 611    for (; Start != End; Start++)
 612    {
 613       if ((*Start)->HasPackages() == false)
 614          continue;
 615       TotalSize += (*Start)->Size();
 616    }
 617    return TotalSize;
 618 }
 619                                                                         /*}}}*/
 620 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 621 // ---------------------------------------------------------------------
 622 /* */
 623 static bool BuildCache(pkgCacheGenerator &Gen,
 624                        OpProgress &Progress,
 625                        unsigned long &CurrentSize,unsigned long TotalSize,
 626                        FileIterator Start, FileIterator End)
 627 {
 628    FileIterator I;
 629    for (I = Start; I != End; I++)
 630    {
 631       if ((*I)->HasPackages() == false)
 632          continue;
 633
 634       if ((*I)->Exists() == false)
 635          continue;
 636
 637       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 638       {
 639          _error->Warning("Duplicate sources.list entry %s",
 640                          (*I)->Describe().c_str());
 641          continue;
 642       }
 643
 644       unsigned long Size = (*I)->Size();
 645       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 646       CurrentSize += Size;
 647
 648       if ((*I)->Merge(Gen,Progress) == false)
 649          return false;
 650    }
 651
 652    if (Gen.HasFileDeps() == true)
 653    {
 654       Progress.Done();
 655       TotalSize = ComputeSize(Start, End);
 656       CurrentSize = 0;
 657       for (I = Start; I != End; I++)
 658       {
 659          unsigned long Size = (*I)->Size();
 660          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 661          CurrentSize += Size;
 662          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 663             return false;
 664       }
 665    }
 666
 667    return true;
 668 }
 669                                                                         /*}}}*/
 670 // MakeStatusCache - Construct the status cache                         /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* This makes sure that the status cache (the cache that has all
 673    index files from the sources list and all local ones) is ready
 674    to be mmaped. If OutMap is not zero then a MMap object representing
 675    the cache will be stored there. This is pretty much mandetory if you
 676    are using AllowMem. AllowMem lets the function be run as non-root
 677    where it builds the cache 'fast' into a memory buffer. */
 678 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 679                         MMap **OutMap,bool AllowMem)
 680 {
 681    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 682
 683    vector<pkgIndexFile *> Files;
 684    for (vector<metaIndex *>::const_iterator i = List.begin();
 685         i != List.end();
 686         i++)
 687    {
 688       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 689       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 690            j != Indexes->end();
 691            j++)
 692          Files.push_back (*j);
 693    }
 694
 695    unsigned long EndOfSource = Files.size();
 696    if (_system->AddStatusFiles(Files) == false)
 697       return false;
 698
 699    // Decide if we can write to the files..
 700    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 701    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 702
 703    // Decide if we can write to the cache
 704    bool Writeable = false;
 705    if (CacheFile.empty() == false)
 706       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 707    else
 708       if (SrcCacheFile.empty() == false)
 709          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 710
 711    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 712       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 713
 714    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 715
 716    // Cache is OK, Fin.
 717    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 718    {
 719       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 720       return true;
 721    }
 722
 723    /* At this point we know we need to reconstruct the package cache,
 724       begin. */
 725    SPtr<FileFd> CacheF;
 726    SPtr<DynamicMMap> Map;
 727    if (Writeable == true && CacheFile.empty() == false)
 728    {
 729       unlink(CacheFile.c_str());
 730       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 731       fchmod(CacheF->Fd(),0644);
 732       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 733       if (_error->PendingError() == true)
 734          return false;
 735    }
 736    else
 737    {
 738       // Just build it in memory..
 739       Map = new DynamicMMap(MMap::Public,MapSize);
 740    }
 741
 742    // Lets try the source cache.
 743    unsigned long CurrentSize = 0;
 744    unsigned long TotalSize = 0;
 745    if (CheckValidity(SrcCacheFile,Files.begin(),
 746                      Files.begin()+EndOfSource) == true)
 747    {
 748       // Preload the map with the source cache
 749       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 750       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 751                        SCacheF.Size()) == false)
 752          return false;
 753
 754       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 755
 756       // Build the status cache
 757       pkgCacheGenerator Gen(Map.Get(),&Progress);
 758       if (_error->PendingError() == true)
 759          return false;
 760       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 761                      Files.begin()+EndOfSource,Files.end()) == false)
 762          return false;
 763    }
 764    else
 765    {
 766       TotalSize = ComputeSize(Files.begin(),Files.end());
 767
 768       // Build the source cache
 769       pkgCacheGenerator Gen(Map.Get(),&Progress);
 770       if (_error->PendingError() == true)
 771          return false;
 772       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 773                      Files.begin(),Files.begin()+EndOfSource) == false)
 774          return false;
 775
 776       // Write it back
 777       if (Writeable == true && SrcCacheFile.empty() == false)
 778       {
 779          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 780          if (_error->PendingError() == true)
 781             return false;
 782
 783          fchmod(SCacheF.Fd(),0644);
 784
 785          // Write out the main data
 786          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 787             return _error->Error(_("IO Error saving source cache"));
 788          SCacheF.Sync();
 789
 790          // Write out the proper header
 791          Gen.GetCache().HeaderP->Dirty = false;
 792          if (SCacheF.Seek(0) == false ||
 793              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 794             return _error->Error(_("IO Error saving source cache"));
 795          Gen.GetCache().HeaderP->Dirty = true;
 796          SCacheF.Sync();
 797       }
 798
 799       // Build the status cache
 800       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 801                      Files.begin()+EndOfSource,Files.end()) == false)
 802          return false;
 803    }
 804
 805    if (_error->PendingError() == true)
 806       return false;
 807    if (OutMap != 0)
 808    {
 809       if (CacheF != 0)
 810       {
 811          delete Map.UnGuard();
 812          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 813       }
 814       else
 815       {
 816          *OutMap = Map.UnGuard();
 817       }
 818    }
 819
 820    return true;
 821 }
 822                                                                         /*}}}*/
 823 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 824 // ---------------------------------------------------------------------
 825 /* */
 826 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 827 {
 828    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 829    vector<pkgIndexFile *> Files;
 830    unsigned long EndOfSource = Files.size();
 831    if (_system->AddStatusFiles(Files) == false)
 832       return false;
 833
 834    SPtr<DynamicMMap> Map;
 835    Map = new DynamicMMap(MMap::Public,MapSize);
 836    unsigned long CurrentSize = 0;
 837    unsigned long TotalSize = 0;
 838
 839    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 840
 841    // Build the status cache
 842    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 843    pkgCacheGenerator Gen(Map.Get(),&Progress);
 844    if (_error->PendingError() == true)
 845       return false;
 846    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 847                   Files.begin()+EndOfSource,Files.end()) == false)
 848       return false;
 849
 850    if (_error->PendingError() == true)
 851       return false;
 852    *OutMap = Map.UnGuard();
 853
 854    return true;
 855 }
 856                                                                         /*}}}*/