apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37 #include <system.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       Map.RawAllocate(sizeof(pkgCache::Header));
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 129             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 130                                  PackageName.c_str());
 131          continue;
 132       }
 133
 134       pkgCache::VerIterator Ver = Pkg.VersionList();
 135       map_ptrloc *Last = &Pkg->VersionList;
 136       int Res = 1;
 137       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 138       {
 139          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 140          if (Res >= 0)
 141             break;
 142       }
 143
 144       /* We already have a version for this item, record that we
 145          saw it */
 146       unsigned long Hash = List.VersionHash();
 147       if (Res == 0 && Ver->Hash == Hash)
 148       {
 149          if (List.UsePackage(Pkg,Ver) == false)
 150             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 151                                  PackageName.c_str());
 152
 153          if (NewFileVer(Ver,List) == false)
 154             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 155                                  PackageName.c_str());
 156
 157          // Read only a single record and return
 158          if (OutVer != 0)
 159          {
 160             *OutVer = Ver;
 161             FoundFileDeps |= List.HasFileDeps();
 162             return true;
 163          }
 164
 165          continue;
 166       }
 167
 168       // Skip to the end of the same version set.
 169       if (Res == 0)
 170       {
 171          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 172          {
 173             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 174             if (Res != 0)
 175                break;
 176          }
 177       }
 178
 179       // Add a new version
 180       *Last = NewVersion(Ver,Version,*Last);
 181       Ver->ParentPkg = Pkg.Index();
 182       Ver->Hash = Hash;
 183       if (List.NewVersion(Ver) == false)
 184          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 185                               PackageName.c_str());
 186
 187       if (List.UsePackage(Pkg,Ver) == false)
 188          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 189                               PackageName.c_str());
 190
 191       if (NewFileVer(Ver,List) == false)
 192          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 193                               PackageName.c_str());
 194
 195       // Read only a single record and return
 196       if (OutVer != 0)
 197       {
 198          *OutVer = Ver;
 199          FoundFileDeps |= List.HasFileDeps();
 200          return true;
 201       }
 202    }
 203
 204    FoundFileDeps |= List.HasFileDeps();
 205
 206    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 207       return _error->Error(_("Wow, you exceeded the number of package "
 208                              "names this APT is capable of."));
 209    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 210       return _error->Error(_("Wow, you exceeded the number of versions "
 211                              "this APT is capable of."));
 212    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 213       return _error->Error(_("Wow, you exceeded the number of dependencies "
 214                              "this APT is capable of."));
 215    return true;
 216 }
 217                                                                         /*}}}*/
 218 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 219 // ---------------------------------------------------------------------
 220 /* If we found any file depends while parsing the main list we need to
 221    resolve them. Since it is undesired to load the entire list of files
 222    into the cache as virtual packages we do a two stage effort. MergeList
 223    identifies the file depends and this creates Provdies for them by
 224    re-parsing all the indexs. */
 225 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 226 {
 227    List.Owner = this;
 228
 229    unsigned int Counter = 0;
 230    while (List.Step() == true)
 231    {
 232       string PackageName = List.Package();
 233       if (PackageName.empty() == true)
 234          return false;
 235       string Version = List.Version();
 236       if (Version.empty() == true)
 237          continue;
 238
 239       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 240       if (Pkg.end() == true)
 241          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 242                                 PackageName.c_str());
 243       Counter++;
 244       if (Counter % 100 == 0 && Progress != 0)
 245          Progress->Progress(List.Offset());
 246
 247       unsigned long Hash = List.VersionHash();
 248       pkgCache::VerIterator Ver = Pkg.VersionList();
 249       for (; Ver.end() == false; Ver++)
 250       {
 251          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 252          {
 253             if (List.CollectFileProvides(Cache,Ver) == false)
 254                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 255             break;
 256          }
 257       }
 258
 259       if (Ver.end() == true)
 260          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 261    }
 262
 263    return true;
 264 }
 265                                                                         /*}}}*/
 266 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 267 // ---------------------------------------------------------------------
 268 /* This creates a new package structure and adds it to the hash table */
 269 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 270 {
 271    Pkg = Cache.FindPkg(Name);
 272    if (Pkg.end() == false)
 273       return true;
 274
 275    // Get a structure
 276    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 277    if (Package == 0)
 278       return false;
 279
 280    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 281
 282    // Insert it into the hash table
 283    unsigned long Hash = Cache.Hash(Name);
 284    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 285    Cache.HeaderP->HashTable[Hash] = Package;
 286
 287    // Set the name and the ID
 288    Pkg->Name = Map.WriteString(Name);
 289    if (Pkg->Name == 0)
 290       return false;
 291    Pkg->ID = Cache.HeaderP->PackageCount++;
 292
 293    return true;
 294 }
 295                                                                         /*}}}*/
 296 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 297 // ---------------------------------------------------------------------
 298 /* */
 299 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 300                                    ListParser &List)
 301 {
 302    if (CurrentFile == 0)
 303       return true;
 304
 305    // Get a structure
 306    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 307    if (VerFile == 0)
 308       return 0;
 309
 310    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 311    VF->File = CurrentFile - Cache.PkgFileP;
 312
 313    // Link it to the end of the list
 314    map_ptrloc *Last = &Ver->FileList;
 315    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 316       Last = &V->NextFile;
 317    VF->NextFile = *Last;
 318    *Last = VF.Index();
 319
 320    VF->Offset = List.Offset();
 321    VF->Size = List.Size();
 322    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 323       Cache.HeaderP->MaxVerFileSize = VF->Size;
 324    Cache.HeaderP->VerFileCount++;
 325
 326    return true;
 327 }
 328                                                                         /*}}}*/
 329 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 330 // ---------------------------------------------------------------------
 331 /* This puts a version structure in the linked list */
 332 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 333                                             const string &VerStr,
 334                                             unsigned long Next)
 335 {
 336    // Get a structure
 337    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 338    if (Version == 0)
 339       return 0;
 340
 341    // Fill it in
 342    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 343    Ver->NextVer = Next;
 344    Ver->ID = Cache.HeaderP->VersionCount++;
 345    Ver->VerStr = Map.WriteString(VerStr);
 346    if (Ver->VerStr == 0)
 347       return 0;
 348
 349    return Version;
 350 }
 351                                                                         /*}}}*/
 352 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 353 // ---------------------------------------------------------------------
 354 /* This creates a dependency element in the tree. It is linked to the
 355    version and to the package that it is pointing to. */
 356 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 357                                                const string &PackageName,
 358                                                const string &Version,
 359                                                unsigned int Op,
 360                                                unsigned int Type)
 361 {
 362    pkgCache &Cache = Owner->Cache;
 363
 364    // Get a structure
 365    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 366    if (Dependency == 0)
 367       return false;
 368
 369    // Fill it in
 370    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 371    Dep->ParentVer = Ver.Index();
 372    Dep->Type = Type;
 373    Dep->CompareOp = Op;
 374    Dep->ID = Cache.HeaderP->DependsCount++;
 375
 376    // Locate the target package
 377    pkgCache::PkgIterator Pkg;
 378    if (Owner->NewPackage(Pkg,PackageName) == false)
 379       return false;
 380
 381    // Probe the reverse dependency list for a version string that matches
 382    if (Version.empty() == false)
 383    {
 384 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 385          if (I->Version != 0 && I.TargetVer() == Version)
 386             Dep->Version = I->Version;*/
 387       if (Dep->Version == 0)
 388          if ((Dep->Version = WriteString(Version)) == 0)
 389             return false;
 390    }
 391
 392    // Link it to the package
 393    Dep->Package = Pkg.Index();
 394    Dep->NextRevDepends = Pkg->RevDepends;
 395    Pkg->RevDepends = Dep.Index();
 396
 397    /* Link it to the version (at the end of the list)
 398       Caching the old end point speeds up generation substantially */
 399    if (OldDepVer != Ver)
 400    {
 401       OldDepLast = &Ver->DependsList;
 402       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 403          OldDepLast = &D->NextDepends;
 404       OldDepVer = Ver;
 405    }
 406
 407    // Is it a file dependency?
 408    if (PackageName[0] == '/')
 409       FoundFileDeps = true;
 410
 411    Dep->NextDepends = *OldDepLast;
 412    *OldDepLast = Dep.Index();
 413    OldDepLast = &Dep->NextDepends;
 414
 415    return true;
 416 }
 417                                                                         /*}}}*/
 418 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 419 // ---------------------------------------------------------------------
 420 /* */
 421 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 422                                                 const string &PackageName,
 423                                                 const string &Version)
 424 {
 425    pkgCache &Cache = Owner->Cache;
 426
 427    // We do not add self referencing provides
 428    if (Ver.ParentPkg().Name() == PackageName)
 429       return true;
 430
 431    // Get a structure
 432    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 433    if (Provides == 0)
 434       return false;
 435    Cache.HeaderP->ProvidesCount++;
 436
 437    // Fill it in
 438    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 439    Prv->Version = Ver.Index();
 440    Prv->NextPkgProv = Ver->ProvidesList;
 441    Ver->ProvidesList = Prv.Index();
 442    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 443       return false;
 444
 445    // Locate the target package
 446    pkgCache::PkgIterator Pkg;
 447    if (Owner->NewPackage(Pkg,PackageName) == false)
 448       return false;
 449
 450    // Link it to the package
 451    Prv->ParentPkg = Pkg.Index();
 452    Prv->NextProvides = Pkg->ProvidesList;
 453    Pkg->ProvidesList = Prv.Index();
 454
 455    return true;
 456 }
 457                                                                         /*}}}*/
 458 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* This is used to select which file is to be associated with all newly
 461    added versions. The caller is responsible for setting the IMS fields. */
 462 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 463                                    const pkgIndexFile &Index,
 464                                    unsigned long Flags)
 465 {
 466    // Get some space for the structure
 467    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 468    if (CurrentFile == Cache.PkgFileP)
 469       return false;
 470
 471    // Fill it in
 472    CurrentFile->FileName = Map.WriteString(File);
 473    CurrentFile->Site = WriteUniqString(Site);
 474    CurrentFile->NextFile = Cache.HeaderP->FileList;
 475    CurrentFile->Flags = Flags;
 476    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 477    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 478    PkgFileName = File;
 479    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 480    Cache.HeaderP->PackageFileCount++;
 481
 482    if (CurrentFile->FileName == 0)
 483       return false;
 484
 485    if (Progress != 0)
 486       Progress->SubProgress(Index.Size());
 487    return true;
 488 }
 489                                                                         /*}}}*/
 490 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 491 // ---------------------------------------------------------------------
 492 /* This is used to create handles to strings. Given the same text it
 493    always returns the same number */
 494 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 495                                                  unsigned int Size)
 496 {
 497    /* We use a very small transient hash table here, this speeds up generation
 498       by a fair amount on slower machines */
 499    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 500    if (Bucket != 0 &&
 501        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 502       return Bucket->String;
 503
 504    // Search for an insertion point
 505    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 506    int Res = 1;
 507    map_ptrloc *Last = &Cache.HeaderP->StringList;
 508    for (; I != Cache.StringItemP; Last = &I->NextItem,
 509         I = Cache.StringItemP + I->NextItem)
 510    {
 511       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 512       if (Res >= 0)
 513          break;
 514    }
 515
 516    // Match
 517    if (Res == 0)
 518    {
 519       Bucket = I;
 520       return I->String;
 521    }
 522
 523    // Get a structure
 524    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 525    if (Item == 0)
 526       return 0;
 527
 528    // Fill in the structure
 529    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 530    ItemP->NextItem = I - Cache.StringItemP;
 531    *Last = Item;
 532    ItemP->String = Map.WriteString(S,Size);
 533    if (ItemP->String == 0)
 534       return 0;
 535
 536    Bucket = ItemP;
 537    return ItemP->String;
 538 }
 539                                                                         /*}}}*/
 540
 541 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 542 // ---------------------------------------------------------------------
 543 /* This just verifies that each file in the list of index files exists,
 544    has matching attributes with the cache and the cache does not have
 545    any extra files. */
 546 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 547                           FileIterator End,MMap **OutMap = 0)
 548 {
 549    // No file, certainly invalid
 550    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 551       return false;
 552
 553    // Map it
 554    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 555    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 556    pkgCache Cache(Map);
 557    if (_error->PendingError() == true || Map->Size() == 0)
 558    {
 559       _error->Discard();
 560       return false;
 561    }
 562
 563    /* Now we check every index file, see if it is in the cache,
 564       verify the IMS data and check that it is on the disk too.. */
 565    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 566    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 567    for (; Start != End; Start++)
 568    {
 569       if ((*Start)->HasPackages() == false)
 570          continue;
 571
 572       if ((*Start)->Exists() == false)
 573       {
 574 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 575          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 576                           (*Start)->Describe().c_str());
 577 #endif
 578          continue;
 579       }
 580
 581       // FindInCache is also expected to do an IMS check.
 582       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 583       if (File.end() == true)
 584          return false;
 585
 586       Visited[File->ID] = true;
 587    }
 588
 589    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 590       if (Visited[I] == false)
 591          return false;
 592
 593    if (_error->PendingError() == true)
 594    {
 595       _error->Discard();
 596       return false;
 597    }
 598
 599    if (OutMap != 0)
 600       *OutMap = Map.UnGuard();
 601    return true;
 602 }
 603                                                                         /*}}}*/
 604 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 605 // ---------------------------------------------------------------------
 606 /* Size is kind of an abstract notion that is only used for the progress
 607    meter */
 608 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 609 {
 610    unsigned long TotalSize = 0;
 611    for (; Start != End; Start++)
 612    {
 613       if ((*Start)->HasPackages() == false)
 614          continue;
 615       TotalSize += (*Start)->Size();
 616    }
 617    return TotalSize;
 618 }
 619                                                                         /*}}}*/
 620 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 621 // ---------------------------------------------------------------------
 622 /* */
 623 static bool BuildCache(pkgCacheGenerator &Gen,
 624                        OpProgress &Progress,
 625                        unsigned long &CurrentSize,unsigned long TotalSize,
 626                        FileIterator Start, FileIterator End)
 627 {
 628    FileIterator I;
 629    for (I = Start; I != End; I++)
 630    {
 631       if ((*I)->HasPackages() == false)
 632          continue;
 633
 634       if ((*I)->Exists() == false)
 635          continue;
 636
 637       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 638       {
 639          _error->Warning("Duplicate sources.list entry %s",
 640                          (*I)->Describe().c_str());
 641          continue;
 642       }
 643
 644       unsigned long Size = (*I)->Size();
 645       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 646       CurrentSize += Size;
 647
 648       if ((*I)->Merge(Gen,Progress) == false)
 649          return false;
 650    }
 651
 652    if (Gen.HasFileDeps() == true)
 653    {
 654       Progress.Done();
 655       TotalSize = ComputeSize(Start, End);
 656       CurrentSize = 0;
 657       for (I = Start; I != End; I++)
 658       {
 659          unsigned long Size = (*I)->Size();
 660          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 661          CurrentSize += Size;
 662          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 663             return false;
 664       }
 665    }
 666
 667    return true;
 668 }
 669                                                                         /*}}}*/
 670 // MakeStatusCache - Construct the status cache                         /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* This makes sure that the status cache (the cache that has all
 673    index files from the sources list and all local ones) is ready
 674    to be mmaped. If OutMap is not zero then a MMap object representing
 675    the cache will be stored there. This is pretty much mandetory if you
 676    are using AllowMem. AllowMem lets the function be run as non-root
 677    where it builds the cache 'fast' into a memory buffer. */
 678 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 679                         MMap **OutMap,bool AllowMem)
 680 {
 681    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 682
 683    vector<pkgIndexFile *> Files;
 684    for (vector<metaIndex *>::const_iterator i = List.begin();
 685         i != List.end();
 686         i++)
 687    {
 688       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 689       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 690            j != Indexes->end();
 691            j++)
 692          Files.push_back (*j);
 693    }
 694
 695    unsigned long EndOfSource = Files.size();
 696    if (_system->AddStatusFiles(Files) == false)
 697       return false;
 698
 699    // Decide if we can write to the files..
 700    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 701    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 702
 703    // Decide if we can write to the cache
 704    bool Writeable = false;
 705    if (CacheFile.empty() == false)
 706       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 707    else
 708       if (SrcCacheFile.empty() == false)
 709          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 710
 711    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 712       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 713
 714    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 715
 716    // Cache is OK, Fin.
 717    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 718    {
 719       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 720       return true;
 721    }
 722
 723    /* At this point we know we need to reconstruct the package cache,
 724       begin. */
 725    SPtr<FileFd> CacheF;
 726    SPtr<DynamicMMap> Map;
 727    if (Writeable == true && CacheFile.empty() == false)
 728    {
 729       unlink(CacheFile.c_str());
 730       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 731       fchmod(CacheF->Fd(),0644);
 732       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 733       if (_error->PendingError() == true)
 734          return false;
 735    }
 736    else
 737    {
 738       // Just build it in memory..
 739       Map = new DynamicMMap(MMap::Public,MapSize);
 740    }
 741
 742    // Lets try the source cache.
 743    unsigned long CurrentSize = 0;
 744    unsigned long TotalSize = 0;
 745    if (CheckValidity(SrcCacheFile,Files.begin(),
 746                      Files.begin()+EndOfSource) == true)
 747    {
 748       // Preload the map with the source cache
 749       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 750       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 751                        SCacheF.Size()) == false)
 752          return false;
 753
 754       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 755
 756       // Build the status cache
 757       pkgCacheGenerator Gen(Map.Get(),&Progress);
 758       if (_error->PendingError() == true)
 759          return false;
 760       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 761                      Files.begin()+EndOfSource,Files.end()) == false)
 762          return false;
 763    }
 764    else
 765    {
 766       TotalSize = ComputeSize(Files.begin(),Files.end());
 767
 768       // Build the source cache
 769       pkgCacheGenerator Gen(Map.Get(),&Progress);
 770       if (_error->PendingError() == true)
 771          return false;
 772       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 773                      Files.begin(),Files.begin()+EndOfSource) == false)
 774          return false;
 775
 776       // Write it back
 777       if (Writeable == true && SrcCacheFile.empty() == false)
 778       {
 779          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 780          if (_error->PendingError() == true)
 781             return false;
 782
 783          fchmod(SCacheF.Fd(),0644);
 784
 785          // Write out the main data
 786          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 787             return _error->Error(_("IO Error saving source cache"));
 788          SCacheF.Sync();
 789
 790          // Write out the proper header
 791          Gen.GetCache().HeaderP->Dirty = false;
 792          if (SCacheF.Seek(0) == false ||
 793              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 794             return _error->Error(_("IO Error saving source cache"));
 795          Gen.GetCache().HeaderP->Dirty = true;
 796          SCacheF.Sync();
 797       }
 798
 799       // Build the status cache
 800       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 801                      Files.begin()+EndOfSource,Files.end()) == false)
 802          return false;
 803    }
 804
 805    if (_error->PendingError() == true)
 806       return false;
 807    if (OutMap != 0)
 808    {
 809       if (CacheF != 0)
 810       {
 811          delete Map.UnGuard();
 812          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 813       }
 814       else
 815       {
 816          *OutMap = Map.UnGuard();
 817       }
 818    }
 819
 820    return true;
 821 }
 822                                                                         /*}}}*/
 823 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 824 // ---------------------------------------------------------------------
 825 /* */
 826 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 827 {
 828    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 829    vector<pkgIndexFile *> Files;
 830    unsigned long EndOfSource = Files.size();
 831    if (_system->AddStatusFiles(Files) == false)
 832       return false;
 833
 834    SPtr<DynamicMMap> Map;
 835    Map = new DynamicMMap(MMap::Public,MapSize);
 836    unsigned long CurrentSize = 0;
 837    unsigned long TotalSize = 0;
 838
 839    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 840
 841    // Build the status cache
 842    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 843    pkgCacheGenerator Gen(Map.Get(),&Progress);
 844    if (_error->PendingError() == true)
 845       return false;
 846    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 847                   Files.begin()+EndOfSource,Files.end()) == false)
 848       return false;
 849
 850    if (_error->PendingError() == true)
 851       return false;
 852    *OutMap = Map.UnGuard();
 853
 854    return true;
 855 }
 856                                                                         /*}}}*/