apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apt-pkg/tagfile.h>
  30
  31 #include <apti18n.h>
  32
  33 #include <vector>
  34
  35 #include <sys/stat.h>
  36 #include <unistd.h>
  37 #include <errno.h>
  38 #include <stdio.h>
  39 #include <system.h>
  40                                                                         /*}}}*/
  41 typedef vector<pkgIndexFile *>::iterator FileIterator;
  42
  43 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* We set the diry flag and make sure that is written to the disk */
  46 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  47                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  48                     FoundFileDeps(0)
  49 {
  50    CurrentFile = 0;
  51    memset(UniqHash,0,sizeof(UniqHash));
  52
  53    if (_error->PendingError() == true)
  54       return;
  55
  56    if (Map.Size() == 0)
  57    {
  58       // Setup the map interface..
  59       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  60       Map.RawAllocate(sizeof(pkgCache::Header));
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108
 109    unsigned int Counter = 0;
 110    while (List.Step() == true)
 111    {
 112       // Get a pointer to the package structure
 113       string PackageName = List.Package();
 114       if (PackageName.empty() == true)
 115          return false;
 116
 117       pkgCache::PkgIterator Pkg;
 118       if (NewPackage(Pkg,PackageName) == false)
 119          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 120       Counter++;
 121       if (Counter % 100 == 0 && Progress != 0)
 122          Progress->Progress(List.Offset());
 123
 124       /* Get a pointer to the version structure. We know the list is sorted
 125          so we use that fact in the search. Insertion of new versions is
 126          done with correct sorting */
 127       string Version = List.Version();
 128       if (Version.empty() == true)
 129       {
 130          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 131             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 132                                  PackageName.c_str());
 133          continue;
 134       }
 135
 136       pkgCache::VerIterator Ver = Pkg.VersionList();
 137       map_ptrloc *Last = &Pkg->VersionList;
 138       int Res = 1;
 139       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 140       {
 141          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 142          if (Res >= 0)
 143             break;
 144       }
 145
 146       /* We already have a version for this item, record that we
 147          saw it */
 148       unsigned long Hash = List.VersionHash();
 149       if (Res == 0 && Ver->Hash == Hash)
 150       {
 151          if (List.UsePackage(Pkg,Ver) == false)
 152             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 153                                  PackageName.c_str());
 154
 155          if (NewFileVer(Ver,List) == false)
 156             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 157                                  PackageName.c_str());
 158
 159          // Read only a single record and return
 160          if (OutVer != 0)
 161          {
 162             *OutVer = Ver;
 163             FoundFileDeps |= List.HasFileDeps();
 164             return true;
 165          }
 166
 167          continue;
 168       }
 169
 170       // Skip to the end of the same version set.
 171       if (Res == 0)
 172       {
 173          for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 174          {
 175             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 176             if (Res != 0)
 177                break;
 178          }
 179       }
 180
 181       // Add a new version
 182       *Last = NewVersion(Ver,Version,*Last);
 183       Ver->ParentPkg = Pkg.Index();
 184       Ver->Hash = Hash;
 185       if (List.NewVersion(Ver) == false)
 186          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 187                               PackageName.c_str());
 188
 189       if (List.UsePackage(Pkg,Ver) == false)
 190          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 191                               PackageName.c_str());
 192
 193       if (NewFileVer(Ver,List) == false)
 194          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 195                               PackageName.c_str());
 196
 197       // Read only a single record and return
 198       if (OutVer != 0)
 199       {
 200          *OutVer = Ver;
 201          FoundFileDeps |= List.HasFileDeps();
 202          return true;
 203       }
 204    }
 205
 206    FoundFileDeps |= List.HasFileDeps();
 207
 208    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 209       return _error->Error(_("Wow, you exceeded the number of package "
 210                              "names this APT is capable of."));
 211    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 212       return _error->Error(_("Wow, you exceeded the number of versions "
 213                              "this APT is capable of."));
 214    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 215       return _error->Error(_("Wow, you exceeded the number of dependencies "
 216                              "this APT is capable of."));
 217    return true;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 221 // ---------------------------------------------------------------------
 222 /* If we found any file depends while parsing the main list we need to
 223    resolve them. Since it is undesired to load the entire list of files
 224    into the cache as virtual packages we do a two stage effort. MergeList
 225    identifies the file depends and this creates Provdies for them by
 226    re-parsing all the indexs. */
 227 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 228 {
 229    List.Owner = this;
 230
 231    unsigned int Counter = 0;
 232    while (List.Step() == true)
 233    {
 234       string PackageName = List.Package();
 235       if (PackageName.empty() == true)
 236          return false;
 237       string Version = List.Version();
 238       if (Version.empty() == true)
 239          continue;
 240
 241       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 242       if (Pkg.end() == true)
 243          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 244                                 PackageName.c_str());
 245       Counter++;
 246       if (Counter % 100 == 0 && Progress != 0)
 247          Progress->Progress(List.Offset());
 248
 249       unsigned long Hash = List.VersionHash();
 250       pkgCache::VerIterator Ver = Pkg.VersionList();
 251       for (; Ver.end() == false; Ver++)
 252       {
 253          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 254          {
 255             if (List.CollectFileProvides(Cache,Ver) == false)
 256                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 257             break;
 258          }
 259       }
 260
 261       if (Ver.end() == true)
 262          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 263    }
 264
 265    return true;
 266 }
 267                                                                         /*}}}*/
 268 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 269 // ---------------------------------------------------------------------
 270 /* This creates a new package structure and adds it to the hash table */
 271 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 272 {
 273    Pkg = Cache.FindPkg(Name);
 274    if (Pkg.end() == false)
 275       return true;
 276
 277    // Get a structure
 278    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 279    if (Package == 0)
 280       return false;
 281
 282    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 283
 284    // Insert it into the hash table
 285    unsigned long Hash = Cache.Hash(Name);
 286    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 287    Cache.HeaderP->HashTable[Hash] = Package;
 288
 289    // Set the name and the ID
 290    Pkg->Name = Map.WriteString(Name);
 291    if (Pkg->Name == 0)
 292       return false;
 293    Pkg->ID = Cache.HeaderP->PackageCount++;
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 299 // ---------------------------------------------------------------------
 300 /* */
 301 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 302                                    ListParser &List)
 303 {
 304    if (CurrentFile == 0)
 305       return true;
 306
 307    // Get a structure
 308    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 309    if (VerFile == 0)
 310       return 0;
 311
 312    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 313    VF->File = CurrentFile - Cache.PkgFileP;
 314
 315    // Link it to the end of the list
 316    map_ptrloc *Last = &Ver->FileList;
 317    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 318       Last = &V->NextFile;
 319    VF->NextFile = *Last;
 320    *Last = VF.Index();
 321
 322    VF->Offset = List.Offset();
 323    VF->Size = List.Size();
 324    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 325       Cache.HeaderP->MaxVerFileSize = VF->Size;
 326    Cache.HeaderP->VerFileCount++;
 327
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* This puts a version structure in the linked list */
 334 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 335                                             const string &VerStr,
 336                                             unsigned long Next)
 337 {
 338    // Get a structure
 339    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 340    if (Version == 0)
 341       return 0;
 342
 343    // Fill it in
 344    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 345    Ver->NextVer = Next;
 346    Ver->ID = Cache.HeaderP->VersionCount++;
 347    Ver->VerStr = Map.WriteString(VerStr);
 348    if (Ver->VerStr == 0)
 349       return 0;
 350
 351    return Version;
 352 }
 353                                                                         /*}}}*/
 354 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 355 // ---------------------------------------------------------------------
 356 /* This creates a dependency element in the tree. It is linked to the
 357    version and to the package that it is pointing to. */
 358 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 359                                                const string &PackageName,
 360                                                const string &Version,
 361                                                unsigned int Op,
 362                                                unsigned int Type)
 363 {
 364    pkgCache &Cache = Owner->Cache;
 365
 366    // Get a structure
 367    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 368    if (Dependency == 0)
 369       return false;
 370
 371    // Fill it in
 372    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 373    Dep->ParentVer = Ver.Index();
 374    Dep->Type = Type;
 375    Dep->CompareOp = Op;
 376    Dep->ID = Cache.HeaderP->DependsCount++;
 377
 378    // Locate the target package
 379    pkgCache::PkgIterator Pkg;
 380    if (Owner->NewPackage(Pkg,PackageName) == false)
 381       return false;
 382
 383    // Probe the reverse dependency list for a version string that matches
 384    if (Version.empty() == false)
 385    {
 386 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 387          if (I->Version != 0 && I.TargetVer() == Version)
 388             Dep->Version = I->Version;*/
 389       if (Dep->Version == 0)
 390          if ((Dep->Version = WriteString(Version)) == 0)
 391             return false;
 392    }
 393
 394    // Link it to the package
 395    Dep->Package = Pkg.Index();
 396    Dep->NextRevDepends = Pkg->RevDepends;
 397    Pkg->RevDepends = Dep.Index();
 398
 399    /* Link it to the version (at the end of the list)
 400       Caching the old end point speeds up generation substantially */
 401    if (OldDepVer != Ver)
 402    {
 403       OldDepLast = &Ver->DependsList;
 404       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 405          OldDepLast = &D->NextDepends;
 406       OldDepVer = Ver;
 407    }
 408
 409    // Is it a file dependency?
 410    if (PackageName[0] == '/')
 411       FoundFileDeps = true;
 412
 413    Dep->NextDepends = *OldDepLast;
 414    *OldDepLast = Dep.Index();
 415    OldDepLast = &Dep->NextDepends;
 416
 417    return true;
 418 }
 419                                                                         /*}}}*/
 420 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 421 // ---------------------------------------------------------------------
 422 /* */
 423 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 424                                                 const string &PackageName,
 425                                                 const string &Version)
 426 {
 427    pkgCache &Cache = Owner->Cache;
 428
 429    // We do not add self referencing provides
 430    if (Ver.ParentPkg().Name() == PackageName)
 431       return true;
 432
 433    // Get a structure
 434    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 435    if (Provides == 0)
 436       return false;
 437    Cache.HeaderP->ProvidesCount++;
 438
 439    // Fill it in
 440    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 441    Prv->Version = Ver.Index();
 442    Prv->NextPkgProv = Ver->ProvidesList;
 443    Ver->ProvidesList = Prv.Index();
 444    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 445       return false;
 446
 447    // Locate the target package
 448    pkgCache::PkgIterator Pkg;
 449    if (Owner->NewPackage(Pkg,PackageName) == false)
 450       return false;
 451
 452    // Link it to the package
 453    Prv->ParentPkg = Pkg.Index();
 454    Prv->NextProvides = Pkg->ProvidesList;
 455    Pkg->ProvidesList = Prv.Index();
 456
 457    return true;
 458 }
 459                                                                         /*}}}*/
 460 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 461 // ---------------------------------------------------------------------
 462 /* This is used to select which file is to be associated with all newly
 463    added versions. The caller is responsible for setting the IMS fields. */
 464 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 465                                    const pkgIndexFile &Index,
 466                                    unsigned long Flags)
 467 {
 468    // Get some space for the structure
 469    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 470    if (CurrentFile == Cache.PkgFileP)
 471       return false;
 472
 473    // Fill it in
 474    CurrentFile->FileName = Map.WriteString(File);
 475    CurrentFile->Site = WriteUniqString(Site);
 476    CurrentFile->NextFile = Cache.HeaderP->FileList;
 477    CurrentFile->Flags = Flags;
 478    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 479    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 480    PkgFileName = File;
 481    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 482    Cache.HeaderP->PackageFileCount++;
 483
 484    if (CurrentFile->FileName == 0)
 485       return false;
 486
 487    if (Progress != 0)
 488       Progress->SubProgress(Index.Size());
 489    return true;
 490 }
 491                                                                         /*}}}*/
 492 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 493 // ---------------------------------------------------------------------
 494 /* This is used to create handles to strings. Given the same text it
 495    always returns the same number */
 496 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 497                                                  unsigned int Size)
 498 {
 499    /* We use a very small transient hash table here, this speeds up generation
 500       by a fair amount on slower machines */
 501    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 502    if (Bucket != 0 &&
 503        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 504       return Bucket->String;
 505
 506    // Search for an insertion point
 507    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 508    int Res = 1;
 509    map_ptrloc *Last = &Cache.HeaderP->StringList;
 510    for (; I != Cache.StringItemP; Last = &I->NextItem,
 511         I = Cache.StringItemP + I->NextItem)
 512    {
 513       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 514       if (Res >= 0)
 515          break;
 516    }
 517
 518    // Match
 519    if (Res == 0)
 520    {
 521       Bucket = I;
 522       return I->String;
 523    }
 524
 525    // Get a structure
 526    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 527    if (Item == 0)
 528       return 0;
 529
 530    // Fill in the structure
 531    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 532    ItemP->NextItem = I - Cache.StringItemP;
 533    *Last = Item;
 534    ItemP->String = Map.WriteString(S,Size);
 535    if (ItemP->String == 0)
 536       return 0;
 537
 538    Bucket = ItemP;
 539    return ItemP->String;
 540 }
 541                                                                         /*}}}*/
 542
 543 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* This just verifies that each file in the list of index files exists,
 546    has matching attributes with the cache and the cache does not have
 547    any extra files. */
 548 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 549                           FileIterator End,MMap **OutMap = 0)
 550 {
 551    // No file, certainly invalid
 552    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 553       return false;
 554
 555    // Map it
 556    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 557    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 558    pkgCache Cache(Map);
 559    if (_error->PendingError() == true || Map->Size() == 0)
 560    {
 561       _error->Discard();
 562       return false;
 563    }
 564
 565    /* Now we check every index file, see if it is in the cache,
 566       verify the IMS data and check that it is on the disk too.. */
 567    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 568    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 569    for (; Start != End; Start++)
 570    {
 571       if ((*Start)->HasPackages() == false)
 572          continue;
 573
 574       if ((*Start)->Exists() == false)
 575       {
 576 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 577          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 578                           (*Start)->Describe().c_str());
 579 #endif
 580          continue;
 581       }
 582
 583       // FindInCache is also expected to do an IMS check.
 584       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 585       if (File.end() == true)
 586          return false;
 587
 588       Visited[File->ID] = true;
 589    }
 590
 591    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 592       if (Visited[I] == false)
 593          return false;
 594
 595    if (_error->PendingError() == true)
 596    {
 597       _error->Discard();
 598       return false;
 599    }
 600
 601    if (OutMap != 0)
 602       *OutMap = Map.UnGuard();
 603    return true;
 604 }
 605                                                                         /*}}}*/
 606 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 607 // ---------------------------------------------------------------------
 608 /* Size is kind of an abstract notion that is only used for the progress
 609    meter */
 610 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 611 {
 612    unsigned long TotalSize = 0;
 613    for (; Start != End; Start++)
 614    {
 615       if ((*Start)->HasPackages() == false)
 616          continue;
 617       TotalSize += (*Start)->Size();
 618    }
 619    return TotalSize;
 620 }
 621                                                                         /*}}}*/
 622 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 623 // ---------------------------------------------------------------------
 624 /* */
 625 static bool BuildCache(pkgCacheGenerator &Gen,
 626                        OpProgress &Progress,
 627                        unsigned long &CurrentSize,unsigned long TotalSize,
 628                        FileIterator Start, FileIterator End)
 629 {
 630    FileIterator I;
 631    for (I = Start; I != End; I++)
 632    {
 633       if ((*I)->HasPackages() == false)
 634          continue;
 635
 636       if ((*I)->Exists() == false)
 637          continue;
 638
 639       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 640       {
 641          _error->Warning("Duplicate sources.list entry %s",
 642                          (*I)->Describe().c_str());
 643          continue;
 644       }
 645
 646       unsigned long Size = (*I)->Size();
 647       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 648       CurrentSize += Size;
 649
 650       if ((*I)->Merge(Gen,Progress) == false)
 651          return false;
 652    }
 653
 654    if (Gen.HasFileDeps() == true)
 655    {
 656       Progress.Done();
 657       TotalSize = ComputeSize(Start, End);
 658       CurrentSize = 0;
 659       for (I = Start; I != End; I++)
 660       {
 661          unsigned long Size = (*I)->Size();
 662          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 663          CurrentSize += Size;
 664          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 665             return false;
 666       }
 667    }
 668
 669    return true;
 670 }
 671                                                                         /*}}}*/
 672 // MakeStatusCache - Construct the status cache                         /*{{{*/
 673 // ---------------------------------------------------------------------
 674 /* This makes sure that the status cache (the cache that has all
 675    index files from the sources list and all local ones) is ready
 676    to be mmaped. If OutMap is not zero then a MMap object representing
 677    the cache will be stored there. This is pretty much mandetory if you
 678    are using AllowMem. AllowMem lets the function be run as non-root
 679    where it builds the cache 'fast' into a memory buffer. */
 680 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 681                         MMap **OutMap,bool AllowMem)
 682 {
 683    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 684
 685    vector<pkgIndexFile *> Files;
 686    for (vector<metaIndex *>::const_iterator i = List.begin();
 687         i != List.end();
 688         i++)
 689    {
 690       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 691       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 692            j != Indexes->end();
 693            j++)
 694          Files.push_back (*j);
 695    }
 696
 697    unsigned long EndOfSource = Files.size();
 698    if (_system->AddStatusFiles(Files) == false)
 699       return false;
 700
 701    // Decide if we can write to the files..
 702    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 703    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 704
 705    // Decide if we can write to the cache
 706    bool Writeable = false;
 707    if (CacheFile.empty() == false)
 708       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 709    else
 710       if (SrcCacheFile.empty() == false)
 711          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 712
 713    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 714       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 715
 716    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 717
 718    // Cache is OK, Fin.
 719    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 720    {
 721       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 722       return true;
 723    }
 724
 725    /* At this point we know we need to reconstruct the package cache,
 726       begin. */
 727    SPtr<FileFd> CacheF;
 728    SPtr<DynamicMMap> Map;
 729    if (Writeable == true && CacheFile.empty() == false)
 730    {
 731       unlink(CacheFile.c_str());
 732       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 733       fchmod(CacheF->Fd(),0644);
 734       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 735       if (_error->PendingError() == true)
 736          return false;
 737    }
 738    else
 739    {
 740       // Just build it in memory..
 741       Map = new DynamicMMap(MMap::Public,MapSize);
 742    }
 743
 744    // Lets try the source cache.
 745    unsigned long CurrentSize = 0;
 746    unsigned long TotalSize = 0;
 747    if (CheckValidity(SrcCacheFile,Files.begin(),
 748                      Files.begin()+EndOfSource) == true)
 749    {
 750       // Preload the map with the source cache
 751       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 752       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 753                        SCacheF.Size()) == false)
 754          return false;
 755
 756       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 757
 758       // Build the status cache
 759       pkgCacheGenerator Gen(Map.Get(),&Progress);
 760       if (_error->PendingError() == true)
 761          return false;
 762       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 763                      Files.begin()+EndOfSource,Files.end()) == false)
 764          return false;
 765    }
 766    else
 767    {
 768       TotalSize = ComputeSize(Files.begin(),Files.end());
 769
 770       // Build the source cache
 771       pkgCacheGenerator Gen(Map.Get(),&Progress);
 772       if (_error->PendingError() == true)
 773          return false;
 774       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 775                      Files.begin(),Files.begin()+EndOfSource) == false)
 776          return false;
 777
 778       // Write it back
 779       if (Writeable == true && SrcCacheFile.empty() == false)
 780       {
 781          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 782          if (_error->PendingError() == true)
 783             return false;
 784
 785          fchmod(SCacheF.Fd(),0644);
 786
 787          // Write out the main data
 788          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 789             return _error->Error(_("IO Error saving source cache"));
 790          SCacheF.Sync();
 791
 792          // Write out the proper header
 793          Gen.GetCache().HeaderP->Dirty = false;
 794          if (SCacheF.Seek(0) == false ||
 795              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 796             return _error->Error(_("IO Error saving source cache"));
 797          Gen.GetCache().HeaderP->Dirty = true;
 798          SCacheF.Sync();
 799       }
 800
 801       // Build the status cache
 802       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 803                      Files.begin()+EndOfSource,Files.end()) == false)
 804          return false;
 805    }
 806
 807    if (_error->PendingError() == true)
 808       return false;
 809    if (OutMap != 0)
 810    {
 811       if (CacheF != 0)
 812       {
 813          delete Map.UnGuard();
 814          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 815       }
 816       else
 817       {
 818          *OutMap = Map.UnGuard();
 819       }
 820    }
 821
 822    return true;
 823 }
 824                                                                         /*}}}*/
 825 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 826 // ---------------------------------------------------------------------
 827 /* */
 828 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 829 {
 830    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 831    vector<pkgIndexFile *> Files;
 832    unsigned long EndOfSource = Files.size();
 833    if (_system->AddStatusFiles(Files) == false)
 834       return false;
 835
 836    SPtr<DynamicMMap> Map;
 837    Map = new DynamicMMap(MMap::Public,MapSize);
 838    unsigned long CurrentSize = 0;
 839    unsigned long TotalSize = 0;
 840
 841    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 842
 843    // Build the status cache
 844    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 845    pkgCacheGenerator Gen(Map.Get(),&Progress);
 846    if (_error->PendingError() == true)
 847       return false;
 848    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 849                   Files.begin()+EndOfSource,Files.end()) == false)
 850       return false;
 851
 852    if (_error->PendingError() == true)
 853       return false;
 854    *OutMap = Map.UnGuard();
 855
 856    return true;
 857 }
 858                                                                         /*}}}*/