apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apt-pkg/tagfile.h>
  30
  31 #include <apti18n.h>
  32
  33 #include <vector>
  34
  35 #include <sys/stat.h>
  36 #include <unistd.h>
  37 #include <errno.h>
  38 #include <stdio.h>
  39 #include <system.h>
  40                                                                         /*}}}*/
  41 typedef vector<pkgIndexFile *>::iterator FileIterator;
  42
  43 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* We set the diry flag and make sure that is written to the disk */
  46 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  47                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  48                     FoundFileDeps(0)
  49 {
  50    CurrentFile = 0;
  51    memset(UniqHash,0,sizeof(UniqHash));
  52
  53    if (_error->PendingError() == true)
  54       return;
  55
  56    if (Map.Size() == 0)
  57    {
  58       // Setup the map interface..
  59       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  60       Map.RawAllocate(sizeof(pkgCache::Header));
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108
 109    unsigned int Counter = 0;
 110    while (List.Step() == true)
 111    {
 112       // Get a pointer to the package structure
 113       string PackageName = List.Package();
 114       if (PackageName.empty() == true)
 115          return false;
 116
 117       pkgCache::PkgIterator Pkg;
 118       if (NewPackage(Pkg,PackageName) == false)
 119          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 120       Counter++;
 121       if (Counter % 100 == 0 && Progress != 0)
 122          Progress->Progress(List.Offset());
 123
 124       /* Get a pointer to the version structure. We know the list is sorted
 125          so we use that fact in the search. Insertion of new versions is
 126          done with correct sorting */
 127       string Version = List.Version();
 128       if (Version.empty() == true)
 129       {
 130          // we first process the package, then the descriptions
 131          // (this has the bonus that we get MMap error when we run out
 132          //  of MMap space)
 133          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 134             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 135                                  PackageName.c_str());
 136
 137          // Find the right version to write the description
 138          MD5SumValue CurMd5 = List.Description_md5();
 139          pkgCache::VerIterator Ver = Pkg.VersionList();
 140          map_ptrloc *LastVer = &Pkg->VersionList;
 141
 142          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 143          {
 144             pkgCache::DescIterator Desc = Ver.DescriptionList();
 145             map_ptrloc *LastDesc = &Ver->DescriptionList;
 146
 147             for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
 148             {
 149
 150                if (MD5SumValue(Desc.md5()) == CurMd5)
 151                {
 152                   // Add new description
 153                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 154                   Desc->ParentPkg = Pkg.Index();
 155
 156                   if (NewFileDesc(Desc,List) == false)
 157                      return _error->Error(_("Error occured while processing %s (NewFileDesc1)"),PackageName.c_str());
 158                   break;
 159                }
 160             }
 161          }
 162
 163          continue;
 164       }
 165
 166       pkgCache::VerIterator Ver = Pkg.VersionList();
 167       map_ptrloc *LastVer = &Pkg->VersionList;
 168       int Res = 1;
 169       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 170       {
 171          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 172          if (Res >= 0)
 173             break;
 174       }
 175
 176       /* We already have a version for this item, record that we
 177          saw it */
 178       unsigned long Hash = List.VersionHash();
 179       if (Res == 0 && Ver->Hash == Hash)
 180       {
 181          if (List.UsePackage(Pkg,Ver) == false)
 182             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 183                                  PackageName.c_str());
 184
 185          if (NewFileVer(Ver,List) == false)
 186             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 187                                  PackageName.c_str());
 188
 189          // Read only a single record and return
 190          if (OutVer != 0)
 191          {
 192             *OutVer = Ver;
 193             FoundFileDeps |= List.HasFileDeps();
 194             return true;
 195          }
 196
 197          continue;
 198       }
 199
 200       // Skip to the end of the same version set.
 201       if (Res == 0)
 202       {
 203          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 204          {
 205             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 206             if (Res != 0)
 207                break;
 208          }
 209       }
 210
 211       // Add a new version
 212       *LastVer = NewVersion(Ver,Version,*LastVer);
 213       Ver->ParentPkg = Pkg.Index();
 214       Ver->Hash = Hash;
 215
 216       if (List.NewVersion(Ver) == false)
 217          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 218                               PackageName.c_str());
 219
 220       if (List.UsePackage(Pkg,Ver) == false)
 221          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 222                               PackageName.c_str());
 223
 224       if (NewFileVer(Ver,List) == false)
 225          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 226                               PackageName.c_str());
 227
 228       // Read only a single record and return
 229       if (OutVer != 0)
 230       {
 231          *OutVer = Ver;
 232          FoundFileDeps |= List.HasFileDeps();
 233          return true;
 234       }
 235
 236       /* Record the Description data. Description data always exist in
 237          Packages and Translation-* files. */
 238       pkgCache::DescIterator Desc = Ver.DescriptionList();
 239       map_ptrloc *LastDesc = &Ver->DescriptionList;
 240
 241       // Skip to the end of description set
 242       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 243
 244       // Add new description
 245       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 246       Desc->ParentPkg = Pkg.Index();
 247
 248       if (NewFileDesc(Desc,List) == false)
 249          return _error->Error(_("Error occured while processing %s (NewFileDesc2)"),PackageName.c_str());
 250    }
 251
 252    FoundFileDeps |= List.HasFileDeps();
 253
 254    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 255       return _error->Error(_("Wow, you exceeded the number of package "
 256                              "names this APT is capable of."));
 257    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 258       return _error->Error(_("Wow, you exceeded the number of versions "
 259                              "this APT is capable of."));
 260    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 261       return _error->Error(_("Wow, you exceeded the number of descriptions "
 262                              "this APT is capable of."));
 263    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 264       return _error->Error(_("Wow, you exceeded the number of dependencies "
 265                              "this APT is capable of."));
 266    return true;
 267 }
 268                                                                         /*}}}*/
 269 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* If we found any file depends while parsing the main list we need to
 272    resolve them. Since it is undesired to load the entire list of files
 273    into the cache as virtual packages we do a two stage effort. MergeList
 274    identifies the file depends and this creates Provdies for them by
 275    re-parsing all the indexs. */
 276 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 277 {
 278    List.Owner = this;
 279
 280    unsigned int Counter = 0;
 281    while (List.Step() == true)
 282    {
 283       string PackageName = List.Package();
 284       if (PackageName.empty() == true)
 285          return false;
 286       string Version = List.Version();
 287       if (Version.empty() == true)
 288          continue;
 289
 290       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 291       if (Pkg.end() == true)
 292          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 293                                 PackageName.c_str());
 294       Counter++;
 295       if (Counter % 100 == 0 && Progress != 0)
 296          Progress->Progress(List.Offset());
 297
 298       unsigned long Hash = List.VersionHash();
 299       pkgCache::VerIterator Ver = Pkg.VersionList();
 300       for (; Ver.end() == false; Ver++)
 301       {
 302          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 303          {
 304             if (List.CollectFileProvides(Cache,Ver) == false)
 305                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 306             break;
 307          }
 308       }
 309
 310       if (Ver.end() == true)
 311          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 312    }
 313
 314    return true;
 315 }
 316                                                                         /*}}}*/
 317 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 318 // ---------------------------------------------------------------------
 319 /* This creates a new package structure and adds it to the hash table */
 320 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 321 {
 322    Pkg = Cache.FindPkg(Name);
 323    if (Pkg.end() == false)
 324       return true;
 325
 326    // Get a structure
 327    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 328    if (Package == 0)
 329       return false;
 330
 331    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 332
 333    // Insert it into the hash table
 334    unsigned long Hash = Cache.Hash(Name);
 335    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 336    Cache.HeaderP->HashTable[Hash] = Package;
 337
 338    // Set the name and the ID
 339    Pkg->Name = Map.WriteString(Name);
 340    if (Pkg->Name == 0)
 341       return false;
 342    Pkg->ID = Cache.HeaderP->PackageCount++;
 343
 344    return true;
 345 }
 346                                                                         /*}}}*/
 347 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 348 // ---------------------------------------------------------------------
 349 /* */
 350 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 351                                    ListParser &List)
 352 {
 353    if (CurrentFile == 0)
 354       return true;
 355
 356    // Get a structure
 357    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 358    if (VerFile == 0)
 359       return 0;
 360
 361    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 362    VF->File = CurrentFile - Cache.PkgFileP;
 363
 364    // Link it to the end of the list
 365    map_ptrloc *Last = &Ver->FileList;
 366    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 367       Last = &V->NextFile;
 368    VF->NextFile = *Last;
 369    *Last = VF.Index();
 370
 371    VF->Offset = List.Offset();
 372    VF->Size = List.Size();
 373    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 374       Cache.HeaderP->MaxVerFileSize = VF->Size;
 375    Cache.HeaderP->VerFileCount++;
 376
 377    return true;
 378 }
 379                                                                         /*}}}*/
 380 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* This puts a version structure in the linked list */
 383 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 384                                             const string &VerStr,
 385                                             unsigned long Next)
 386 {
 387    // Get a structure
 388    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 389    if (Version == 0)
 390       return 0;
 391
 392    // Fill it in
 393    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 394    Ver->NextVer = Next;
 395    Ver->ID = Cache.HeaderP->VersionCount++;
 396    Ver->VerStr = Map.WriteString(VerStr);
 397    if (Ver->VerStr == 0)
 398       return 0;
 399
 400    return Version;
 401 }
 402                                                                         /*}}}*/
 403 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 404 // ---------------------------------------------------------------------
 405 /* */
 406 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 407                                    ListParser &List)
 408 {
 409    if (CurrentFile == 0)
 410       return true;
 411
 412    // Get a structure
 413    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 414    if (DescFile == 0)
 415       return 0;
 416
 417    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 418    DF->File = CurrentFile - Cache.PkgFileP;
 419
 420    // Link it to the end of the list
 421    map_ptrloc *Last = &Desc->FileList;
 422    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 423       Last = &D->NextFile;
 424
 425    DF->NextFile = *Last;
 426    *Last = DF.Index();
 427
 428    DF->Offset = List.Offset();
 429    DF->Size = List.Size();
 430    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 431       Cache.HeaderP->MaxDescFileSize = DF->Size;
 432    Cache.HeaderP->DescFileCount++;
 433
 434    return true;
 435 }
 436                                                                         /*}}}*/
 437 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 438 // ---------------------------------------------------------------------
 439 /* This puts a description structure in the linked list */
 440 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 441                                             const string &Lang, const MD5SumValue &md5sum,
 442                                             map_ptrloc Next)
 443 {
 444    // Get a structure
 445    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 446    if (Description == 0)
 447       return 0;
 448
 449    // Fill it in
 450    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 451    Desc->NextDesc = Next;
 452    Desc->ID = Cache.HeaderP->DescriptionCount++;
 453    Desc->language_code = Map.WriteString(Lang);
 454    Desc->md5sum = Map.WriteString(md5sum.Value());
 455
 456    return Description;
 457 }
 458                                                                         /*}}}*/
 459 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 460 // ---------------------------------------------------------------------
 461 /* This creates a dependency element in the tree. It is linked to the
 462    version and to the package that it is pointing to. */
 463 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 464                                                const string &PackageName,
 465                                                const string &Version,
 466                                                unsigned int Op,
 467                                                unsigned int Type)
 468 {
 469    pkgCache &Cache = Owner->Cache;
 470
 471    // Get a structure
 472    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 473    if (Dependency == 0)
 474       return false;
 475
 476    // Fill it in
 477    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 478    Dep->ParentVer = Ver.Index();
 479    Dep->Type = Type;
 480    Dep->CompareOp = Op;
 481    Dep->ID = Cache.HeaderP->DependsCount++;
 482
 483    // Locate the target package
 484    pkgCache::PkgIterator Pkg;
 485    if (Owner->NewPackage(Pkg,PackageName) == false)
 486       return false;
 487
 488    // Probe the reverse dependency list for a version string that matches
 489    if (Version.empty() == false)
 490    {
 491 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 492          if (I->Version != 0 && I.TargetVer() == Version)
 493             Dep->Version = I->Version;*/
 494       if (Dep->Version == 0)
 495          if ((Dep->Version = WriteString(Version)) == 0)
 496             return false;
 497    }
 498
 499    // Link it to the package
 500    Dep->Package = Pkg.Index();
 501    Dep->NextRevDepends = Pkg->RevDepends;
 502    Pkg->RevDepends = Dep.Index();
 503
 504    /* Link it to the version (at the end of the list)
 505       Caching the old end point speeds up generation substantially */
 506    if (OldDepVer != Ver)
 507    {
 508       OldDepLast = &Ver->DependsList;
 509       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 510          OldDepLast = &D->NextDepends;
 511       OldDepVer = Ver;
 512    }
 513
 514    // Is it a file dependency?
 515    if (PackageName[0] == '/')
 516       FoundFileDeps = true;
 517
 518    Dep->NextDepends = *OldDepLast;
 519    *OldDepLast = Dep.Index();
 520    OldDepLast = &Dep->NextDepends;
 521
 522    return true;
 523 }
 524                                                                         /*}}}*/
 525 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 526 // ---------------------------------------------------------------------
 527 /* */
 528 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 529                                                 const string &PackageName,
 530                                                 const string &Version)
 531 {
 532    pkgCache &Cache = Owner->Cache;
 533
 534    // We do not add self referencing provides
 535    if (Ver.ParentPkg().Name() == PackageName)
 536       return true;
 537
 538    // Get a structure
 539    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 540    if (Provides == 0)
 541       return false;
 542    Cache.HeaderP->ProvidesCount++;
 543
 544    // Fill it in
 545    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 546    Prv->Version = Ver.Index();
 547    Prv->NextPkgProv = Ver->ProvidesList;
 548    Ver->ProvidesList = Prv.Index();
 549    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 550       return false;
 551
 552    // Locate the target package
 553    pkgCache::PkgIterator Pkg;
 554    if (Owner->NewPackage(Pkg,PackageName) == false)
 555       return false;
 556
 557    // Link it to the package
 558    Prv->ParentPkg = Pkg.Index();
 559    Prv->NextProvides = Pkg->ProvidesList;
 560    Pkg->ProvidesList = Prv.Index();
 561
 562    return true;
 563 }
 564                                                                         /*}}}*/
 565 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 566 // ---------------------------------------------------------------------
 567 /* This is used to select which file is to be associated with all newly
 568    added versions. The caller is responsible for setting the IMS fields. */
 569 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 570                                    const pkgIndexFile &Index,
 571                                    unsigned long Flags)
 572 {
 573    // Get some space for the structure
 574    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 575    if (CurrentFile == Cache.PkgFileP)
 576       return false;
 577
 578    // Fill it in
 579    CurrentFile->FileName = Map.WriteString(File);
 580    CurrentFile->Site = WriteUniqString(Site);
 581    CurrentFile->NextFile = Cache.HeaderP->FileList;
 582    CurrentFile->Flags = Flags;
 583    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 584    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 585    PkgFileName = File;
 586    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 587    Cache.HeaderP->PackageFileCount++;
 588
 589    if (CurrentFile->FileName == 0)
 590       return false;
 591
 592    if (Progress != 0)
 593       Progress->SubProgress(Index.Size());
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This is used to create handles to strings. Given the same text it
 600    always returns the same number */
 601 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 602                                                  unsigned int Size)
 603 {
 604    /* We use a very small transient hash table here, this speeds up generation
 605       by a fair amount on slower machines */
 606    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 607    if (Bucket != 0 &&
 608        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 609       return Bucket->String;
 610
 611    // Search for an insertion point
 612    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 613    int Res = 1;
 614    map_ptrloc *Last = &Cache.HeaderP->StringList;
 615    for (; I != Cache.StringItemP; Last = &I->NextItem,
 616         I = Cache.StringItemP + I->NextItem)
 617    {
 618       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 619       if (Res >= 0)
 620          break;
 621    }
 622
 623    // Match
 624    if (Res == 0)
 625    {
 626       Bucket = I;
 627       return I->String;
 628    }
 629
 630    // Get a structure
 631    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 632    if (Item == 0)
 633       return 0;
 634
 635    // Fill in the structure
 636    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 637    ItemP->NextItem = I - Cache.StringItemP;
 638    *Last = Item;
 639    ItemP->String = Map.WriteString(S,Size);
 640    if (ItemP->String == 0)
 641       return 0;
 642
 643    Bucket = ItemP;
 644    return ItemP->String;
 645 }
 646                                                                         /*}}}*/
 647
 648 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 649 // ---------------------------------------------------------------------
 650 /* This just verifies that each file in the list of index files exists,
 651    has matching attributes with the cache and the cache does not have
 652    any extra files. */
 653 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 654                           FileIterator End,MMap **OutMap = 0)
 655 {
 656    // No file, certainly invalid
 657    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 658       return false;
 659
 660    // Map it
 661    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 662    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 663    pkgCache Cache(Map);
 664    if (_error->PendingError() == true || Map->Size() == 0)
 665    {
 666       _error->Discard();
 667       return false;
 668    }
 669
 670    /* Now we check every index file, see if it is in the cache,
 671       verify the IMS data and check that it is on the disk too.. */
 672    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 673    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 674    for (; Start != End; Start++)
 675    {
 676       if ((*Start)->HasPackages() == false)
 677          continue;
 678
 679       if ((*Start)->Exists() == false)
 680       {
 681          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 682                           (*Start)->Describe().c_str());
 683          continue;
 684       }
 685
 686       // FindInCache is also expected to do an IMS check.
 687       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 688       if (File.end() == true)
 689          return false;
 690
 691       Visited[File->ID] = true;
 692    }
 693
 694    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 695       if (Visited[I] == false)
 696          return false;
 697
 698    if (_error->PendingError() == true)
 699    {
 700       _error->Discard();
 701       return false;
 702    }
 703
 704    if (OutMap != 0)
 705       *OutMap = Map.UnGuard();
 706    return true;
 707 }
 708                                                                         /*}}}*/
 709 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 710 // ---------------------------------------------------------------------
 711 /* Size is kind of an abstract notion that is only used for the progress
 712    meter */
 713 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 714 {
 715    unsigned long TotalSize = 0;
 716    for (; Start != End; Start++)
 717    {
 718       if ((*Start)->HasPackages() == false)
 719          continue;
 720       TotalSize += (*Start)->Size();
 721    }
 722    return TotalSize;
 723 }
 724                                                                         /*}}}*/
 725 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 726 // ---------------------------------------------------------------------
 727 /* */
 728 static bool BuildCache(pkgCacheGenerator &Gen,
 729                        OpProgress &Progress,
 730                        unsigned long &CurrentSize,unsigned long TotalSize,
 731                        FileIterator Start, FileIterator End)
 732 {
 733    FileIterator I;
 734    for (I = Start; I != End; I++)
 735    {
 736       if ((*I)->HasPackages() == false)
 737          continue;
 738
 739       if ((*I)->Exists() == false)
 740          continue;
 741
 742       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 743       {
 744          _error->Warning("Duplicate sources.list entry %s",
 745                          (*I)->Describe().c_str());
 746          continue;
 747       }
 748
 749       unsigned long Size = (*I)->Size();
 750       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 751       CurrentSize += Size;
 752
 753       if ((*I)->Merge(Gen,Progress) == false)
 754          return false;
 755    }
 756
 757    if (Gen.HasFileDeps() == true)
 758    {
 759       Progress.Done();
 760       TotalSize = ComputeSize(Start, End);
 761       CurrentSize = 0;
 762       for (I = Start; I != End; I++)
 763       {
 764          unsigned long Size = (*I)->Size();
 765          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 766          CurrentSize += Size;
 767          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 768             return false;
 769       }
 770    }
 771
 772    return true;
 773 }
 774                                                                         /*}}}*/
 775 // MakeStatusCache - Construct the status cache                         /*{{{*/
 776 // ---------------------------------------------------------------------
 777 /* This makes sure that the status cache (the cache that has all
 778    index files from the sources list and all local ones) is ready
 779    to be mmaped. If OutMap is not zero then a MMap object representing
 780    the cache will be stored there. This is pretty much mandetory if you
 781    are using AllowMem. AllowMem lets the function be run as non-root
 782    where it builds the cache 'fast' into a memory buffer. */
 783 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 784                         MMap **OutMap,bool AllowMem)
 785 {
 786    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 787
 788    vector<pkgIndexFile *> Files;
 789    for (vector<metaIndex *>::const_iterator i = List.begin();
 790         i != List.end();
 791         i++)
 792    {
 793       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 794       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 795            j != Indexes->end();
 796            j++)
 797          Files.push_back (*j);
 798    }
 799
 800    unsigned long EndOfSource = Files.size();
 801    if (_system->AddStatusFiles(Files) == false)
 802       return false;
 803
 804    // Decide if we can write to the files..
 805    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 806    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 807
 808    // Decide if we can write to the cache
 809    bool Writeable = false;
 810    if (CacheFile.empty() == false)
 811       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 812    else
 813       if (SrcCacheFile.empty() == false)
 814          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 815
 816    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 817       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 818
 819    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 820
 821    // Cache is OK, Fin.
 822    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 823    {
 824       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 825       return true;
 826    }
 827
 828    /* At this point we know we need to reconstruct the package cache,
 829       begin. */
 830    SPtr<FileFd> CacheF;
 831    SPtr<DynamicMMap> Map;
 832    if (Writeable == true && CacheFile.empty() == false)
 833    {
 834       unlink(CacheFile.c_str());
 835       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 836       fchmod(CacheF->Fd(),0644);
 837       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 838       if (_error->PendingError() == true)
 839          return false;
 840    }
 841    else
 842    {
 843       // Just build it in memory..
 844       Map = new DynamicMMap(MMap::Public,MapSize);
 845    }
 846
 847    // Lets try the source cache.
 848    unsigned long CurrentSize = 0;
 849    unsigned long TotalSize = 0;
 850    if (CheckValidity(SrcCacheFile,Files.begin(),
 851                      Files.begin()+EndOfSource) == true)
 852    {
 853       // Preload the map with the source cache
 854       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 855       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 856                        SCacheF.Size()) == false)
 857          return false;
 858
 859       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 860
 861       // Build the status cache
 862       pkgCacheGenerator Gen(Map.Get(),&Progress);
 863       if (_error->PendingError() == true)
 864          return false;
 865       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 866                      Files.begin()+EndOfSource,Files.end()) == false)
 867          return false;
 868    }
 869    else
 870    {
 871       TotalSize = ComputeSize(Files.begin(),Files.end());
 872
 873       // Build the source cache
 874       pkgCacheGenerator Gen(Map.Get(),&Progress);
 875       if (_error->PendingError() == true)
 876          return false;
 877       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 878                      Files.begin(),Files.begin()+EndOfSource) == false)
 879          return false;
 880
 881       // Write it back
 882       if (Writeable == true && SrcCacheFile.empty() == false)
 883       {
 884          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 885          if (_error->PendingError() == true)
 886             return false;
 887
 888          fchmod(SCacheF.Fd(),0644);
 889
 890          // Write out the main data
 891          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 892             return _error->Error(_("IO Error saving source cache"));
 893          SCacheF.Sync();
 894
 895          // Write out the proper header
 896          Gen.GetCache().HeaderP->Dirty = false;
 897          if (SCacheF.Seek(0) == false ||
 898              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 899             return _error->Error(_("IO Error saving source cache"));
 900          Gen.GetCache().HeaderP->Dirty = true;
 901          SCacheF.Sync();
 902       }
 903
 904       // Build the status cache
 905       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 906                      Files.begin()+EndOfSource,Files.end()) == false)
 907          return false;
 908    }
 909
 910    if (_error->PendingError() == true)
 911       return false;
 912    if (OutMap != 0)
 913    {
 914       if (CacheF != 0)
 915       {
 916          delete Map.UnGuard();
 917          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 918       }
 919       else
 920       {
 921          *OutMap = Map.UnGuard();
 922       }
 923    }
 924
 925    return true;
 926 }
 927                                                                         /*}}}*/
 928 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 929 // ---------------------------------------------------------------------
 930 /* */
 931 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 932 {
 933    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 934    vector<pkgIndexFile *> Files;
 935    unsigned long EndOfSource = Files.size();
 936    if (_system->AddStatusFiles(Files) == false)
 937       return false;
 938
 939    SPtr<DynamicMMap> Map;
 940    Map = new DynamicMMap(MMap::Public,MapSize);
 941    unsigned long CurrentSize = 0;
 942    unsigned long TotalSize = 0;
 943
 944    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 945
 946    // Build the status cache
 947    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 948    pkgCacheGenerator Gen(Map.Get(),&Progress);
 949    if (_error->PendingError() == true)
 950       return false;
 951    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 952                   Files.begin()+EndOfSource,Files.end()) == false)
 953       return false;
 954
 955    if (_error->PendingError() == true)
 956       return false;
 957    *OutMap = Map.UnGuard();
 958
 959    return true;
 960 }
 961                                                                         /*}}}*/