apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37 #include <system.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       Map.RawAllocate(sizeof(pkgCache::Header));
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          // we first process the package, then the descriptions
 129          // (this has the bonus that we get MMap error when we run out
 130          //  of MMap space)
 131          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 132             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 133                                  PackageName.c_str());
 134
 135          // Find the right version to write the description
 136          MD5SumValue CurMd5 = List.Description_md5();
 137          pkgCache::VerIterator Ver = Pkg.VersionList();
 138          map_ptrloc *LastVer = &Pkg->VersionList;
 139
 140          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 141          {
 142             pkgCache::DescIterator Desc = Ver.DescriptionList();
 143             map_ptrloc *LastDesc = &Ver->DescriptionList;
 144
 145             for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
 146             {
 147
 148                if (MD5SumValue(Desc.md5()) == CurMd5)
 149                {
 150                   // Add new description
 151                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 152                   Desc->ParentPkg = Pkg.Index();
 153
 154                   if (NewFileDesc(Desc,List) == false)
 155                      return _error->Error(_("Error occured while processing %s (NewFileDesc1)"),PackageName.c_str());
 156                   break;
 157                }
 158             }
 159          }
 160
 161          continue;
 162       }
 163
 164       pkgCache::VerIterator Ver = Pkg.VersionList();
 165       map_ptrloc *LastVer = &Pkg->VersionList;
 166       int Res = 1;
 167       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 168       {
 169          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 170          if (Res >= 0)
 171             break;
 172       }
 173
 174       /* We already have a version for this item, record that we
 175          saw it */
 176       unsigned long Hash = List.VersionHash();
 177       if (Res == 0 && Ver->Hash == Hash)
 178       {
 179          if (List.UsePackage(Pkg,Ver) == false)
 180             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 181                                  PackageName.c_str());
 182
 183          if (NewFileVer(Ver,List) == false)
 184             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 185                                  PackageName.c_str());
 186
 187          // Read only a single record and return
 188          if (OutVer != 0)
 189          {
 190             *OutVer = Ver;
 191             FoundFileDeps |= List.HasFileDeps();
 192             return true;
 193          }
 194
 195          continue;
 196       }
 197
 198       // Skip to the end of the same version set.
 199       if (Res == 0)
 200       {
 201          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 202          {
 203             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 204             if (Res != 0)
 205                break;
 206          }
 207       }
 208
 209       // Add a new version
 210       *LastVer = NewVersion(Ver,Version,*LastVer);
 211       Ver->ParentPkg = Pkg.Index();
 212       Ver->Hash = Hash;
 213
 214       if (List.NewVersion(Ver) == false)
 215          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 216                               PackageName.c_str());
 217
 218       if (List.UsePackage(Pkg,Ver) == false)
 219          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 220                               PackageName.c_str());
 221
 222       if (NewFileVer(Ver,List) == false)
 223          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 224                               PackageName.c_str());
 225
 226       // Read only a single record and return
 227       if (OutVer != 0)
 228       {
 229          *OutVer = Ver;
 230          FoundFileDeps |= List.HasFileDeps();
 231          return true;
 232       }
 233
 234       /* Record the Description data. Description data always exist in
 235          Packages and Translation-* files. */
 236       pkgCache::DescIterator Desc = Ver.DescriptionList();
 237       map_ptrloc *LastDesc = &Ver->DescriptionList;
 238
 239       // Skip to the end of description set
 240       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 241
 242       // Add new description
 243       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 244       Desc->ParentPkg = Pkg.Index();
 245
 246       if (NewFileDesc(Desc,List) == false)
 247          return _error->Error(_("Error occured while processing %s (NewFileDesc2)"),PackageName.c_str());
 248    }
 249
 250    FoundFileDeps |= List.HasFileDeps();
 251
 252    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 253       return _error->Error(_("Wow, you exceeded the number of package "
 254                              "names this APT is capable of."));
 255    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 256       return _error->Error(_("Wow, you exceeded the number of versions "
 257                              "this APT is capable of."));
 258    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 259       return _error->Error(_("Wow, you exceeded the number of descriptions "
 260                              "this APT is capable of."));
 261    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 262       return _error->Error(_("Wow, you exceeded the number of dependencies "
 263                              "this APT is capable of."));
 264    return true;
 265 }
 266                                                                         /*}}}*/
 267 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 268 // ---------------------------------------------------------------------
 269 /* If we found any file depends while parsing the main list we need to
 270    resolve them. Since it is undesired to load the entire list of files
 271    into the cache as virtual packages we do a two stage effort. MergeList
 272    identifies the file depends and this creates Provdies for them by
 273    re-parsing all the indexs. */
 274 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 275 {
 276    List.Owner = this;
 277
 278    unsigned int Counter = 0;
 279    while (List.Step() == true)
 280    {
 281       string PackageName = List.Package();
 282       if (PackageName.empty() == true)
 283          return false;
 284       string Version = List.Version();
 285       if (Version.empty() == true)
 286          continue;
 287
 288       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 289       if (Pkg.end() == true)
 290          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 291                                 PackageName.c_str());
 292       Counter++;
 293       if (Counter % 100 == 0 && Progress != 0)
 294          Progress->Progress(List.Offset());
 295
 296       unsigned long Hash = List.VersionHash();
 297       pkgCache::VerIterator Ver = Pkg.VersionList();
 298       for (; Ver.end() == false; Ver++)
 299       {
 300          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 301          {
 302             if (List.CollectFileProvides(Cache,Ver) == false)
 303                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 304             break;
 305          }
 306       }
 307
 308       if (Ver.end() == true)
 309          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 310    }
 311
 312    return true;
 313 }
 314                                                                         /*}}}*/
 315 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 316 // ---------------------------------------------------------------------
 317 /* This creates a new package structure and adds it to the hash table */
 318 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 319 {
 320    Pkg = Cache.FindPkg(Name);
 321    if (Pkg.end() == false)
 322       return true;
 323
 324    // Get a structure
 325    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 326    if (Package == 0)
 327       return false;
 328
 329    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 330
 331    // Insert it into the hash table
 332    unsigned long Hash = Cache.Hash(Name);
 333    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 334    Cache.HeaderP->HashTable[Hash] = Package;
 335
 336    // Set the name and the ID
 337    Pkg->Name = Map.WriteString(Name);
 338    if (Pkg->Name == 0)
 339       return false;
 340    Pkg->ID = Cache.HeaderP->PackageCount++;
 341
 342    return true;
 343 }
 344                                                                         /*}}}*/
 345 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 346 // ---------------------------------------------------------------------
 347 /* */
 348 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 349                                    ListParser &List)
 350 {
 351    if (CurrentFile == 0)
 352       return true;
 353
 354    // Get a structure
 355    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 356    if (VerFile == 0)
 357       return 0;
 358
 359    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 360    VF->File = CurrentFile - Cache.PkgFileP;
 361
 362    // Link it to the end of the list
 363    map_ptrloc *Last = &Ver->FileList;
 364    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 365       Last = &V->NextFile;
 366    VF->NextFile = *Last;
 367    *Last = VF.Index();
 368
 369    VF->Offset = List.Offset();
 370    VF->Size = List.Size();
 371    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 372       Cache.HeaderP->MaxVerFileSize = VF->Size;
 373    Cache.HeaderP->VerFileCount++;
 374
 375    return true;
 376 }
 377                                                                         /*}}}*/
 378 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 379 // ---------------------------------------------------------------------
 380 /* This puts a version structure in the linked list */
 381 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 382                                             string VerStr,
 383                                             unsigned long Next)
 384 {
 385    // Get a structure
 386    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 387    if (Version == 0)
 388       return 0;
 389
 390    // Fill it in
 391    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 392    Ver->NextVer = Next;
 393    Ver->ID = Cache.HeaderP->VersionCount++;
 394    Ver->VerStr = Map.WriteString(VerStr);
 395    if (Ver->VerStr == 0)
 396       return 0;
 397
 398    return Version;
 399 }
 400                                                                         /*}}}*/
 401 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 402 // ---------------------------------------------------------------------
 403 /* */
 404 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 405                                    ListParser &List)
 406 {
 407    if (CurrentFile == 0)
 408       return true;
 409
 410    // Get a structure
 411    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 412    if (DescFile == 0)
 413       return 0;
 414
 415    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 416    DF->File = CurrentFile - Cache.PkgFileP;
 417
 418    // Link it to the end of the list
 419    map_ptrloc *Last = &Desc->FileList;
 420    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 421       Last = &D->NextFile;
 422
 423    DF->NextFile = *Last;
 424    *Last = DF.Index();
 425
 426    DF->Offset = List.Offset();
 427    DF->Size = List.Size();
 428    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 429       Cache.HeaderP->MaxDescFileSize = DF->Size;
 430    Cache.HeaderP->DescFileCount++;
 431
 432    return true;
 433 }
 434                                                                         /*}}}*/
 435 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 436 // ---------------------------------------------------------------------
 437 /* This puts a description structure in the linked list */
 438 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 439                                             const string &Lang, const MD5SumValue &md5sum,
 440                                             map_ptrloc Next)
 441 {
 442    // Get a structure
 443    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 444    if (Description == 0)
 445       return 0;
 446
 447    // Fill it in
 448    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 449    Desc->NextDesc = Next;
 450    Desc->ID = Cache.HeaderP->DescriptionCount++;
 451    Desc->language_code = Map.WriteString(Lang);
 452    Desc->md5sum = Map.WriteString(md5sum.Value());
 453
 454    return Description;
 455 }
 456                                                                         /*}}}*/
 457 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 458 // ---------------------------------------------------------------------
 459 /* This creates a dependency element in the tree. It is linked to the
 460    version and to the package that it is pointing to. */
 461 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 462                                                string PackageName,
 463                                                string Version,
 464                                                unsigned int Op,
 465                                                unsigned int Type)
 466 {
 467    pkgCache &Cache = Owner->Cache;
 468
 469    // Get a structure
 470    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 471    if (Dependency == 0)
 472       return false;
 473
 474    // Fill it in
 475    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 476    Dep->ParentVer = Ver.Index();
 477    Dep->Type = Type;
 478    Dep->CompareOp = Op;
 479    Dep->ID = Cache.HeaderP->DependsCount++;
 480
 481    // Locate the target package
 482    pkgCache::PkgIterator Pkg;
 483    if (Owner->NewPackage(Pkg,PackageName) == false)
 484       return false;
 485
 486    // Probe the reverse dependency list for a version string that matches
 487    if (Version.empty() == false)
 488    {
 489 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 490          if (I->Version != 0 && I.TargetVer() == Version)
 491             Dep->Version = I->Version;*/
 492       if (Dep->Version == 0)
 493          if ((Dep->Version = WriteString(Version)) == 0)
 494             return false;
 495    }
 496
 497    // Link it to the package
 498    Dep->Package = Pkg.Index();
 499    Dep->NextRevDepends = Pkg->RevDepends;
 500    Pkg->RevDepends = Dep.Index();
 501
 502    /* Link it to the version (at the end of the list)
 503       Caching the old end point speeds up generation substantially */
 504    if (OldDepVer != Ver)
 505    {
 506       OldDepLast = &Ver->DependsList;
 507       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 508          OldDepLast = &D->NextDepends;
 509       OldDepVer = Ver;
 510    }
 511
 512    // Is it a file dependency?
 513    if (PackageName[0] == '/')
 514       FoundFileDeps = true;
 515
 516    Dep->NextDepends = *OldDepLast;
 517    *OldDepLast = Dep.Index();
 518    OldDepLast = &Dep->NextDepends;
 519
 520    return true;
 521 }
 522                                                                         /*}}}*/
 523 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 524 // ---------------------------------------------------------------------
 525 /* */
 526 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 527                                                 string PackageName,
 528                                                 string Version)
 529 {
 530    pkgCache &Cache = Owner->Cache;
 531
 532    // We do not add self referencing provides
 533    if (Ver.ParentPkg().Name() == PackageName)
 534       return true;
 535
 536    // Get a structure
 537    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 538    if (Provides == 0)
 539       return false;
 540    Cache.HeaderP->ProvidesCount++;
 541
 542    // Fill it in
 543    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 544    Prv->Version = Ver.Index();
 545    Prv->NextPkgProv = Ver->ProvidesList;
 546    Ver->ProvidesList = Prv.Index();
 547    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 548       return false;
 549
 550    // Locate the target package
 551    pkgCache::PkgIterator Pkg;
 552    if (Owner->NewPackage(Pkg,PackageName) == false)
 553       return false;
 554
 555    // Link it to the package
 556    Prv->ParentPkg = Pkg.Index();
 557    Prv->NextProvides = Pkg->ProvidesList;
 558    Pkg->ProvidesList = Prv.Index();
 559
 560    return true;
 561 }
 562                                                                         /*}}}*/
 563 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 564 // ---------------------------------------------------------------------
 565 /* This is used to select which file is to be associated with all newly
 566    added versions. The caller is responsible for setting the IMS fields. */
 567 bool pkgCacheGenerator::SelectFile(string File,string Site,
 568                                    const pkgIndexFile &Index,
 569                                    unsigned long Flags)
 570 {
 571    // Get some space for the structure
 572    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 573    if (CurrentFile == Cache.PkgFileP)
 574       return false;
 575
 576    // Fill it in
 577    CurrentFile->FileName = Map.WriteString(File);
 578    CurrentFile->Site = WriteUniqString(Site);
 579    CurrentFile->NextFile = Cache.HeaderP->FileList;
 580    CurrentFile->Flags = Flags;
 581    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 582    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 583    PkgFileName = File;
 584    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 585    Cache.HeaderP->PackageFileCount++;
 586
 587    if (CurrentFile->FileName == 0)
 588       return false;
 589
 590    if (Progress != 0)
 591       Progress->SubProgress(Index.Size());
 592    return true;
 593 }
 594                                                                         /*}}}*/
 595 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 596 // ---------------------------------------------------------------------
 597 /* This is used to create handles to strings. Given the same text it
 598    always returns the same number */
 599 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 600                                                  unsigned int Size)
 601 {
 602    /* We use a very small transient hash table here, this speeds up generation
 603       by a fair amount on slower machines */
 604    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 605    if (Bucket != 0 &&
 606        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 607       return Bucket->String;
 608
 609    // Search for an insertion point
 610    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 611    int Res = 1;
 612    map_ptrloc *Last = &Cache.HeaderP->StringList;
 613    for (; I != Cache.StringItemP; Last = &I->NextItem,
 614         I = Cache.StringItemP + I->NextItem)
 615    {
 616       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 617       if (Res >= 0)
 618          break;
 619    }
 620
 621    // Match
 622    if (Res == 0)
 623    {
 624       Bucket = I;
 625       return I->String;
 626    }
 627
 628    // Get a structure
 629    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 630    if (Item == 0)
 631       return 0;
 632
 633    // Fill in the structure
 634    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 635    ItemP->NextItem = I - Cache.StringItemP;
 636    *Last = Item;
 637    ItemP->String = Map.WriteString(S,Size);
 638    if (ItemP->String == 0)
 639       return 0;
 640
 641    Bucket = ItemP;
 642    return ItemP->String;
 643 }
 644                                                                         /*}}}*/
 645
 646 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 647 // ---------------------------------------------------------------------
 648 /* This just verifies that each file in the list of index files exists,
 649    has matching attributes with the cache and the cache does not have
 650    any extra files. */
 651 static bool CheckValidity(string CacheFile, FileIterator Start,
 652                           FileIterator End,MMap **OutMap = 0)
 653 {
 654    // No file, certainly invalid
 655    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 656       return false;
 657
 658    // Map it
 659    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 660    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 661    pkgCache Cache(Map);
 662    if (_error->PendingError() == true || Map->Size() == 0)
 663    {
 664       _error->Discard();
 665       return false;
 666    }
 667
 668    /* Now we check every index file, see if it is in the cache,
 669       verify the IMS data and check that it is on the disk too.. */
 670    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 671    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 672    for (; Start != End; Start++)
 673    {
 674       if ((*Start)->HasPackages() == false)
 675          continue;
 676
 677       if ((*Start)->Exists() == false)
 678       {
 679          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 680                           (*Start)->Describe().c_str());
 681          continue;
 682       }
 683
 684       // FindInCache is also expected to do an IMS check.
 685       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 686       if (File.end() == true)
 687          return false;
 688
 689       Visited[File->ID] = true;
 690    }
 691
 692    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 693       if (Visited[I] == false)
 694          return false;
 695
 696    if (_error->PendingError() == true)
 697    {
 698       _error->Discard();
 699       return false;
 700    }
 701
 702    if (OutMap != 0)
 703       *OutMap = Map.UnGuard();
 704    return true;
 705 }
 706                                                                         /*}}}*/
 707 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 708 // ---------------------------------------------------------------------
 709 /* Size is kind of an abstract notion that is only used for the progress
 710    meter */
 711 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 712 {
 713    unsigned long TotalSize = 0;
 714    for (; Start != End; Start++)
 715    {
 716       if ((*Start)->HasPackages() == false)
 717          continue;
 718       TotalSize += (*Start)->Size();
 719    }
 720    return TotalSize;
 721 }
 722                                                                         /*}}}*/
 723 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 724 // ---------------------------------------------------------------------
 725 /* */
 726 static bool BuildCache(pkgCacheGenerator &Gen,
 727                        OpProgress &Progress,
 728                        unsigned long &CurrentSize,unsigned long TotalSize,
 729                        FileIterator Start, FileIterator End)
 730 {
 731    FileIterator I;
 732    for (I = Start; I != End; I++)
 733    {
 734       if ((*I)->HasPackages() == false)
 735          continue;
 736
 737       if ((*I)->Exists() == false)
 738          continue;
 739
 740       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 741       {
 742          _error->Warning("Duplicate sources.list entry %s",
 743                          (*I)->Describe().c_str());
 744          continue;
 745       }
 746
 747       unsigned long Size = (*I)->Size();
 748       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 749       CurrentSize += Size;
 750
 751       if ((*I)->Merge(Gen,Progress) == false)
 752          return false;
 753    }
 754
 755    if (Gen.HasFileDeps() == true)
 756    {
 757       Progress.Done();
 758       TotalSize = ComputeSize(Start, End);
 759       CurrentSize = 0;
 760       for (I = Start; I != End; I++)
 761       {
 762          unsigned long Size = (*I)->Size();
 763          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 764          CurrentSize += Size;
 765          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 766             return false;
 767       }
 768    }
 769
 770    return true;
 771 }
 772                                                                         /*}}}*/
 773 // MakeStatusCache - Construct the status cache                         /*{{{*/
 774 // ---------------------------------------------------------------------
 775 /* This makes sure that the status cache (the cache that has all
 776    index files from the sources list and all local ones) is ready
 777    to be mmaped. If OutMap is not zero then a MMap object representing
 778    the cache will be stored there. This is pretty much mandetory if you
 779    are using AllowMem. AllowMem lets the function be run as non-root
 780    where it builds the cache 'fast' into a memory buffer. */
 781 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 782                         MMap **OutMap,bool AllowMem)
 783 {
 784    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 785
 786    vector<pkgIndexFile *> Files;
 787    for (vector<metaIndex *>::const_iterator i = List.begin();
 788         i != List.end();
 789         i++)
 790    {
 791       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 792       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 793            j != Indexes->end();
 794            j++)
 795          Files.push_back (*j);
 796    }
 797
 798    unsigned long EndOfSource = Files.size();
 799    if (_system->AddStatusFiles(Files) == false)
 800       return false;
 801
 802    // Decide if we can write to the files..
 803    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 804    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 805
 806    // Decide if we can write to the cache
 807    bool Writeable = false;
 808    if (CacheFile.empty() == false)
 809       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 810    else
 811       if (SrcCacheFile.empty() == false)
 812          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 813
 814    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 815       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 816
 817    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 818
 819    // Cache is OK, Fin.
 820    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 821    {
 822       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 823       return true;
 824    }
 825
 826    /* At this point we know we need to reconstruct the package cache,
 827       begin. */
 828    SPtr<FileFd> CacheF;
 829    SPtr<DynamicMMap> Map;
 830    if (Writeable == true && CacheFile.empty() == false)
 831    {
 832       unlink(CacheFile.c_str());
 833       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 834       fchmod(CacheF->Fd(),0644);
 835       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 836       if (_error->PendingError() == true)
 837          return false;
 838    }
 839    else
 840    {
 841       // Just build it in memory..
 842       Map = new DynamicMMap(MMap::Public,MapSize);
 843    }
 844
 845    // Lets try the source cache.
 846    unsigned long CurrentSize = 0;
 847    unsigned long TotalSize = 0;
 848    if (CheckValidity(SrcCacheFile,Files.begin(),
 849                      Files.begin()+EndOfSource) == true)
 850    {
 851       // Preload the map with the source cache
 852       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 853       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 854                        SCacheF.Size()) == false)
 855          return false;
 856
 857       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 858
 859       // Build the status cache
 860       pkgCacheGenerator Gen(Map.Get(),&Progress);
 861       if (_error->PendingError() == true)
 862          return false;
 863       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 864                      Files.begin()+EndOfSource,Files.end()) == false)
 865          return false;
 866    }
 867    else
 868    {
 869       TotalSize = ComputeSize(Files.begin(),Files.end());
 870
 871       // Build the source cache
 872       pkgCacheGenerator Gen(Map.Get(),&Progress);
 873       if (_error->PendingError() == true)
 874          return false;
 875       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 876                      Files.begin(),Files.begin()+EndOfSource) == false)
 877          return false;
 878
 879       // Write it back
 880       if (Writeable == true && SrcCacheFile.empty() == false)
 881       {
 882          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 883          if (_error->PendingError() == true)
 884             return false;
 885
 886          fchmod(SCacheF.Fd(),0644);
 887
 888          // Write out the main data
 889          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 890             return _error->Error(_("IO Error saving source cache"));
 891          SCacheF.Sync();
 892
 893          // Write out the proper header
 894          Gen.GetCache().HeaderP->Dirty = false;
 895          if (SCacheF.Seek(0) == false ||
 896              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 897             return _error->Error(_("IO Error saving source cache"));
 898          Gen.GetCache().HeaderP->Dirty = true;
 899          SCacheF.Sync();
 900       }
 901
 902       // Build the status cache
 903       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 904                      Files.begin()+EndOfSource,Files.end()) == false)
 905          return false;
 906    }
 907
 908    if (_error->PendingError() == true)
 909       return false;
 910    if (OutMap != 0)
 911    {
 912       if (CacheF != 0)
 913       {
 914          delete Map.UnGuard();
 915          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 916       }
 917       else
 918       {
 919          *OutMap = Map.UnGuard();
 920       }
 921    }
 922
 923    return true;
 924 }
 925                                                                         /*}}}*/
 926 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 927 // ---------------------------------------------------------------------
 928 /* */
 929 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 930 {
 931    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 932    vector<pkgIndexFile *> Files;
 933    unsigned long EndOfSource = Files.size();
 934    if (_system->AddStatusFiles(Files) == false)
 935       return false;
 936
 937    SPtr<DynamicMMap> Map;
 938    Map = new DynamicMMap(MMap::Public,MapSize);
 939    unsigned long CurrentSize = 0;
 940    unsigned long TotalSize = 0;
 941
 942    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 943
 944    // Build the status cache
 945    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 946    pkgCacheGenerator Gen(Map.Get(),&Progress);
 947    if (_error->PendingError() == true)
 948       return false;
 949    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 950                   Files.begin()+EndOfSource,Files.end()) == false)
 951       return false;
 952
 953    if (_error->PendingError() == true)
 954       return false;
 955    *OutMap = Map.UnGuard();
 956
 957    return true;
 958 }
 959                                                                         /*}}}*/