apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #define APT_COMPATIBILITY 986
  18
  19 #include <apt-pkg/pkgcachegen.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/version.h>
  22 #include <apt-pkg/progress.h>
  23 #include <apt-pkg/sourcelist.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/strutl.h>
  26 #include <apt-pkg/sptr.h>
  27 #include <apt-pkg/pkgsystem.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37 #include <system.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       Map.RawAllocate(sizeof(pkgCache::Header));
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          // Find the right version to write the description
 129          MD5SumValue CurMd5 = List.Description_md5();
 130          pkgCache::VerIterator Ver = Pkg.VersionList();
 131          map_ptrloc *LastVer = &Pkg->VersionList;
 132
 133          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 134          {
 135             pkgCache::DescIterator Desc = Ver.DescriptionList();
 136             map_ptrloc *LastDesc = &Ver->DescriptionList;
 137
 138             for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
 139                if (MD5SumValue(Desc.md5()) == CurMd5) {
 140                   // Add new description
 141                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 142                   Desc->ParentPkg = Pkg.Index();
 143
 144                   if (NewFileDesc(Desc,List) == false)
 145                      return _error->Error(_("Error occured while processing %s (NewFileDesc1)"),PackageName.c_str());
 146                   break;
 147                }
 148          }
 149
 150          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 151             return _error->Error(_("Error occured while processing %s (UsePackage1)"),
 152                                  PackageName.c_str());
 153          continue;
 154       }
 155
 156       pkgCache::VerIterator Ver = Pkg.VersionList();
 157       map_ptrloc *LastVer = &Pkg->VersionList;
 158       int Res = 1;
 159       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 160       {
 161          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 162          if (Res >= 0)
 163             break;
 164       }
 165
 166       /* We already have a version for this item, record that we
 167          saw it */
 168       unsigned long Hash = List.VersionHash();
 169       if (Res == 0 && Ver->Hash == Hash)
 170       {
 171          if (List.UsePackage(Pkg,Ver) == false)
 172             return _error->Error(_("Error occured while processing %s (UsePackage2)"),
 173                                  PackageName.c_str());
 174
 175          if (NewFileVer(Ver,List) == false)
 176             return _error->Error(_("Error occured while processing %s (NewFileVer1)"),
 177                                  PackageName.c_str());
 178
 179          // Read only a single record and return
 180          if (OutVer != 0)
 181          {
 182             *OutVer = Ver;
 183             FoundFileDeps |= List.HasFileDeps();
 184             return true;
 185          }
 186
 187          continue;
 188       }
 189
 190       // Skip to the end of the same version set.
 191       if (Res == 0)
 192       {
 193          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 194          {
 195             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 196             if (Res != 0)
 197                break;
 198          }
 199       }
 200
 201       // Add a new version
 202       *LastVer = NewVersion(Ver,Version,*LastVer);
 203       Ver->ParentPkg = Pkg.Index();
 204       Ver->Hash = Hash;
 205
 206       if (List.NewVersion(Ver) == false)
 207          return _error->Error(_("Error occured while processing %s (NewVersion1)"),
 208                               PackageName.c_str());
 209
 210       if (List.UsePackage(Pkg,Ver) == false)
 211          return _error->Error(_("Error occured while processing %s (UsePackage3)"),
 212                               PackageName.c_str());
 213
 214       if (NewFileVer(Ver,List) == false)
 215          return _error->Error(_("Error occured while processing %s (NewVersion2)"),
 216                               PackageName.c_str());
 217
 218       // Read only a single record and return
 219       if (OutVer != 0)
 220       {
 221          *OutVer = Ver;
 222          FoundFileDeps |= List.HasFileDeps();
 223          return true;
 224       }
 225
 226       /* Record the Description data. Description data always exist in
 227          Packages and Translation-* files. */
 228       pkgCache::DescIterator Desc = Ver.DescriptionList();
 229       map_ptrloc *LastDesc = &Ver->DescriptionList;
 230
 231       // Skip to the end of description set
 232       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 233
 234       // Add new description
 235       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 236       Desc->ParentPkg = Pkg.Index();
 237
 238       if (NewFileDesc(Desc,List) == false)
 239          return _error->Error(_("Error occured while processing %s (NewFileDesc2)"),PackageName.c_str());
 240    }
 241
 242    FoundFileDeps |= List.HasFileDeps();
 243
 244    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 245       return _error->Error(_("Wow, you exceeded the number of package "
 246                              "names this APT is capable of."));
 247    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 248       return _error->Error(_("Wow, you exceeded the number of versions "
 249                              "this APT is capable of."));
 250    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 251       return _error->Error(_("Wow, you exceeded the number of descriptions "
 252                              "this APT is capable of."));
 253    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 254       return _error->Error(_("Wow, you exceeded the number of dependencies "
 255                              "this APT is capable of."));
 256    return true;
 257 }
 258                                                                         /*}}}*/
 259 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* If we found any file depends while parsing the main list we need to
 262    resolve them. Since it is undesired to load the entire list of files
 263    into the cache as virtual packages we do a two stage effort. MergeList
 264    identifies the file depends and this creates Provdies for them by
 265    re-parsing all the indexs. */
 266 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 267 {
 268    List.Owner = this;
 269
 270    unsigned int Counter = 0;
 271    while (List.Step() == true)
 272    {
 273       string PackageName = List.Package();
 274       if (PackageName.empty() == true)
 275          return false;
 276       string Version = List.Version();
 277       if (Version.empty() == true)
 278          continue;
 279
 280       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 281       if (Pkg.end() == true)
 282          return _error->Error(_("Error occured while processing %s (FindPkg)"),
 283                                 PackageName.c_str());
 284       Counter++;
 285       if (Counter % 100 == 0 && Progress != 0)
 286          Progress->Progress(List.Offset());
 287
 288       unsigned long Hash = List.VersionHash();
 289       pkgCache::VerIterator Ver = Pkg.VersionList();
 290       for (; Ver.end() == false; Ver++)
 291       {
 292          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 293          {
 294             if (List.CollectFileProvides(Cache,Ver) == false)
 295                return _error->Error(_("Error occured while processing %s (CollectFileProvides)"),PackageName.c_str());
 296             break;
 297          }
 298       }
 299
 300       if (Ver.end() == true)
 301          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 302    }
 303
 304    return true;
 305 }
 306                                                                         /*}}}*/
 307 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 308 // ---------------------------------------------------------------------
 309 /* This creates a new package structure and adds it to the hash table */
 310 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 311 {
 312    Pkg = Cache.FindPkg(Name);
 313    if (Pkg.end() == false)
 314       return true;
 315
 316    // Get a structure
 317    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 318    if (Package == 0)
 319       return false;
 320
 321    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 322
 323    // Insert it into the hash table
 324    unsigned long Hash = Cache.Hash(Name);
 325    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 326    Cache.HeaderP->HashTable[Hash] = Package;
 327
 328    // Set the name and the ID
 329    Pkg->Name = Map.WriteString(Name);
 330    if (Pkg->Name == 0)
 331       return false;
 332    Pkg->ID = Cache.HeaderP->PackageCount++;
 333
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* */
 340 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 341                                    ListParser &List)
 342 {
 343    if (CurrentFile == 0)
 344       return true;
 345
 346    // Get a structure
 347    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 348    if (VerFile == 0)
 349       return 0;
 350
 351    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 352    VF->File = CurrentFile - Cache.PkgFileP;
 353
 354    // Link it to the end of the list
 355    map_ptrloc *Last = &Ver->FileList;
 356    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 357       Last = &V->NextFile;
 358    VF->NextFile = *Last;
 359    *Last = VF.Index();
 360
 361    VF->Offset = List.Offset();
 362    VF->Size = List.Size();
 363    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 364       Cache.HeaderP->MaxVerFileSize = VF->Size;
 365    Cache.HeaderP->VerFileCount++;
 366
 367    return true;
 368 }
 369                                                                         /*}}}*/
 370 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 371 // ---------------------------------------------------------------------
 372 /* This puts a version structure in the linked list */
 373 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 374                                             string VerStr,
 375                                             unsigned long Next)
 376 {
 377    // Get a structure
 378    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 379    if (Version == 0)
 380       return 0;
 381
 382    // Fill it in
 383    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 384    Ver->NextVer = Next;
 385    Ver->ID = Cache.HeaderP->VersionCount++;
 386    Ver->VerStr = Map.WriteString(VerStr);
 387    if (Ver->VerStr == 0)
 388       return 0;
 389
 390    return Version;
 391 }
 392                                                                         /*}}}*/
 393 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 394 // ---------------------------------------------------------------------
 395 /* */
 396 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 397                                    ListParser &List)
 398 {
 399    if (CurrentFile == 0)
 400       return true;
 401
 402    // Get a structure
 403    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 404    if (DescFile == 0)
 405       return 0;
 406
 407    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 408    DF->File = CurrentFile - Cache.PkgFileP;
 409
 410    // Link it to the end of the list
 411    map_ptrloc *Last = &Desc->FileList;
 412    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 413       Last = &D->NextFile;
 414    DF->NextFile = *Last;
 415    *Last = DF.Index();
 416
 417    DF->Offset = List.Offset();
 418    DF->Size = List.Size();
 419    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 420       Cache.HeaderP->MaxDescFileSize = DF->Size;
 421    Cache.HeaderP->DescFileCount++;
 422
 423    return true;
 424 }
 425                                                                         /*}}}*/
 426 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 427 // ---------------------------------------------------------------------
 428 /* This puts a description structure in the linked list */
 429 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 430                                             const string &Lang, const MD5SumValue &md5sum,
 431                                             map_ptrloc Next)
 432 {
 433    // Get a structure
 434    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 435    if (Description == 0)
 436       return 0;
 437
 438    // Fill it in
 439    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 440    Desc->NextDesc = Next;
 441    Desc->ID = Cache.HeaderP->DescriptionCount++;
 442    Desc->language_code = Map.WriteString(Lang);
 443    Desc->md5sum = Map.WriteString(md5sum.Value());
 444
 445    return Description;
 446 }
 447                                                                         /*}}}*/
 448 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 449 // ---------------------------------------------------------------------
 450 /* This creates a dependency element in the tree. It is linked to the
 451    version and to the package that it is pointing to. */
 452 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 453                                                string PackageName,
 454                                                string Version,
 455                                                unsigned int Op,
 456                                                unsigned int Type)
 457 {
 458    pkgCache &Cache = Owner->Cache;
 459
 460    // Get a structure
 461    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 462    if (Dependency == 0)
 463       return false;
 464
 465    // Fill it in
 466    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 467    Dep->ParentVer = Ver.Index();
 468    Dep->Type = Type;
 469    Dep->CompareOp = Op;
 470    Dep->ID = Cache.HeaderP->DependsCount++;
 471
 472    // Locate the target package
 473    pkgCache::PkgIterator Pkg;
 474    if (Owner->NewPackage(Pkg,PackageName) == false)
 475       return false;
 476
 477    // Probe the reverse dependency list for a version string that matches
 478    if (Version.empty() == false)
 479    {
 480 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 481          if (I->Version != 0 && I.TargetVer() == Version)
 482             Dep->Version = I->Version;*/
 483       if (Dep->Version == 0)
 484          if ((Dep->Version = WriteString(Version)) == 0)
 485             return false;
 486    }
 487
 488    // Link it to the package
 489    Dep->Package = Pkg.Index();
 490    Dep->NextRevDepends = Pkg->RevDepends;
 491    Pkg->RevDepends = Dep.Index();
 492
 493    /* Link it to the version (at the end of the list)
 494       Caching the old end point speeds up generation substantially */
 495    if (OldDepVer != Ver)
 496    {
 497       OldDepLast = &Ver->DependsList;
 498       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 499          OldDepLast = &D->NextDepends;
 500       OldDepVer = Ver;
 501    }
 502
 503    // Is it a file dependency?
 504    if (PackageName[0] == '/')
 505       FoundFileDeps = true;
 506
 507    Dep->NextDepends = *OldDepLast;
 508    *OldDepLast = Dep.Index();
 509    OldDepLast = &Dep->NextDepends;
 510
 511    return true;
 512 }
 513                                                                         /*}}}*/
 514 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 515 // ---------------------------------------------------------------------
 516 /* */
 517 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 518                                                 string PackageName,
 519                                                 string Version)
 520 {
 521    pkgCache &Cache = Owner->Cache;
 522
 523    // We do not add self referencing provides
 524    if (Ver.ParentPkg().Name() == PackageName)
 525       return true;
 526
 527    // Get a structure
 528    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 529    if (Provides == 0)
 530       return false;
 531    Cache.HeaderP->ProvidesCount++;
 532
 533    // Fill it in
 534    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 535    Prv->Version = Ver.Index();
 536    Prv->NextPkgProv = Ver->ProvidesList;
 537    Ver->ProvidesList = Prv.Index();
 538    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 539       return false;
 540
 541    // Locate the target package
 542    pkgCache::PkgIterator Pkg;
 543    if (Owner->NewPackage(Pkg,PackageName) == false)
 544       return false;
 545
 546    // Link it to the package
 547    Prv->ParentPkg = Pkg.Index();
 548    Prv->NextProvides = Pkg->ProvidesList;
 549    Pkg->ProvidesList = Prv.Index();
 550
 551    return true;
 552 }
 553                                                                         /*}}}*/
 554 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 555 // ---------------------------------------------------------------------
 556 /* This is used to select which file is to be associated with all newly
 557    added versions. The caller is responsible for setting the IMS fields. */
 558 bool pkgCacheGenerator::SelectFile(string File,string Site,
 559                                    const pkgIndexFile &Index,
 560                                    unsigned long Flags)
 561 {
 562    // Get some space for the structure
 563    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 564    if (CurrentFile == Cache.PkgFileP)
 565       return false;
 566
 567    // Fill it in
 568    CurrentFile->FileName = Map.WriteString(File);
 569    CurrentFile->Site = WriteUniqString(Site);
 570    CurrentFile->NextFile = Cache.HeaderP->FileList;
 571    CurrentFile->Flags = Flags;
 572    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 573    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 574    PkgFileName = File;
 575    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 576    Cache.HeaderP->PackageFileCount++;
 577
 578    if (CurrentFile->FileName == 0)
 579       return false;
 580
 581    if (Progress != 0)
 582       Progress->SubProgress(Index.Size());
 583    return true;
 584 }
 585                                                                         /*}}}*/
 586 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 587 // ---------------------------------------------------------------------
 588 /* This is used to create handles to strings. Given the same text it
 589    always returns the same number */
 590 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 591                                                  unsigned int Size)
 592 {
 593    /* We use a very small transient hash table here, this speeds up generation
 594       by a fair amount on slower machines */
 595    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 596    if (Bucket != 0 &&
 597        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 598       return Bucket->String;
 599
 600    // Search for an insertion point
 601    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 602    int Res = 1;
 603    map_ptrloc *Last = &Cache.HeaderP->StringList;
 604    for (; I != Cache.StringItemP; Last = &I->NextItem,
 605         I = Cache.StringItemP + I->NextItem)
 606    {
 607       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 608       if (Res >= 0)
 609          break;
 610    }
 611
 612    // Match
 613    if (Res == 0)
 614    {
 615       Bucket = I;
 616       return I->String;
 617    }
 618
 619    // Get a structure
 620    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 621    if (Item == 0)
 622       return 0;
 623
 624    // Fill in the structure
 625    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 626    ItemP->NextItem = I - Cache.StringItemP;
 627    *Last = Item;
 628    ItemP->String = Map.WriteString(S,Size);
 629    if (ItemP->String == 0)
 630       return 0;
 631
 632    Bucket = ItemP;
 633    return ItemP->String;
 634 }
 635                                                                         /*}}}*/
 636
 637 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 638 // ---------------------------------------------------------------------
 639 /* This just verifies that each file in the list of index files exists,
 640    has matching attributes with the cache and the cache does not have
 641    any extra files. */
 642 static bool CheckValidity(string CacheFile, FileIterator Start,
 643                           FileIterator End,MMap **OutMap = 0)
 644 {
 645    // No file, certainly invalid
 646    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 647       return false;
 648
 649    // Map it
 650    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 651    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 652    pkgCache Cache(Map);
 653    if (_error->PendingError() == true || Map->Size() == 0)
 654    {
 655       _error->Discard();
 656       return false;
 657    }
 658
 659    /* Now we check every index file, see if it is in the cache,
 660       verify the IMS data and check that it is on the disk too.. */
 661    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 662    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 663    for (; Start != End; Start++)
 664    {
 665       if ((*Start)->HasPackages() == false)
 666          continue;
 667
 668       if ((*Start)->Exists() == false)
 669       {
 670          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 671                           (*Start)->Describe().c_str());
 672          continue;
 673       }
 674
 675       // FindInCache is also expected to do an IMS check.
 676       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 677       if (File.end() == true)
 678          return false;
 679
 680       Visited[File->ID] = true;
 681    }
 682
 683    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 684       if (Visited[I] == false)
 685          return false;
 686
 687    if (_error->PendingError() == true)
 688    {
 689       _error->Discard();
 690       return false;
 691    }
 692
 693    if (OutMap != 0)
 694       *OutMap = Map.UnGuard();
 695    return true;
 696 }
 697                                                                         /*}}}*/
 698 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 699 // ---------------------------------------------------------------------
 700 /* Size is kind of an abstract notion that is only used for the progress
 701    meter */
 702 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 703 {
 704    unsigned long TotalSize = 0;
 705    for (; Start != End; Start++)
 706    {
 707       if ((*Start)->HasPackages() == false)
 708          continue;
 709       TotalSize += (*Start)->Size();
 710    }
 711    return TotalSize;
 712 }
 713                                                                         /*}}}*/
 714 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 715 // ---------------------------------------------------------------------
 716 /* */
 717 static bool BuildCache(pkgCacheGenerator &Gen,
 718                        OpProgress &Progress,
 719                        unsigned long &CurrentSize,unsigned long TotalSize,
 720                        FileIterator Start, FileIterator End)
 721 {
 722    FileIterator I;
 723    for (I = Start; I != End; I++)
 724    {
 725       if ((*I)->HasPackages() == false)
 726          continue;
 727
 728       if ((*I)->Exists() == false)
 729          continue;
 730
 731       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 732       {
 733          _error->Warning("Duplicate sources.list entry %s",
 734                          (*I)->Describe().c_str());
 735          continue;
 736       }
 737
 738       unsigned long Size = (*I)->Size();
 739       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading Package Lists"));
 740       CurrentSize += Size;
 741
 742       if ((*I)->Merge(Gen,Progress) == false)
 743          return false;
 744    }
 745
 746    if (Gen.HasFileDeps() == true)
 747    {
 748       Progress.Done();
 749       TotalSize = ComputeSize(Start, End);
 750       CurrentSize = 0;
 751       for (I = Start; I != End; I++)
 752       {
 753          unsigned long Size = (*I)->Size();
 754          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 755          CurrentSize += Size;
 756          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 757             return false;
 758       }
 759    }
 760
 761    return true;
 762 }
 763                                                                         /*}}}*/
 764 // MakeStatusCache - Construct the status cache                         /*{{{*/
 765 // ---------------------------------------------------------------------
 766 /* This makes sure that the status cache (the cache that has all
 767    index files from the sources list and all local ones) is ready
 768    to be mmaped. If OutMap is not zero then a MMap object representing
 769    the cache will be stored there. This is pretty much mandetory if you
 770    are using AllowMem. AllowMem lets the function be run as non-root
 771    where it builds the cache 'fast' into a memory buffer. */
 772 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 773                         MMap **OutMap,bool AllowMem)
 774 {
 775    unsigned long MapSize = _config->FindI("APT::Cache-Limit",12*1024*1024);
 776
 777    vector<pkgIndexFile *> Files;
 778    for (vector<metaIndex *>::const_iterator i = List.begin();
 779         i != List.end();
 780         i++)
 781    {
 782       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 783       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 784            j != Indexes->end();
 785            j++)
 786          Files.push_back (*j);
 787    }
 788
 789    unsigned long EndOfSource = Files.size();
 790    if (_system->AddStatusFiles(Files) == false)
 791       return false;
 792
 793    // Decide if we can write to the files..
 794    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 795    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 796
 797    // Decide if we can write to the cache
 798    bool Writeable = false;
 799    if (CacheFile.empty() == false)
 800       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 801    else
 802       if (SrcCacheFile.empty() == false)
 803          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 804
 805    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 806       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 807
 808    Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
 809
 810    // Cache is OK, Fin.
 811    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 812    {
 813       Progress.OverallProgress(1,1,1,_("Reading Package Lists"));
 814       return true;
 815    }
 816
 817    /* At this point we know we need to reconstruct the package cache,
 818       begin. */
 819    SPtr<FileFd> CacheF;
 820    SPtr<DynamicMMap> Map;
 821    if (Writeable == true && CacheFile.empty() == false)
 822    {
 823       unlink(CacheFile.c_str());
 824       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 825       fchmod(CacheF->Fd(),0644);
 826       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 827       if (_error->PendingError() == true)
 828          return false;
 829    }
 830    else
 831    {
 832       // Just build it in memory..
 833       Map = new DynamicMMap(MMap::Public,MapSize);
 834    }
 835
 836    // Lets try the source cache.
 837    unsigned long CurrentSize = 0;
 838    unsigned long TotalSize = 0;
 839    if (CheckValidity(SrcCacheFile,Files.begin(),
 840                      Files.begin()+EndOfSource) == true)
 841    {
 842       // Preload the map with the source cache
 843       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 844       if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
 845                        SCacheF.Size()) == false)
 846          return false;
 847
 848       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 849
 850       // Build the status cache
 851       pkgCacheGenerator Gen(Map.Get(),&Progress);
 852       if (_error->PendingError() == true)
 853          return false;
 854       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 855                      Files.begin()+EndOfSource,Files.end()) == false)
 856          return false;
 857    }
 858    else
 859    {
 860       TotalSize = ComputeSize(Files.begin(),Files.end());
 861
 862       // Build the source cache
 863       pkgCacheGenerator Gen(Map.Get(),&Progress);
 864       if (_error->PendingError() == true)
 865          return false;
 866       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 867                      Files.begin(),Files.begin()+EndOfSource) == false)
 868          return false;
 869
 870       // Write it back
 871       if (Writeable == true && SrcCacheFile.empty() == false)
 872       {
 873          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 874          if (_error->PendingError() == true)
 875             return false;
 876
 877          fchmod(SCacheF.Fd(),0644);
 878
 879          // Write out the main data
 880          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 881             return _error->Error(_("IO Error saving source cache"));
 882          SCacheF.Sync();
 883
 884          // Write out the proper header
 885          Gen.GetCache().HeaderP->Dirty = false;
 886          if (SCacheF.Seek(0) == false ||
 887              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 888             return _error->Error(_("IO Error saving source cache"));
 889          Gen.GetCache().HeaderP->Dirty = true;
 890          SCacheF.Sync();
 891       }
 892
 893       // Build the status cache
 894       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 895                      Files.begin()+EndOfSource,Files.end()) == false)
 896          return false;
 897    }
 898
 899    if (_error->PendingError() == true)
 900       return false;
 901    if (OutMap != 0)
 902    {
 903       if (CacheF != 0)
 904       {
 905          delete Map.UnGuard();
 906          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 907       }
 908       else
 909       {
 910          *OutMap = Map.UnGuard();
 911       }
 912    }
 913
 914    return true;
 915 }
 916                                                                         /*}}}*/
 917 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 918 // ---------------------------------------------------------------------
 919 /* */
 920 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 921 {
 922    unsigned long MapSize = _config->FindI("APT::Cache-Limit",8*1024*1024);
 923    vector<pkgIndexFile *> Files;
 924    unsigned long EndOfSource = Files.size();
 925    if (_system->AddStatusFiles(Files) == false)
 926       return false;
 927
 928    SPtr<DynamicMMap> Map;
 929    Map = new DynamicMMap(MMap::Public,MapSize);
 930    unsigned long CurrentSize = 0;
 931    unsigned long TotalSize = 0;
 932
 933    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 934
 935    // Build the status cache
 936    Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
 937    pkgCacheGenerator Gen(Map.Get(),&Progress);
 938    if (_error->PendingError() == true)
 939       return false;
 940    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 941                   Files.begin()+EndOfSource,Files.end()) == false)
 942       return false;
 943
 944    if (_error->PendingError() == true)
 945       return false;
 946    *OutMap = Map.UnGuard();
 947
 948    return true;
 949 }
 950                                                                         /*}}}*/