apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <vector>
  31
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36
  37 #include <apt-pkg/deblistparser.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108    debListParser *debian(dynamic_cast<debListParser *>(&List));
 109
 110    unsigned int Counter = 0;
 111    while (List.Step() == true)
 112    {
 113       // Get a pointer to the package structure
 114       srkString PackageName;
 115       if (debian != NULL)
 116          PackageName = debian->Find("Package");
 117       else
 118          PackageName = List.Package();
 119       if (PackageName.empty() == true)
 120          return false;
 121
 122       pkgCache::PkgIterator Pkg;
 123       if (NewPackage(Pkg,PackageName) == false)
 124          return _error->Error(_("Error occurred while processing %s (NewPackage)"),std::string(PackageName).c_str());
 125       Counter++;
 126       if (Counter % 100 == 0 && Progress != 0)
 127          Progress->Progress(List.Offset());
 128
 129       string language(List.DescriptionLanguage());
 130
 131       /* Get a pointer to the version structure. We know the list is sorted
 132          so we use that fact in the search. Insertion of new versions is
 133          done with correct sorting */
 134       srkString Version;
 135       if (debian != NULL)
 136          Version = debian->Find("Version");
 137       else
 138          Version = List.Version();
 139       if (Version.empty() == true)
 140       {
 141          // we first process the package, then the descriptions
 142          // (this has the bonus that we get MMap error when we run out
 143          //  of MMap space)
 144          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 145             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 146                                  std::string(PackageName).c_str());
 147
 148          // Find the right version to write the description
 149          MD5SumValue CurMd5 = List.Description_md5();
 150          pkgCache::VerIterator Ver = Pkg.VersionList();
 151          map_ptrloc *LastVer = &Pkg->VersionList;
 152
 153          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 154          {
 155             pkgCache::DescIterator Desc = Ver.DescriptionList();
 156             map_ptrloc *LastDesc = &Ver->DescriptionList;
 157             bool duplicate=false;
 158
 159             // don't add a new description if we have one for the given
 160             // md5 && language
 161             for ( ; Desc.end() == false; Desc++)
 162                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 163                    Desc.LanguageCode() == language)
 164                   duplicate=true;
 165             if(duplicate)
 166                continue;
 167
 168             for (Desc = Ver.DescriptionList();
 169                  Desc.end() == false;
 170                  LastDesc = &Desc->NextDesc, Desc++)
 171             {
 172                if (MD5SumValue(Desc.md5()) == CurMd5)
 173                {
 174                   // Add new description
 175                   *LastDesc = NewDescription(Desc, language, CurMd5, *LastDesc);
 176                   Desc->ParentPkg = Pkg.Index();
 177
 178                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 179                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),std::string(PackageName).c_str());
 180                   break;
 181                }
 182             }
 183          }
 184
 185          continue;
 186       }
 187
 188       pkgCache::VerIterator Ver = Pkg.VersionList();
 189       map_ptrloc *LastVer = &Pkg->VersionList;
 190       int Res = 1;
 191       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 192       {
 193          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 194          if (Res >= 0)
 195             break;
 196       }
 197
 198       /* We already have a version for this item, record that we
 199          saw it */
 200       unsigned long Hash = List.VersionHash();
 201       if (Res == 0 && Ver->Hash == Hash)
 202       {
 203          if (List.UsePackage(Pkg,Ver) == false)
 204             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 205                                  std::string(PackageName).c_str());
 206
 207          if (NewFileVer(Ver,List) == false)
 208             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 209                                  std::string(PackageName).c_str());
 210
 211          // Read only a single record and return
 212          if (OutVer != 0)
 213          {
 214             *OutVer = Ver;
 215             FoundFileDeps |= List.HasFileDeps();
 216             return true;
 217          }
 218
 219          continue;
 220       }
 221
 222       // Skip to the end of the same version set.
 223       if (Res == 0)
 224       {
 225          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 226          {
 227             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 228             if (Res != 0)
 229                break;
 230          }
 231       }
 232
 233       // Add a new version
 234       *LastVer = NewVersion(Ver,Version,*LastVer);
 235       Ver->ParentPkg = Pkg.Index();
 236       Ver->Hash = Hash;
 237
 238       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)
 239          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 240                               std::string(PackageName).c_str());
 241
 242       if (List.UsePackage(Pkg,Ver) == false)
 243          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 244                               std::string(PackageName).c_str());
 245
 246       if (NewFileVer(Ver,List) == false)
 247          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 248                               std::string(PackageName).c_str());
 249
 250       // Read only a single record and return
 251       if (OutVer != 0)
 252       {
 253          *OutVer = Ver;
 254          FoundFileDeps |= List.HasFileDeps();
 255          return true;
 256       }
 257
 258       /* Record the Description data. Description data always exist in
 259          Packages and Translation-* files. */
 260       pkgCache::DescIterator Desc = Ver.DescriptionList();
 261       map_ptrloc *LastDesc = &Ver->DescriptionList;
 262
 263       // Skip to the end of description set
 264       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 265
 266       // Add new description
 267       *LastDesc = NewDescription(Desc, language, List.Description_md5(), *LastDesc);
 268       Desc->ParentPkg = Pkg.Index();
 269
 270       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 271          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),std::string(PackageName).c_str());
 272    }
 273
 274    FoundFileDeps |= List.HasFileDeps();
 275
 276    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 277       return _error->Error(_("Wow, you exceeded the number of package "
 278                              "names this APT is capable of."));
 279    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 280       return _error->Error(_("Wow, you exceeded the number of versions "
 281                              "this APT is capable of."));
 282    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 283       return _error->Error(_("Wow, you exceeded the number of descriptions "
 284                              "this APT is capable of."));
 285    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 286       return _error->Error(_("Wow, you exceeded the number of dependencies "
 287                              "this APT is capable of."));
 288    return true;
 289 }
 290                                                                         /*}}}*/
 291 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* If we found any file depends while parsing the main list we need to
 294    resolve them. Since it is undesired to load the entire list of files
 295    into the cache as virtual packages we do a two stage effort. MergeList
 296    identifies the file depends and this creates Provdies for them by
 297    re-parsing all the indexs. */
 298 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 299 {
 300    List.Owner = this;
 301
 302    unsigned int Counter = 0;
 303    while (List.Step() == true)
 304    {
 305       string PackageName = List.Package();
 306       if (PackageName.empty() == true)
 307          return false;
 308       string Version = List.Version();
 309       if (Version.empty() == true)
 310          continue;
 311
 312       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 313       if (Pkg.end() == true)
 314          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 315                                 PackageName.c_str());
 316       Counter++;
 317       if (Counter % 100 == 0 && Progress != 0)
 318          Progress->Progress(List.Offset());
 319
 320       unsigned long Hash = List.VersionHash();
 321       pkgCache::VerIterator Ver = Pkg.VersionList();
 322       for (; Ver.end() == false; Ver++)
 323       {
 324          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 325          {
 326             if (List.CollectFileProvides(Cache,Ver) == false)
 327                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 328             break;
 329          }
 330       }
 331
 332       if (Ver.end() == true)
 333          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 334    }
 335
 336    return true;
 337 }
 338                                                                         /*}}}*/
 339 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* This creates a new package structure and adds it to the hash table */
 342 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 343 {
 344    return NewPackage(Pkg, srkString(Name));
 345 }
 346
 347 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const srkString &Name)
 348 {
 349    Pkg = Cache.FindPkg(Name);
 350    if (Pkg.end() == false)
 351       return true;
 352
 353    // Get a structure
 354    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 355    if (Package == 0)
 356       return false;
 357
 358    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 359
 360    // Insert it into the hash table
 361    unsigned long Hash = Cache.Hash(Name);
 362    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 363    Cache.HeaderP->HashTable[Hash] = Package;
 364
 365    // Set the name and the ID
 366    Pkg->Name = Map.WriteString(Name.Start,Name.Size);
 367    if (Pkg->Name == 0)
 368       return false;
 369    Pkg->ID = Cache.HeaderP->PackageCount++;
 370
 371    return true;
 372 }
 373                                                                         /*}}}*/
 374 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* */
 377 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 378                                    ListParser &List)
 379 {
 380    if (CurrentFile == 0)
 381       return true;
 382
 383    // Get a structure
 384    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 385    if (VerFile == 0)
 386       return 0;
 387
 388    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 389    VF->File = CurrentFile - Cache.PkgFileP;
 390
 391    // Link it to the end of the list
 392    map_ptrloc *Last = &Ver->FileList;
 393    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 394       Last = &V->NextFile;
 395    VF->NextFile = *Last;
 396    *Last = VF.Index();
 397
 398    VF->Offset = List.Offset();
 399    VF->Size = List.Size();
 400    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 401       Cache.HeaderP->MaxVerFileSize = VF->Size;
 402    Cache.HeaderP->VerFileCount++;
 403
 404    return true;
 405 }
 406                                                                         /*}}}*/
 407 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 408 // ---------------------------------------------------------------------
 409 /* This puts a version structure in the linked list */
 410 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 411                                             const string &VerStr,
 412                                             unsigned long Next)
 413 {
 414    return NewVersion(Ver, srkString(VerStr), Next);
 415 }
 416
 417 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 418                                             const srkString &VerStr,
 419                                             unsigned long Next)
 420 {
 421    // Get a structure
 422    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 423    if (Version == 0)
 424       return 0;
 425
 426    // Fill it in
 427    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 428    Ver->NextVer = Next;
 429    Ver->ID = Cache.HeaderP->VersionCount++;
 430    Ver->VerStr = Map.WriteString(VerStr.Start, VerStr.Size);
 431    if (Ver->VerStr == 0)
 432       return 0;
 433
 434    return Version;
 435 }
 436                                                                         /*}}}*/
 437 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 438 // ---------------------------------------------------------------------
 439 /* */
 440 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 441                                    ListParser &List)
 442 {
 443    if (CurrentFile == 0)
 444       return true;
 445
 446    // Get a structure
 447    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 448    if (DescFile == 0)
 449       return false;
 450
 451    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 452    DF->File = CurrentFile - Cache.PkgFileP;
 453
 454    // Link it to the end of the list
 455    map_ptrloc *Last = &Desc->FileList;
 456    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 457       Last = &D->NextFile;
 458
 459    DF->NextFile = *Last;
 460    *Last = DF.Index();
 461
 462    DF->Offset = List.Offset();
 463    DF->Size = List.Size();
 464    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 465       Cache.HeaderP->MaxDescFileSize = DF->Size;
 466    Cache.HeaderP->DescFileCount++;
 467
 468    return true;
 469 }
 470                                                                         /*}}}*/
 471 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 472 // ---------------------------------------------------------------------
 473 /* This puts a description structure in the linked list */
 474 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 475                                             const string &Lang,
 476                                             const MD5SumValue &md5sum,
 477                                             map_ptrloc Next)
 478 {
 479    // Get a structure
 480    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 481    if (Description == 0)
 482       return 0;
 483
 484    // Fill it in
 485    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 486    Desc->NextDesc = Next;
 487    Desc->ID = Cache.HeaderP->DescriptionCount++;
 488    Desc->language_code = Map.WriteString(Lang);
 489    Desc->md5sum = Map.WriteString(md5sum.Value());
 490    if (Desc->language_code == 0 || Desc->md5sum == 0)
 491       return 0;
 492
 493    return Description;
 494 }
 495                                                                         /*}}}*/
 496 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* This creates a dependency element in the tree. It is linked to the
 499    version and to the package that it is pointing to. */
 500 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 501                                                const string &PackageName,
 502                                                const string &Version,
 503                                                unsigned int Op,
 504                                                unsigned int Type)
 505 {
 506     return NewDepends(Ver, srkString(PackageName), srkString(Version), Op, Type);
 507 }
 508
 509 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 510                                                const srkString &PackageName,
 511                                                const srkString &Version,
 512                                                unsigned int Op,
 513                                                unsigned int Type)
 514 {
 515    pkgCache &Cache = Owner->Cache;
 516
 517    // Get a structure
 518    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 519    if (Dependency == 0)
 520       return false;
 521
 522    // Fill it in
 523    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 524    Dep->ParentVer = Ver.Index();
 525    Dep->Type = Type;
 526    Dep->CompareOp = Op;
 527    Dep->ID = Cache.HeaderP->DependsCount++;
 528
 529    // Locate the target package
 530    pkgCache::PkgIterator Pkg;
 531    if (Owner->NewPackage(Pkg,PackageName) == false)
 532       return false;
 533
 534    // Probe the reverse dependency list for a version string that matches
 535    if (Version.empty() == false)
 536    {
 537 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 538          if (I->Version != 0 && I.TargetVer() == Version)
 539             Dep->Version = I->Version;*/
 540       if (Dep->Version == 0)
 541          if ((Dep->Version = WriteString(Version)) == 0)
 542             return false;
 543    }
 544
 545    // Link it to the package
 546    Dep->Package = Pkg.Index();
 547    Dep->NextRevDepends = Pkg->RevDepends;
 548    Pkg->RevDepends = Dep.Index();
 549
 550    /* Link it to the version (at the end of the list)
 551       Caching the old end point speeds up generation substantially */
 552    if (OldDepVer != Ver)
 553    {
 554       OldDepLast = &Ver->DependsList;
 555       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 556          OldDepLast = &D->NextDepends;
 557       OldDepVer = Ver;
 558    }
 559
 560    // Is it a file dependency?
 561    if (PackageName[0] == '/')
 562       FoundFileDeps = true;
 563
 564    Dep->NextDepends = *OldDepLast;
 565    *OldDepLast = Dep.Index();
 566    OldDepLast = &Dep->NextDepends;
 567
 568    return true;
 569 }
 570                                                                         /*}}}*/
 571 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 572 // ---------------------------------------------------------------------
 573 /* */
 574 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 575                                                 const string &PackageName,
 576                                                 const string &Version)
 577 {
 578    return NewProvides(Ver, srkString(PackageName), srkString(Version));
 579 }
 580
 581 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 582                                                 const srkString &PackageName,
 583                                                 const srkString &Version)
 584 {
 585    pkgCache &Cache = Owner->Cache;
 586
 587    // We do not add self referencing provides
 588    if (Ver.ParentPkg().Name() == PackageName)
 589       return true;
 590
 591    // Get a structure
 592    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 593    if (Provides == 0)
 594       return false;
 595    Cache.HeaderP->ProvidesCount++;
 596
 597    // Fill it in
 598    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 599    Prv->Version = Ver.Index();
 600    Prv->NextPkgProv = Ver->ProvidesList;
 601    Ver->ProvidesList = Prv.Index();
 602    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 603       return false;
 604
 605    // Locate the target package
 606    pkgCache::PkgIterator Pkg;
 607    if (Owner->NewPackage(Pkg,PackageName) == false)
 608       return false;
 609
 610    // Link it to the package
 611    Prv->ParentPkg = Pkg.Index();
 612    Prv->NextProvides = Pkg->ProvidesList;
 613    Pkg->ProvidesList = Prv.Index();
 614
 615    return true;
 616 }
 617                                                                         /*}}}*/
 618 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 619 // ---------------------------------------------------------------------
 620 /* */
 621 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 622                                            const char *NameStart,
 623                                            unsigned int NameSize)
 624 {
 625    pkgCache &Cache = Owner->Cache;
 626
 627    // Get a structure
 628    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 629    if (Tagg == 0)
 630       return false;
 631    Cache.HeaderP->TagCount++;
 632
 633    // Fill it in
 634    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 635    Tg->Name = WriteString(NameStart,NameSize);
 636    if (Tg->Name == 0)
 637       return false;
 638    Tg->NextTag = Pkg->TagList;
 639    Pkg->TagList = Tg.Index();
 640
 641    return true;
 642 }
 643                                                                         /*}}}*/
 644 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 645 // ---------------------------------------------------------------------
 646 /* This is used to select which file is to be associated with all newly
 647    added versions. The caller is responsible for setting the IMS fields. */
 648 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 649                                    const pkgIndexFile &Index,
 650                                    unsigned long Flags)
 651 {
 652    // Get some space for the structure
 653    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 654    if (CurrentFile == Cache.PkgFileP)
 655       return false;
 656
 657    // Fill it in
 658    CurrentFile->FileName = Map.WriteString(File);
 659    CurrentFile->Site = WriteUniqString(Site);
 660    CurrentFile->NextFile = Cache.HeaderP->FileList;
 661    CurrentFile->Flags = Flags;
 662    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 663    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 664    PkgFileName = File;
 665    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 666    Cache.HeaderP->PackageFileCount++;
 667
 668    if (CurrentFile->FileName == 0)
 669       return false;
 670
 671    if (Progress != 0)
 672       Progress->SubProgress(Index.Size());
 673    return true;
 674 }
 675                                                                         /*}}}*/
 676 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 677 // ---------------------------------------------------------------------
 678 /* This is used to create handles to strings. Given the same text it
 679    always returns the same number */
 680 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 681                                                  unsigned int Size)
 682 {
 683    return WriteString(srkString(S, Size), srkSeriouslyUnique);
 684 }
 685
 686 unsigned long pkgCacheGenerator::WriteString(const srkString &S,
 687                                              enum srkLevel level)
 688 {
 689    if (level == srkRunOfTheMillNormal)
 690       return Map.WriteString(S.Start,S.Size);
 691
 692    /* We use a very small transient hash table here, this speeds up generation
 693       by a fair amount on slower machines */
 694    pkgCache::StringItem *&Bucket(level == srkReasonablySpecial ? SpecHash[(S[0]*5 + S[1]) % _count(SpecHash)] : UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)]);
 695    if (Bucket != 0 &&
 696        stringcmp(S,Cache.StrP + Bucket->String) == 0)
 697       return Bucket->String;
 698
 699    pkgCache::StringItem *I;
 700    map_ptrloc *Last;
 701
 702    if (level != srkSeriouslyUnique) {
 703       I = NULL;
 704       Last = NULL;
 705    } else {
 706
 707    // Search for an insertion point
 708    I = Cache.StringItemP + Cache.HeaderP->StringList;
 709    int Res = 1;
 710    Last = &Cache.HeaderP->StringList;
 711    for (; I != Cache.StringItemP; Last = &I->NextItem,
 712         I = Cache.StringItemP + I->NextItem)
 713    {
 714       Res = stringcmp(S,Cache.StrP + I->String);
 715       if (Res >= 0)
 716          break;
 717    }
 718
 719    // Match
 720    if (Res == 0)
 721    {
 722       Bucket = I;
 723       return I->String;
 724    }
 725
 726    }
 727
 728    // Get a structure
 729    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 730    if (Item == 0)
 731       return 0;
 732
 733    // Fill in the structure
 734    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 735    ItemP->NextItem = I - Cache.StringItemP;
 736    if (Last != NULL)
 737       *Last = Item;
 738    ItemP->String = Map.WriteString(S.Start,S.Size);
 739    if (ItemP->String == 0)
 740       return 0;
 741
 742    Bucket = ItemP;
 743    return ItemP->String;
 744 }
 745                                                                         /*}}}*/
 746 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 747 // ---------------------------------------------------------------------
 748 /* This just verifies that each file in the list of index files exists,
 749    has matching attributes with the cache and the cache does not have
 750    any extra files. */
 751 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 752                           FileIterator End,MMap **OutMap = 0)
 753 {
 754    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 755    // No file, certainly invalid
 756    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 757    {
 758       if (Debug == true)
 759          std::clog << "CacheFile doesn't exist" << std::endl;
 760       return false;
 761    }
 762
 763    // Map it
 764    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 765    SPtr<MMap> Map = new MMap(CacheF,0);
 766    pkgCache Cache(Map);
 767    if (_error->PendingError() == true || Map->Size() == 0)
 768    {
 769       if (Debug == true)
 770          std::clog << "Errors are pending or Map is empty()" << std::endl;
 771       _error->Discard();
 772       return false;
 773    }
 774
 775    /* Now we check every index file, see if it is in the cache,
 776       verify the IMS data and check that it is on the disk too.. */
 777    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 778    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 779    for (; Start != End; Start++)
 780    {
 781       if (Debug == true)
 782          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 783       if ((*Start)->HasPackages() == false)
 784       {
 785          if (Debug == true)
 786             std::clog << "Has NO packages" << std::endl;
 787          continue;
 788       }
 789
 790       if ((*Start)->Exists() == false)
 791       {
 792 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 793          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 794                           (*Start)->Describe().c_str());
 795 #endif
 796          if (Debug == true)
 797             std::clog << "file doesn't exist" << std::endl;
 798          continue;
 799       }
 800
 801       // FindInCache is also expected to do an IMS check.
 802       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 803       if (File.end() == true)
 804       {
 805          if (Debug == true)
 806             std::clog << "FindInCache returned end-Pointer" << std::endl;
 807          return false;
 808       }
 809
 810       Visited[File->ID] = true;
 811       if (Debug == true)
 812          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 813    }
 814
 815    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 816       if (Visited[I] == false)
 817       {
 818          if (Debug == true)
 819             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 820          return false;
 821       }
 822
 823    if (_error->PendingError() == true)
 824    {
 825       if (Debug == true)
 826       {
 827          std::clog << "Validity failed because of pending errors:" << std::endl;
 828          _error->DumpErrors();
 829       }
 830       _error->Discard();
 831       return false;
 832    }
 833
 834    if (OutMap != 0)
 835       *OutMap = Map.UnGuard();
 836    return true;
 837 }
 838                                                                         /*}}}*/
 839 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 840 // ---------------------------------------------------------------------
 841 /* Size is kind of an abstract notion that is only used for the progress
 842    meter */
 843 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 844 {
 845    unsigned long TotalSize = 0;
 846    for (; Start != End; Start++)
 847    {
 848       if ((*Start)->HasPackages() == false)
 849          continue;
 850       TotalSize += (*Start)->Size();
 851    }
 852    return TotalSize;
 853 }
 854                                                                         /*}}}*/
 855 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 856 // ---------------------------------------------------------------------
 857 /* */
 858 static bool BuildCache(pkgCacheGenerator &Gen,
 859                        OpProgress &Progress,
 860                        unsigned long &CurrentSize,unsigned long TotalSize,
 861                        FileIterator Start, FileIterator End)
 862 {
 863    FileIterator I;
 864    for (I = Start; I != End; I++)
 865    {
 866       if ((*I)->HasPackages() == false)
 867          continue;
 868
 869       if ((*I)->Exists() == false)
 870          continue;
 871
 872       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 873       {
 874          _error->Warning("Duplicate sources.list entry %s",
 875                          (*I)->Describe().c_str());
 876          continue;
 877       }
 878
 879       unsigned long Size = (*I)->Size();
 880       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 881       CurrentSize += Size;
 882
 883       if ((*I)->Merge(Gen,Progress) == false)
 884          return false;
 885    }
 886
 887    if (Gen.HasFileDeps() == true)
 888    {
 889       Progress.Done();
 890       TotalSize = ComputeSize(Start, End);
 891       CurrentSize = 0;
 892       for (I = Start; I != End; I++)
 893       {
 894          unsigned long Size = (*I)->Size();
 895          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 896          CurrentSize += Size;
 897          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 898             return false;
 899       }
 900    }
 901
 902    return true;
 903 }
 904                                                                         /*}}}*/
 905 // MakeStatusCache - Construct the status cache                         /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* This makes sure that the status cache (the cache that has all
 908    index files from the sources list and all local ones) is ready
 909    to be mmaped. If OutMap is not zero then a MMap object representing
 910    the cache will be stored there. This is pretty much mandetory if you
 911    are using AllowMem. AllowMem lets the function be run as non-root
 912    where it builds the cache 'fast' into a memory buffer. */
 913 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 914                         MMap **OutMap,bool AllowMem)
 915 {
 916    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 917    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 918
 919    vector<pkgIndexFile *> Files;
 920    for (vector<metaIndex *>::const_iterator i = List.begin();
 921         i != List.end();
 922         i++)
 923    {
 924       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 925       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 926            j != Indexes->end();
 927            j++)
 928          Files.push_back (*j);
 929    }
 930
 931    unsigned long const EndOfSource = Files.size();
 932    if (_system->AddStatusFiles(Files) == false)
 933       return false;
 934
 935    // Decide if we can write to the files..
 936    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 937    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 938
 939    // Decide if we can write to the cache
 940    bool Writeable = false;
 941    if (CacheFile.empty() == false)
 942       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 943    else
 944       if (SrcCacheFile.empty() == false)
 945          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 946    if (Debug == true)
 947       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
 948
 949    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 950       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 951
 952    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 953
 954    // Cache is OK, Fin.
 955    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 956    {
 957       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 958       if (Debug == true)
 959          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
 960       return true;
 961    }
 962    else if (Debug == true)
 963          std::clog << "pkgcache.bin is NOT valid" << std::endl;
 964
 965    /* At this point we know we need to reconstruct the package cache,
 966       begin. */
 967    SPtr<FileFd> CacheF;
 968    SPtr<DynamicMMap> Map;
 969    if (Writeable == true && CacheFile.empty() == false)
 970    {
 971       unlink(CacheFile.c_str());
 972       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 973       fchmod(CacheF->Fd(),0644);
 974       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 975       if (_error->PendingError() == true)
 976          return false;
 977       if (Debug == true)
 978          std::clog << "Open filebased MMap" << std::endl;
 979    }
 980    else
 981    {
 982       // Just build it in memory..
 983       Map = new DynamicMMap(0,MapSize);
 984       if (Debug == true)
 985          std::clog << "Open memory Map (not filebased)" << std::endl;
 986    }
 987
 988    // Lets try the source cache.
 989    unsigned long CurrentSize = 0;
 990    unsigned long TotalSize = 0;
 991    if (CheckValidity(SrcCacheFile,Files.begin(),
 992                      Files.begin()+EndOfSource) == true)
 993    {
 994       if (Debug == true)
 995          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
 996       // Preload the map with the source cache
 997       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 998       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
 999       if ((alloc == 0 && _error->PendingError())
1000                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1001                                 SCacheF.Size()) == false)
1002          return false;
1003
1004       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1005
1006       // Build the status cache
1007       pkgCacheGenerator Gen(Map.Get(),&Progress);
1008       if (_error->PendingError() == true)
1009          return false;
1010       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1011                      Files.begin()+EndOfSource,Files.end()) == false)
1012          return false;
1013    }
1014    else
1015    {
1016       if (Debug == true)
1017          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1018       TotalSize = ComputeSize(Files.begin(),Files.end());
1019
1020       // Build the source cache
1021       pkgCacheGenerator Gen(Map.Get(),&Progress);
1022       if (_error->PendingError() == true)
1023          return false;
1024       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1025                      Files.begin(),Files.begin()+EndOfSource) == false)
1026          return false;
1027
1028       // Write it back
1029       if (Writeable == true && SrcCacheFile.empty() == false)
1030       {
1031          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
1032          if (_error->PendingError() == true)
1033             return false;
1034
1035          fchmod(SCacheF.Fd(),0644);
1036
1037          // Write out the main data
1038          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1039             return _error->Error(_("IO Error saving source cache"));
1040          SCacheF.Sync();
1041
1042          // Write out the proper header
1043          Gen.GetCache().HeaderP->Dirty = false;
1044          if (SCacheF.Seek(0) == false ||
1045              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1046             return _error->Error(_("IO Error saving source cache"));
1047          Gen.GetCache().HeaderP->Dirty = true;
1048          SCacheF.Sync();
1049       }
1050
1051       // Build the status cache
1052       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1053                      Files.begin()+EndOfSource,Files.end()) == false)
1054          return false;
1055    }
1056    if (Debug == true)
1057       std::clog << "Caches are ready for shipping" << std::endl;
1058
1059    if (_error->PendingError() == true)
1060       return false;
1061    if (OutMap != 0)
1062    {
1063       if (CacheF != 0)
1064       {
1065          delete Map.UnGuard();
1066          *OutMap = new MMap(*CacheF,0);
1067       }
1068       else
1069       {
1070          *OutMap = Map.UnGuard();
1071       }
1072    }
1073
1074    return true;
1075 }
1076                                                                         /*}}}*/
1077 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
1078 // ---------------------------------------------------------------------
1079 /* */
1080 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1081 {
1082    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1083    vector<pkgIndexFile *> Files;
1084    unsigned long EndOfSource = Files.size();
1085    if (_system->AddStatusFiles(Files) == false)
1086       return false;
1087
1088    SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
1089    unsigned long CurrentSize = 0;
1090    unsigned long TotalSize = 0;
1091
1092    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1093
1094    // Build the status cache
1095    Progress.OverallProgress(0,1,1,_("Reading package lists"));
1096    pkgCacheGenerator Gen(Map.Get(),&Progress);
1097    if (_error->PendingError() == true)
1098       return false;
1099    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1100                   Files.begin()+EndOfSource,Files.end()) == false)
1101       return false;
1102
1103    if (_error->PendingError() == true)
1104       return false;
1105    *OutMap = Map.UnGuard();
1106
1107    return true;
1108 }
1109                                                                         /*}}}*/