apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <vector>
  31
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36
  37 #include <apt-pkg/deblistparser.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108    debListParser *debian(dynamic_cast<debListParser *>(&List));
 109
 110    unsigned int Counter = 0;
 111    while (List.Step() == true)
 112    {
 113       // Get a pointer to the package structure
 114       srkString PackageName;
 115       if (debian != NULL)
 116          PackageName = debian->Find("Package");
 117       else
 118          PackageName = List.Package();
 119       if (PackageName.empty() == true)
 120          return false;
 121
 122       pkgCache::PkgIterator Pkg;
 123       if (NewPackage(Pkg,PackageName) == false)
 124          return _error->Error(_("Error occurred while processing %s (NewPackage)"),std::string(PackageName).c_str());
 125       Counter++;
 126       if (Counter % 100 == 0 && Progress != 0)
 127          Progress->Progress(List.Offset());
 128
 129       string language(List.DescriptionLanguage());
 130
 131       /* Get a pointer to the version structure. We know the list is sorted
 132          so we use that fact in the search. Insertion of new versions is
 133          done with correct sorting */
 134       srkString Version;
 135       if (debian != NULL)
 136          Version = debian->Find("Version");
 137       else
 138          Version = List.Version();
 139       if (Version.empty() == true)
 140       {
 141          // we first process the package, then the descriptions
 142          // (this has the bonus that we get MMap error when we run out
 143          //  of MMap space)
 144          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 145             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 146                                  std::string(PackageName).c_str());
 147
 148          // Find the right version to write the description
 149          MD5SumValue CurMd5 = List.Description_md5();
 150          pkgCache::VerIterator Ver = Pkg.VersionList();
 151          map_ptrloc *LastVer = &Pkg->VersionList;
 152
 153          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 154          {
 155             pkgCache::DescIterator Desc = Ver.DescriptionList();
 156             map_ptrloc *LastDesc = &Ver->DescriptionList;
 157             bool duplicate=false;
 158
 159             // don't add a new description if we have one for the given
 160             // md5 && language
 161             for ( ; Desc.end() == false; Desc++)
 162                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 163                    Desc.LanguageCode() == language)
 164                   duplicate=true;
 165             if(duplicate)
 166                continue;
 167
 168             for (Desc = Ver.DescriptionList();
 169                  Desc.end() == false;
 170                  LastDesc = &Desc->NextDesc, Desc++)
 171             {
 172                if (MD5SumValue(Desc.md5()) == CurMd5)
 173                {
 174                   // Add new description
 175                   *LastDesc = NewDescription(Desc, language, CurMd5, *LastDesc);
 176                   Desc->ParentPkg = Pkg.Index();
 177
 178                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 179                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),std::string(PackageName).c_str());
 180                   break;
 181                }
 182             }
 183          }
 184
 185          continue;
 186       }
 187
 188       pkgCache::VerIterator Ver = Pkg.VersionList();
 189       map_ptrloc *LastVer = &Pkg->VersionList;
 190       int Res = 1;
 191       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 192       {
 193          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 194          if (Res >= 0)
 195             break;
 196       }
 197
 198       /* We already have a version for this item, record that we
 199          saw it */
 200       unsigned long Hash = List.VersionHash();
 201       if (Res == 0 && Ver->Hash == Hash)
 202       {
 203          if (List.UsePackage(Pkg,Ver) == false)
 204             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 205                                  std::string(PackageName).c_str());
 206
 207          if (NewFileVer(Ver,List) == false)
 208             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 209                                  std::string(PackageName).c_str());
 210
 211          // Read only a single record and return
 212          if (OutVer != 0)
 213          {
 214             *OutVer = Ver;
 215             FoundFileDeps |= List.HasFileDeps();
 216             return true;
 217          }
 218
 219          continue;
 220       }
 221
 222       // Skip to the end of the same version set.
 223       if (Res == 0)
 224       {
 225          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 226          {
 227             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 228             if (Res != 0)
 229                break;
 230          }
 231       }
 232
 233       // Add a new version
 234       *LastVer = NewVersion(Ver,Version,*LastVer);
 235       Ver->ParentPkg = Pkg.Index();
 236       Ver->Hash = Hash;
 237
 238       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)
 239          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 240                               std::string(PackageName).c_str());
 241
 242       if (List.UsePackage(Pkg,Ver) == false)
 243          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 244                               std::string(PackageName).c_str());
 245
 246       if (NewFileVer(Ver,List) == false)
 247          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 248                               std::string(PackageName).c_str());
 249
 250       // Read only a single record and return
 251       if (OutVer != 0)
 252       {
 253          *OutVer = Ver;
 254          FoundFileDeps |= List.HasFileDeps();
 255          return true;
 256       }
 257
 258       /* Record the Description data. Description data always exist in
 259          Packages and Translation-* files. */
 260       pkgCache::DescIterator Desc = Ver.DescriptionList();
 261       map_ptrloc *LastDesc = &Ver->DescriptionList;
 262
 263       // Skip to the end of description set
 264       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 265
 266       // Add new description
 267       *LastDesc = NewDescription(Desc, language, List.Description_md5(), *LastDesc);
 268       Desc->ParentPkg = Pkg.Index();
 269
 270       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 271          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),std::string(PackageName).c_str());
 272    }
 273
 274    FoundFileDeps |= List.HasFileDeps();
 275
 276    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 277       return _error->Error(_("Wow, you exceeded the number of package "
 278                              "names this APT is capable of."));
 279    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 280       return _error->Error(_("Wow, you exceeded the number of versions "
 281                              "this APT is capable of."));
 282    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 283       return _error->Error(_("Wow, you exceeded the number of descriptions "
 284                              "this APT is capable of."));
 285    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 286       return _error->Error(_("Wow, you exceeded the number of dependencies "
 287                              "this APT is capable of."));
 288    return true;
 289 }
 290                                                                         /*}}}*/
 291 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* If we found any file depends while parsing the main list we need to
 294    resolve them. Since it is undesired to load the entire list of files
 295    into the cache as virtual packages we do a two stage effort. MergeList
 296    identifies the file depends and this creates Provdies for them by
 297    re-parsing all the indexs. */
 298 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 299 {
 300    List.Owner = this;
 301
 302    unsigned int Counter = 0;
 303    while (List.Step() == true)
 304    {
 305       string PackageName = List.Package();
 306       if (PackageName.empty() == true)
 307          return false;
 308       string Version = List.Version();
 309       if (Version.empty() == true)
 310          continue;
 311
 312       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 313       if (Pkg.end() == true)
 314          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 315                                 PackageName.c_str());
 316       Counter++;
 317       if (Counter % 100 == 0 && Progress != 0)
 318          Progress->Progress(List.Offset());
 319
 320       unsigned long Hash = List.VersionHash();
 321       pkgCache::VerIterator Ver = Pkg.VersionList();
 322       for (; Ver.end() == false; Ver++)
 323       {
 324          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 325          {
 326             if (List.CollectFileProvides(Cache,Ver) == false)
 327                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 328             break;
 329          }
 330       }
 331
 332       if (Ver.end() == true)
 333          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 334    }
 335
 336    return true;
 337 }
 338                                                                         /*}}}*/
 339 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* This creates a new package structure and adds it to the hash table */
 342 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 343 {
 344    return NewPackage(Pkg, srkString(Name));
 345 }
 346
 347 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const srkString &Name)
 348 {
 349    Pkg = Cache.FindPkg(Name);
 350    if (Pkg.end() == false)
 351       return true;
 352
 353    // Get a structure
 354    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 355    if (Package == 0)
 356       return false;
 357
 358    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 359
 360    // Insert it into the hash table
 361    unsigned long Hash = Cache.Hash(Name);
 362    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 363    Cache.HeaderP->HashTable[Hash] = Package;
 364
 365    // Set the name and the ID
 366    Pkg->Name = Map.WriteString(Name.Start,Name.Size);
 367    if (Pkg->Name == 0)
 368       return false;
 369    Pkg->ID = Cache.HeaderP->PackageCount++;
 370
 371    return true;
 372 }
 373                                                                         /*}}}*/
 374 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* */
 377 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 378                                    ListParser &List)
 379 {
 380    if (CurrentFile == 0)
 381       return true;
 382
 383    // Get a structure
 384    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 385    if (VerFile == 0)
 386       return 0;
 387
 388    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 389    VF->File = CurrentFile - Cache.PkgFileP;
 390
 391    // Link it to the end of the list
 392    map_ptrloc *Last = &Ver->FileList;
 393    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 394       Last = &V->NextFile;
 395    VF->NextFile = *Last;
 396    *Last = VF.Index();
 397
 398    VF->Offset = List.Offset();
 399    VF->Size = List.Size();
 400    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 401       Cache.HeaderP->MaxVerFileSize = VF->Size;
 402    Cache.HeaderP->VerFileCount++;
 403
 404    return true;
 405 }
 406                                                                         /*}}}*/
 407 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 408 // ---------------------------------------------------------------------
 409 /* This puts a version structure in the linked list */
 410 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 411                                             const string &VerStr,
 412                                             unsigned long Next)
 413 {
 414    return NewVersion(Ver, srkString(VerStr), Next);
 415 }
 416
 417 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 418                                             const srkString &VerStr,
 419                                             unsigned long Next)
 420 {
 421    // Get a structure
 422    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 423    if (Version == 0)
 424       return 0;
 425
 426    // Fill it in
 427    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 428    Ver->NextVer = Next;
 429    Ver->ID = Cache.HeaderP->VersionCount++;
 430    Ver->VerStr = Map.WriteString(VerStr.Start, VerStr.Size);
 431    if (Ver->VerStr == 0)
 432       return 0;
 433
 434    return Version;
 435 }
 436                                                                         /*}}}*/
 437 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 438 // ---------------------------------------------------------------------
 439 /* */
 440 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 441                                    ListParser &List)
 442 {
 443    if (CurrentFile == 0)
 444       return true;
 445
 446    // Get a structure
 447    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 448    if (DescFile == 0)
 449       return false;
 450
 451    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 452    DF->File = CurrentFile - Cache.PkgFileP;
 453
 454    // Link it to the end of the list
 455    map_ptrloc *Last = &Desc->FileList;
 456    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 457       Last = &D->NextFile;
 458
 459    DF->NextFile = *Last;
 460    *Last = DF.Index();
 461
 462    DF->Offset = List.Offset();
 463    DF->Size = List.Size();
 464    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 465       Cache.HeaderP->MaxDescFileSize = DF->Size;
 466    Cache.HeaderP->DescFileCount++;
 467
 468    return true;
 469 }
 470                                                                         /*}}}*/
 471 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 472 // ---------------------------------------------------------------------
 473 /* This puts a description structure in the linked list */
 474 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 475                                             const string &Lang,
 476                                             const MD5SumValue &md5sum,
 477                                             map_ptrloc Next)
 478 {
 479    // Get a structure
 480    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 481    if (Description == 0)
 482       return 0;
 483
 484    // Fill it in
 485    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 486    Desc->NextDesc = Next;
 487    Desc->ID = Cache.HeaderP->DescriptionCount++;
 488    Desc->language_code = Map.WriteString(Lang);
 489    Desc->md5sum = Map.WriteString(md5sum.Value());
 490    if (Desc->language_code == 0 || Desc->md5sum == 0)
 491       return 0;
 492
 493    return Description;
 494 }
 495                                                                         /*}}}*/
 496 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* This creates a dependency element in the tree. It is linked to the
 499    version and to the package that it is pointing to. */
 500 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 501                                                const string &PackageName,
 502                                                const string &Version,
 503                                                unsigned int Op,
 504                                                unsigned int Type)
 505 {
 506     return NewDepends(Ver, srkString(PackageName), srkString(Version), Op, Type);
 507 }
 508
 509 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 510                                                const srkString &PackageName,
 511                                                const srkString &Version,
 512                                                unsigned int Op,
 513                                                unsigned int Type)
 514 {
 515    pkgCache &Cache = Owner->Cache;
 516
 517    // Get a structure
 518    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 519    if (Dependency == 0)
 520       return false;
 521
 522    // Fill it in
 523    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 524    Dep->ParentVer = Ver.Index();
 525    Dep->Type = Type;
 526    Dep->CompareOp = Op;
 527    Dep->ID = Cache.HeaderP->DependsCount++;
 528
 529    // Locate the target package
 530    pkgCache::PkgIterator Pkg;
 531    if (Owner->NewPackage(Pkg,PackageName) == false)
 532       return false;
 533
 534    // Probe the reverse dependency list for a version string that matches
 535    if (Version.empty() == false)
 536    {
 537 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 538          if (I->Version != 0 && I.TargetVer() == Version)
 539             Dep->Version = I->Version;*/
 540       if (Dep->Version == 0)
 541          if ((Dep->Version = WriteString(Version)) == 0)
 542             return false;
 543    }
 544
 545    // Link it to the package
 546    Dep->Package = Pkg.Index();
 547    Dep->NextRevDepends = Pkg->RevDepends;
 548    Pkg->RevDepends = Dep.Index();
 549
 550    /* Link it to the version (at the end of the list)
 551       Caching the old end point speeds up generation substantially */
 552    if (OldDepVer != Ver)
 553    {
 554       OldDepLast = &Ver->DependsList;
 555       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 556          OldDepLast = &D->NextDepends;
 557       OldDepVer = Ver;
 558    }
 559
 560    // Is it a file dependency?
 561    if (PackageName[0] == '/')
 562       FoundFileDeps = true;
 563
 564    Dep->NextDepends = *OldDepLast;
 565    *OldDepLast = Dep.Index();
 566    OldDepLast = &Dep->NextDepends;
 567
 568    return true;
 569 }
 570                                                                         /*}}}*/
 571 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 572 // ---------------------------------------------------------------------
 573 /* */
 574 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 575                                                 const string &PackageName,
 576                                                 const string &Version)
 577 {
 578    return NewProvides(Ver, srkString(PackageName), srkString(Version));
 579 }
 580
 581 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 582                                                 const srkString &PackageName,
 583                                                 const srkString &Version)
 584 {
 585    pkgCache &Cache = Owner->Cache;
 586
 587    // We do not add self referencing provides
 588    if (Ver.ParentPkg().Name() == PackageName)
 589       return true;
 590
 591    // Get a structure
 592    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 593    if (Provides == 0)
 594       return false;
 595    Cache.HeaderP->ProvidesCount++;
 596
 597    // Fill it in
 598    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 599    Prv->Version = Ver.Index();
 600    Prv->NextPkgProv = Ver->ProvidesList;
 601    Ver->ProvidesList = Prv.Index();
 602    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 603       return false;
 604
 605    // Locate the target package
 606    pkgCache::PkgIterator Pkg;
 607    if (Owner->NewPackage(Pkg,PackageName) == false)
 608       return false;
 609
 610    // Link it to the package
 611    Prv->ParentPkg = Pkg.Index();
 612    Prv->NextProvides = Pkg->ProvidesList;
 613    Pkg->ProvidesList = Prv.Index();
 614
 615    return true;
 616 }
 617                                                                         /*}}}*/
 618 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 619 // ---------------------------------------------------------------------
 620 /* */
 621 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 622                                            const char *NameStart,
 623                                            unsigned int NameSize)
 624 {
 625    pkgCache &Cache = Owner->Cache;
 626
 627    // Get a structure
 628    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 629    if (Tagg == 0)
 630       return false;
 631    Cache.HeaderP->TagCount++;
 632
 633    // Fill it in
 634    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 635    Tg->Name = WriteString(NameStart,NameSize);
 636    if (Tg->Name == 0)
 637       return false;
 638    Tg->NextTag = Pkg->TagList;
 639    Pkg->TagList = Tg.Index();
 640
 641    return true;
 642 }
 643                                                                         /*}}}*/
 644 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 645 // ---------------------------------------------------------------------
 646 /* This is used to select which file is to be associated with all newly
 647    added versions. The caller is responsible for setting the IMS fields. */
 648 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 649                                    const pkgIndexFile &Index,
 650                                    unsigned long Flags)
 651 {
 652    // Get some space for the structure
 653    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 654    if (CurrentFile == Cache.PkgFileP)
 655       return false;
 656
 657    // Fill it in
 658    CurrentFile->FileName = Map.WriteString(File);
 659    CurrentFile->Site = WriteUniqString(Site);
 660    CurrentFile->NextFile = Cache.HeaderP->FileList;
 661    CurrentFile->Flags = Flags;
 662    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 663    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 664    PkgFileName = File;
 665    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 666    Cache.HeaderP->PackageFileCount++;
 667
 668    if (CurrentFile->FileName == 0)
 669       return false;
 670
 671    if (Progress != 0)
 672       Progress->SubProgress(Index.Size());
 673    return true;
 674 }
 675                                                                         /*}}}*/
 676 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 677 // ---------------------------------------------------------------------
 678 /* This is used to create handles to strings. Given the same text it
 679    always returns the same number */
 680 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 681                                                  unsigned int Size)
 682 {
 683    /* We use a very small transient hash table here, this speeds up generation
 684       by a fair amount on slower machines */
 685    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 686    if (Bucket != 0 &&
 687        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 688       return Bucket->String;
 689
 690    // Search for an insertion point
 691    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 692    int Res = 1;
 693    map_ptrloc *Last = &Cache.HeaderP->StringList;
 694    for (; I != Cache.StringItemP; Last = &I->NextItem,
 695         I = Cache.StringItemP + I->NextItem)
 696    {
 697       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 698       if (Res >= 0)
 699          break;
 700    }
 701
 702    // Match
 703    if (Res == 0)
 704    {
 705       Bucket = I;
 706       return I->String;
 707    }
 708
 709    // Get a structure
 710    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 711    if (Item == 0)
 712       return 0;
 713
 714    // Fill in the structure
 715    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 716    ItemP->NextItem = I - Cache.StringItemP;
 717    *Last = Item;
 718    ItemP->String = Map.WriteString(S,Size);
 719    if (ItemP->String == 0)
 720       return 0;
 721
 722    Bucket = ItemP;
 723    return ItemP->String;
 724 }
 725                                                                         /*}}}*/
 726 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 727 // ---------------------------------------------------------------------
 728 /* This just verifies that each file in the list of index files exists,
 729    has matching attributes with the cache and the cache does not have
 730    any extra files. */
 731 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 732                           FileIterator End,MMap **OutMap = 0)
 733 {
 734    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 735    // No file, certainly invalid
 736    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 737    {
 738       if (Debug == true)
 739          std::clog << "CacheFile doesn't exist" << std::endl;
 740       return false;
 741    }
 742
 743    // Map it
 744    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 745    SPtr<MMap> Map = new MMap(CacheF,0);
 746    pkgCache Cache(Map);
 747    if (_error->PendingError() == true || Map->Size() == 0)
 748    {
 749       if (Debug == true)
 750          std::clog << "Errors are pending or Map is empty()" << std::endl;
 751       _error->Discard();
 752       return false;
 753    }
 754
 755    /* Now we check every index file, see if it is in the cache,
 756       verify the IMS data and check that it is on the disk too.. */
 757    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 758    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 759    for (; Start != End; Start++)
 760    {
 761       if (Debug == true)
 762          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 763       if ((*Start)->HasPackages() == false)
 764       {
 765          if (Debug == true)
 766             std::clog << "Has NO packages" << std::endl;
 767          continue;
 768       }
 769
 770       if ((*Start)->Exists() == false)
 771       {
 772 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 773          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 774                           (*Start)->Describe().c_str());
 775 #endif
 776          if (Debug == true)
 777             std::clog << "file doesn't exist" << std::endl;
 778          continue;
 779       }
 780
 781       // FindInCache is also expected to do an IMS check.
 782       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 783       if (File.end() == true)
 784       {
 785          if (Debug == true)
 786             std::clog << "FindInCache returned end-Pointer" << std::endl;
 787          return false;
 788       }
 789
 790       Visited[File->ID] = true;
 791       if (Debug == true)
 792          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 793    }
 794
 795    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 796       if (Visited[I] == false)
 797       {
 798          if (Debug == true)
 799             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 800          return false;
 801       }
 802
 803    if (_error->PendingError() == true)
 804    {
 805       if (Debug == true)
 806       {
 807          std::clog << "Validity failed because of pending errors:" << std::endl;
 808          _error->DumpErrors();
 809       }
 810       _error->Discard();
 811       return false;
 812    }
 813
 814    if (OutMap != 0)
 815       *OutMap = Map.UnGuard();
 816    return true;
 817 }
 818                                                                         /*}}}*/
 819 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 820 // ---------------------------------------------------------------------
 821 /* Size is kind of an abstract notion that is only used for the progress
 822    meter */
 823 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 824 {
 825    unsigned long TotalSize = 0;
 826    for (; Start != End; Start++)
 827    {
 828       if ((*Start)->HasPackages() == false)
 829          continue;
 830       TotalSize += (*Start)->Size();
 831    }
 832    return TotalSize;
 833 }
 834                                                                         /*}}}*/
 835 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 836 // ---------------------------------------------------------------------
 837 /* */
 838 static bool BuildCache(pkgCacheGenerator &Gen,
 839                        OpProgress &Progress,
 840                        unsigned long &CurrentSize,unsigned long TotalSize,
 841                        FileIterator Start, FileIterator End)
 842 {
 843    FileIterator I;
 844    for (I = Start; I != End; I++)
 845    {
 846       if ((*I)->HasPackages() == false)
 847          continue;
 848
 849       if ((*I)->Exists() == false)
 850          continue;
 851
 852       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 853       {
 854          _error->Warning("Duplicate sources.list entry %s",
 855                          (*I)->Describe().c_str());
 856          continue;
 857       }
 858
 859       unsigned long Size = (*I)->Size();
 860       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 861       CurrentSize += Size;
 862
 863       if ((*I)->Merge(Gen,Progress) == false)
 864          return false;
 865    }
 866
 867    if (Gen.HasFileDeps() == true)
 868    {
 869       Progress.Done();
 870       TotalSize = ComputeSize(Start, End);
 871       CurrentSize = 0;
 872       for (I = Start; I != End; I++)
 873       {
 874          unsigned long Size = (*I)->Size();
 875          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 876          CurrentSize += Size;
 877          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 878             return false;
 879       }
 880    }
 881
 882    return true;
 883 }
 884                                                                         /*}}}*/
 885 // MakeStatusCache - Construct the status cache                         /*{{{*/
 886 // ---------------------------------------------------------------------
 887 /* This makes sure that the status cache (the cache that has all
 888    index files from the sources list and all local ones) is ready
 889    to be mmaped. If OutMap is not zero then a MMap object representing
 890    the cache will be stored there. This is pretty much mandetory if you
 891    are using AllowMem. AllowMem lets the function be run as non-root
 892    where it builds the cache 'fast' into a memory buffer. */
 893 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 894                         MMap **OutMap,bool AllowMem)
 895 {
 896    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 897    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 898
 899    vector<pkgIndexFile *> Files;
 900    for (vector<metaIndex *>::const_iterator i = List.begin();
 901         i != List.end();
 902         i++)
 903    {
 904       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 905       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 906            j != Indexes->end();
 907            j++)
 908          Files.push_back (*j);
 909    }
 910
 911    unsigned long const EndOfSource = Files.size();
 912    if (_system->AddStatusFiles(Files) == false)
 913       return false;
 914
 915    // Decide if we can write to the files..
 916    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 917    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 918
 919    // Decide if we can write to the cache
 920    bool Writeable = false;
 921    if (CacheFile.empty() == false)
 922       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 923    else
 924       if (SrcCacheFile.empty() == false)
 925          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 926    if (Debug == true)
 927       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
 928
 929    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 930       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 931
 932    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 933
 934    // Cache is OK, Fin.
 935    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 936    {
 937       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 938       if (Debug == true)
 939          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
 940       return true;
 941    }
 942    else if (Debug == true)
 943          std::clog << "pkgcache.bin is NOT valid" << std::endl;
 944
 945    /* At this point we know we need to reconstruct the package cache,
 946       begin. */
 947    SPtr<FileFd> CacheF;
 948    SPtr<DynamicMMap> Map;
 949    if (Writeable == true && CacheFile.empty() == false)
 950    {
 951       unlink(CacheFile.c_str());
 952       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 953       fchmod(CacheF->Fd(),0644);
 954       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 955       if (_error->PendingError() == true)
 956          return false;
 957       if (Debug == true)
 958          std::clog << "Open filebased MMap" << std::endl;
 959    }
 960    else
 961    {
 962       // Just build it in memory..
 963       Map = new DynamicMMap(0,MapSize);
 964       if (Debug == true)
 965          std::clog << "Open memory Map (not filebased)" << std::endl;
 966    }
 967
 968    // Lets try the source cache.
 969    unsigned long CurrentSize = 0;
 970    unsigned long TotalSize = 0;
 971    if (CheckValidity(SrcCacheFile,Files.begin(),
 972                      Files.begin()+EndOfSource) == true)
 973    {
 974       if (Debug == true)
 975          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
 976       // Preload the map with the source cache
 977       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 978       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
 979       if ((alloc == 0 && _error->PendingError())
 980                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
 981                                 SCacheF.Size()) == false)
 982          return false;
 983
 984       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 985
 986       // Build the status cache
 987       pkgCacheGenerator Gen(Map.Get(),&Progress);
 988       if (_error->PendingError() == true)
 989          return false;
 990       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 991                      Files.begin()+EndOfSource,Files.end()) == false)
 992          return false;
 993    }
 994    else
 995    {
 996       if (Debug == true)
 997          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
 998       TotalSize = ComputeSize(Files.begin(),Files.end());
 999
1000       // Build the source cache
1001       pkgCacheGenerator Gen(Map.Get(),&Progress);
1002       if (_error->PendingError() == true)
1003          return false;
1004       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1005                      Files.begin(),Files.begin()+EndOfSource) == false)
1006          return false;
1007
1008       // Write it back
1009       if (Writeable == true && SrcCacheFile.empty() == false)
1010       {
1011          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
1012          if (_error->PendingError() == true)
1013             return false;
1014
1015          fchmod(SCacheF.Fd(),0644);
1016
1017          // Write out the main data
1018          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1019             return _error->Error(_("IO Error saving source cache"));
1020          SCacheF.Sync();
1021
1022          // Write out the proper header
1023          Gen.GetCache().HeaderP->Dirty = false;
1024          if (SCacheF.Seek(0) == false ||
1025              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1026             return _error->Error(_("IO Error saving source cache"));
1027          Gen.GetCache().HeaderP->Dirty = true;
1028          SCacheF.Sync();
1029       }
1030
1031       // Build the status cache
1032       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1033                      Files.begin()+EndOfSource,Files.end()) == false)
1034          return false;
1035    }
1036    if (Debug == true)
1037       std::clog << "Caches are ready for shipping" << std::endl;
1038
1039    if (_error->PendingError() == true)
1040       return false;
1041    if (OutMap != 0)
1042    {
1043       if (CacheF != 0)
1044       {
1045          delete Map.UnGuard();
1046          *OutMap = new MMap(*CacheF,0);
1047       }
1048       else
1049       {
1050          *OutMap = Map.UnGuard();
1051       }
1052    }
1053
1054    return true;
1055 }
1056                                                                         /*}}}*/
1057 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
1058 // ---------------------------------------------------------------------
1059 /* */
1060 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1061 {
1062    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1063    vector<pkgIndexFile *> Files;
1064    unsigned long EndOfSource = Files.size();
1065    if (_system->AddStatusFiles(Files) == false)
1066       return false;
1067
1068    SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
1069    unsigned long CurrentSize = 0;
1070    unsigned long TotalSize = 0;
1071
1072    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1073
1074    // Build the status cache
1075    Progress.OverallProgress(0,1,1,_("Reading package lists"));
1076    pkgCacheGenerator Gen(Map.Get(),&Progress);
1077    if (_error->PendingError() == true)
1078       return false;
1079    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1080                   Files.begin()+EndOfSource,Files.end()) == false)
1081       return false;
1082
1083    if (_error->PendingError() == true)
1084       return false;
1085    *OutMap = Map.UnGuard();
1086
1087    return true;
1088 }
1089                                                                         /*}}}*/