apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <vector>
  31
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36
  37 #include <apt-pkg/deblistparser.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108    debListParser *debian(dynamic_cast<debListParser *>(&List));
 109
 110    unsigned int Counter = 0;
 111   step:
 112    while (List.Step() == true)
 113    {
 114       // Get a pointer to the package structure
 115       srkString PackageName;
 116       if (debian != NULL)
 117          PackageName = debian->Find("Package");
 118       else
 119          PackageName = List.Package();
 120       if (PackageName.empty() == true)
 121          return false;
 122
 123       pkgCache::PkgIterator Pkg;
 124       if (NewPackage(Pkg,PackageName) == false) {
 125          _error->Warning(_("Error occurred while processing %s (NewPackage)"),std::string(PackageName).c_str());
 126          goto step;
 127       }
 128
 129       Counter++;
 130       if (Counter % 100 == 0 && Progress != 0)
 131          Progress->Progress(List.Offset());
 132
 133       string language(List.DescriptionLanguage());
 134
 135       /* Get a pointer to the version structure. We know the list is sorted
 136          so we use that fact in the search. Insertion of new versions is
 137          done with correct sorting */
 138       srkString Version;
 139       if (debian != NULL)
 140          Version = debian->Find("Version");
 141       else
 142          Version = List.Version();
 143       if (Version.empty() == true)
 144       {
 145          // we first process the package, then the descriptions
 146          // (this has the bonus that we get MMap error when we run out
 147          //  of MMap space)
 148          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false) {
 149             _error->Warning(_("Error occurred while processing %s (UsePackage1)"),
 150                                  std::string(PackageName).c_str());
 151             goto step;
 152          }
 153
 154          // Find the right version to write the description
 155          MD5SumValue CurMd5 = List.Description_md5();
 156          pkgCache::VerIterator Ver = Pkg.VersionList();
 157          map_ptrloc *LastVer = &Pkg->VersionList;
 158
 159          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 160          {
 161             pkgCache::DescIterator Desc = Ver.DescriptionList();
 162             map_ptrloc *LastDesc = &Ver->DescriptionList;
 163             bool duplicate=false;
 164
 165             // don't add a new description if we have one for the given
 166             // md5 && language
 167             for ( ; Desc.end() == false; Desc++)
 168                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 169                    Desc.LanguageCode() == language)
 170                   duplicate=true;
 171             if(duplicate)
 172                continue;
 173
 174             for (Desc = Ver.DescriptionList();
 175                  Desc.end() == false;
 176                  LastDesc = &Desc->NextDesc, Desc++)
 177             {
 178                if (MD5SumValue(Desc.md5()) == CurMd5)
 179                {
 180                   // Add new description
 181                   *LastDesc = NewDescription(Desc, language, CurMd5, *LastDesc);
 182                   Desc->ParentPkg = Pkg.Index();
 183
 184                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) {
 185                      _error->Warning(_("Error occurred while processing %s (NewFileDesc1)"),std::string(PackageName).c_str());
 186                      goto step;
 187                   }
 188                   break;
 189                }
 190             }
 191          }
 192
 193          continue;
 194       }
 195
 196       pkgCache::VerIterator Ver = Pkg.VersionList();
 197       map_ptrloc *LastVer = &Pkg->VersionList;
 198       int Res = 1;
 199       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 200       {
 201          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 202          if (Res >= 0)
 203             break;
 204       }
 205
 206       /* We already have a version for this item, record that we
 207          saw it */
 208       unsigned long Hash = List.VersionHash();
 209       if (Res == 0 && Ver->Hash == Hash)
 210       {
 211          if (List.UsePackage(Pkg,Ver) == false) {
 212             _error->Warning(_("Error occurred while processing %s (UsePackage2)"),
 213                                  std::string(PackageName).c_str());
 214             goto step;
 215          }
 216
 217          if (NewFileVer(Ver,List) == false) {
 218             _error->Warning(_("Error occurred while processing %s (NewFileVer1)"),
 219                                  std::string(PackageName).c_str());
 220             goto step;
 221          }
 222
 223          // Read only a single record and return
 224          if (OutVer != 0)
 225          {
 226             *OutVer = Ver;
 227             FoundFileDeps |= List.HasFileDeps();
 228             return true;
 229          }
 230
 231          continue;
 232       }
 233
 234       // Skip to the end of the same version set.
 235       if (Res == 0)
 236       {
 237          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 238          {
 239             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 240             if (Res != 0)
 241                break;
 242          }
 243       }
 244
 245       // Add a new version
 246       *LastVer = NewVersion(Ver,Version,*LastVer);
 247       Ver->ParentPkg = Pkg.Index();
 248       Ver->Hash = Hash;
 249
 250       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false) {
 251          _error->Warning(_("Error occurred while processing %s (NewVersion1)"),
 252                               std::string(PackageName).c_str());
 253          goto step;
 254       }
 255
 256       if (List.UsePackage(Pkg,Ver) == false) {
 257          _error->Warning(_("Error occurred while processing %s (UsePackage3)"),
 258                               std::string(PackageName).c_str());
 259          goto step;
 260       }
 261
 262       if (NewFileVer(Ver,List) == false) {
 263          _error->Warning(_("Error occurred while processing %s (NewVersion2)"),
 264                               std::string(PackageName).c_str());
 265          goto step;
 266       }
 267
 268       // Read only a single record and return
 269       if (OutVer != 0)
 270       {
 271          *OutVer = Ver;
 272          FoundFileDeps |= List.HasFileDeps();
 273          return true;
 274       }
 275
 276       /* Record the Description data. Description data always exist in
 277          Packages and Translation-* files. */
 278       pkgCache::DescIterator Desc = Ver.DescriptionList();
 279       map_ptrloc *LastDesc = &Ver->DescriptionList;
 280
 281       // Skip to the end of description set
 282       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 283
 284       // Add new description
 285       *LastDesc = NewDescription(Desc, language, List.Description_md5(), *LastDesc);
 286       Desc->ParentPkg = Pkg.Index();
 287
 288       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) {
 289          _error->Warning(_("Error occurred while processing %s (NewFileDesc2)"),std::string(PackageName).c_str());
 290          goto step;
 291       }
 292    }
 293
 294    FoundFileDeps |= List.HasFileDeps();
 295
 296    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 297       return _error->Error(_("Wow, you exceeded the number of package "
 298                              "names this APT is capable of."));
 299    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID1)*8+sizeof(Cache.VerP->ID2)*8))-1)
 300       return _error->Error(_("Wow, you exceeded the number of versions "
 301                              "this APT is capable of."));
 302    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 303       return _error->Error(_("Wow, you exceeded the number of descriptions "
 304                              "this APT is capable of."));
 305    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 306       return _error->Error(_("Wow, you exceeded the number of dependencies "
 307                              "this APT is capable of."));
 308    return true;
 309 }
 310                                                                         /*}}}*/
 311 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* If we found any file depends while parsing the main list we need to
 314    resolve them. Since it is undesired to load the entire list of files
 315    into the cache as virtual packages we do a two stage effort. MergeList
 316    identifies the file depends and this creates Provdies for them by
 317    re-parsing all the indexs. */
 318 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 319 {
 320    List.Owner = this;
 321
 322    unsigned int Counter = 0;
 323    while (List.Step() == true)
 324    {
 325       string PackageName = List.Package();
 326       if (PackageName.empty() == true)
 327          return false;
 328       string Version = List.Version();
 329       if (Version.empty() == true)
 330          continue;
 331
 332       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 333       if (Pkg.end() == true)
 334          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 335                                 PackageName.c_str());
 336       Counter++;
 337       if (Counter % 100 == 0 && Progress != 0)
 338          Progress->Progress(List.Offset());
 339
 340       unsigned long Hash = List.VersionHash();
 341       pkgCache::VerIterator Ver = Pkg.VersionList();
 342       for (; Ver.end() == false; Ver++)
 343       {
 344          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 345          {
 346             if (List.CollectFileProvides(Cache,Ver) == false)
 347                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 348             break;
 349          }
 350       }
 351
 352       if (Ver.end() == true)
 353          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 354    }
 355
 356    return true;
 357 }
 358                                                                         /*}}}*/
 359 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* This creates a new package structure and adds it to the hash table */
 362 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 363 {
 364    return NewPackage(Pkg, srkString(Name));
 365 }
 366
 367 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const srkString &Name)
 368 {
 369    Pkg = Cache.FindPkg(Name);
 370    if (Pkg.end() == false)
 371       return true;
 372
 373    // Get a structure
 374    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 375    if (Package == 0)
 376       return false;
 377
 378    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 379
 380    // Insert it into the hash table
 381    unsigned long Hash = Cache.Hash(Name);
 382    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 383    Cache.HeaderP->HashTable[Hash] = Package;
 384
 385    // Set the name and the ID
 386    Pkg->Name = Map.WriteString(Name.Start,Name.Size);
 387    if (Pkg->Name == 0)
 388       return false;
 389    Pkg->ID = Cache.HeaderP->PackageCount++;
 390
 391    return true;
 392 }
 393                                                                         /*}}}*/
 394 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 395 // ---------------------------------------------------------------------
 396 /* */
 397 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 398                                    ListParser &List)
 399 {
 400    if (CurrentFile == 0)
 401       return true;
 402
 403    // Get a structure
 404    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 405    if (VerFile == 0)
 406       return 0;
 407
 408    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 409    VF->File = CurrentFile - Cache.PkgFileP;
 410
 411    // Link it to the end of the list
 412    map_ptrloc *Last = &Ver->FileList;
 413    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 414       Last = &V->NextFile;
 415    VF->NextFile = *Last;
 416    *Last = VF.Index();
 417
 418    VF->Offset = List.Offset();
 419    VF->Size = List.Size();
 420    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 421       Cache.HeaderP->MaxVerFileSize = VF->Size;
 422    Cache.HeaderP->VerFileCount++;
 423
 424    return true;
 425 }
 426                                                                         /*}}}*/
 427 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 428 // ---------------------------------------------------------------------
 429 /* This puts a version structure in the linked list */
 430 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 431                                             const string &VerStr,
 432                                             unsigned long Next)
 433 {
 434    return NewVersion(Ver, srkString(VerStr), Next);
 435 }
 436
 437 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 438                                             const srkString &VerStr,
 439                                             unsigned long Next)
 440 {
 441    // Get a structure
 442    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 443    if (Version == 0)
 444       return 0;
 445
 446    // Fill it in
 447    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 448    Ver->NextVer = Next;
 449    unsigned int ID = Cache.HeaderP->VersionCount++;
 450    Ver->ID1 = ID & 0xffff;
 451    Ver->ID2 = ID >> 16;
 452    Ver->VerStr = Map.WriteString(VerStr.Start, VerStr.Size);
 453    if (Ver->VerStr == 0)
 454       return 0;
 455
 456    return Version;
 457 }
 458                                                                         /*}}}*/
 459 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 460 // ---------------------------------------------------------------------
 461 /* */
 462 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 463                                    ListParser &List)
 464 {
 465    if (CurrentFile == 0)
 466       return true;
 467
 468    // Get a structure
 469    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 470    if (DescFile == 0)
 471       return false;
 472
 473    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 474    DF->File = CurrentFile - Cache.PkgFileP;
 475
 476    // Link it to the end of the list
 477    map_ptrloc *Last = &Desc->FileList;
 478    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 479       Last = &D->NextFile;
 480
 481    DF->NextFile = *Last;
 482    *Last = DF.Index();
 483
 484    DF->Offset = List.Offset();
 485    DF->Size = List.Size();
 486    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 487       Cache.HeaderP->MaxDescFileSize = DF->Size;
 488    Cache.HeaderP->DescFileCount++;
 489
 490    return true;
 491 }
 492                                                                         /*}}}*/
 493 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* This puts a description structure in the linked list */
 496 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 497                                             const string &Lang,
 498                                             const MD5SumValue &md5sum,
 499                                             map_ptrloc Next)
 500 {
 501    // Get a structure
 502    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 503    if (Description == 0)
 504       return 0;
 505
 506    // Fill it in
 507    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 508    Desc->NextDesc = Next;
 509    Desc->ID = Cache.HeaderP->DescriptionCount++;
 510    Desc->language_code = Map.WriteString(Lang);
 511    Desc->md5sum = Map.WriteString(md5sum.Value());
 512    if (Desc->language_code == 0 || Desc->md5sum == 0)
 513       return 0;
 514
 515    return Description;
 516 }
 517                                                                         /*}}}*/
 518 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 519 // ---------------------------------------------------------------------
 520 /* This creates a dependency element in the tree. It is linked to the
 521    version and to the package that it is pointing to. */
 522 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 523                                                const string &PackageName,
 524                                                const string &Version,
 525                                                unsigned int Op,
 526                                                unsigned int Type)
 527 {
 528     return NewDepends(Ver, srkString(PackageName), srkString(Version), Op, Type);
 529 }
 530
 531 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 532                                                const srkString &PackageName,
 533                                                const srkString &Version,
 534                                                unsigned int Op,
 535                                                unsigned int Type)
 536 {
 537    pkgCache &Cache = Owner->Cache;
 538
 539    // Get a structure
 540    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 541    if (Dependency == 0)
 542       return false;
 543
 544    // Fill it in
 545    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 546    Dep->ParentVer = Ver.Index();
 547    Dep->Type = Type;
 548    Dep->CompareOp = Op;
 549    Dep->ID = Cache.HeaderP->DependsCount++;
 550
 551    // Locate the target package
 552    pkgCache::PkgIterator Pkg;
 553    if (Owner->NewPackage(Pkg,PackageName) == false)
 554       return false;
 555
 556    // Probe the reverse dependency list for a version string that matches
 557    if (Version.empty() == false)
 558    {
 559 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 560          if (I->Version != 0 && I.TargetVer() == Version)
 561             Dep->Version = I->Version;*/
 562       if (Dep->Version == 0)
 563          if ((Dep->Version = WriteString(Version)) == 0)
 564             return false;
 565    }
 566
 567    // Link it to the package
 568    Dep->Package = Pkg.Index();
 569    Dep->NextRevDepends = Pkg->RevDepends;
 570    Pkg->RevDepends = Dep.Index();
 571
 572    /* Link it to the version (at the end of the list)
 573       Caching the old end point speeds up generation substantially */
 574    if (OldDepVer != Ver)
 575    {
 576       OldDepLast = &Ver->DependsList;
 577       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 578          OldDepLast = &D->NextDepends;
 579       OldDepVer = Ver;
 580    }
 581
 582    // Is it a file dependency?
 583    if (PackageName[0] == '/')
 584       FoundFileDeps = true;
 585
 586    Dep->NextDepends = *OldDepLast;
 587    *OldDepLast = Dep.Index();
 588    OldDepLast = &Dep->NextDepends;
 589
 590    return true;
 591 }
 592                                                                         /*}}}*/
 593 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 594 // ---------------------------------------------------------------------
 595 /* */
 596 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 597                                                 const string &PackageName,
 598                                                 const string &Version)
 599 {
 600    return NewProvides(Ver, srkString(PackageName), srkString(Version));
 601 }
 602
 603 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 604                                                 const srkString &PackageName,
 605                                                 const srkString &Version)
 606 {
 607    pkgCache &Cache = Owner->Cache;
 608
 609    // We do not add self referencing provides
 610    if (Ver.ParentPkg().Name() == PackageName)
 611       return true;
 612
 613    // Get a structure
 614    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 615    if (Provides == 0)
 616       return false;
 617    Cache.HeaderP->ProvidesCount++;
 618
 619    // Fill it in
 620    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 621    Prv->Version = Ver.Index();
 622    Prv->NextPkgProv = Ver->ProvidesList;
 623    Ver->ProvidesList = Prv.Index();
 624    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 625       return false;
 626
 627    // Locate the target package
 628    pkgCache::PkgIterator Pkg;
 629    if (Owner->NewPackage(Pkg,PackageName) == false)
 630       return false;
 631
 632    // Link it to the package
 633    Prv->ParentPkg = Pkg.Index();
 634    Prv->NextProvides = Pkg->ProvidesList;
 635    Pkg->ProvidesList = Prv.Index();
 636
 637    return true;
 638 }
 639                                                                         /*}}}*/
 640 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 641 // ---------------------------------------------------------------------
 642 /* */
 643 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 644                                            const char *NameStart,
 645                                            unsigned int NameSize)
 646 {
 647    pkgCache &Cache = Owner->Cache;
 648
 649    // Get a structure
 650    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 651    if (Tagg == 0)
 652       return false;
 653    Cache.HeaderP->TagCount++;
 654
 655    // Fill it in
 656    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 657    Tg->Name = WriteString(NameStart,NameSize);
 658    if (Tg->Name == 0)
 659       return false;
 660    Tg->NextTag = Pkg->TagList;
 661    Pkg->TagList = Tg.Index();
 662
 663    return true;
 664 }
 665                                                                         /*}}}*/
 666 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 667 // ---------------------------------------------------------------------
 668 /* This is used to select which file is to be associated with all newly
 669    added versions. The caller is responsible for setting the IMS fields. */
 670 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 671                                    const pkgIndexFile &Index,
 672                                    unsigned long Flags)
 673 {
 674    // Get some space for the structure
 675    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 676    if (CurrentFile == Cache.PkgFileP)
 677       return false;
 678
 679    // Fill it in
 680    CurrentFile->FileName = Map.WriteString(File);
 681    CurrentFile->Site = WriteUniqString(Site);
 682    CurrentFile->NextFile = Cache.HeaderP->FileList;
 683    CurrentFile->Flags = Flags;
 684    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 685    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 686    PkgFileName = File;
 687    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 688    Cache.HeaderP->PackageFileCount++;
 689
 690    if (CurrentFile->FileName == 0)
 691       return false;
 692
 693    if (Progress != 0)
 694       Progress->SubProgress(Index.Size());
 695    return true;
 696 }
 697                                                                         /*}}}*/
 698 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 699 // ---------------------------------------------------------------------
 700 /* This is used to create handles to strings. Given the same text it
 701    always returns the same number */
 702 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 703                                                  unsigned int Size)
 704 {
 705    /* We use a very small transient hash table here, this speeds up generation
 706       by a fair amount on slower machines */
 707    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 708    if (Bucket != 0 &&
 709        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 710       return Bucket->String;
 711
 712    // Search for an insertion point
 713    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 714    int Res = 1;
 715    map_ptrloc *Last = &Cache.HeaderP->StringList;
 716    for (; I != Cache.StringItemP; Last = &I->NextItem,
 717         I = Cache.StringItemP + I->NextItem)
 718    {
 719       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 720       if (Res >= 0)
 721          break;
 722    }
 723
 724    // Match
 725    if (Res == 0)
 726    {
 727       Bucket = I;
 728       return I->String;
 729    }
 730
 731    // Get a structure
 732    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 733    if (Item == 0)
 734       return 0;
 735
 736    // Fill in the structure
 737    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 738    ItemP->NextItem = I - Cache.StringItemP;
 739    *Last = Item;
 740    ItemP->String = Map.WriteString(S,Size);
 741    if (ItemP->String == 0)
 742       return 0;
 743
 744    Bucket = ItemP;
 745    return ItemP->String;
 746 }
 747                                                                         /*}}}*/
 748 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 749 // ---------------------------------------------------------------------
 750 /* This just verifies that each file in the list of index files exists,
 751    has matching attributes with the cache and the cache does not have
 752    any extra files. */
 753 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 754                           FileIterator End,MMap **OutMap = 0)
 755 {
 756    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 757    // No file, certainly invalid
 758    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 759    {
 760       if (Debug == true)
 761          std::clog << "CacheFile doesn't exist" << std::endl;
 762       return false;
 763    }
 764
 765    // Map it
 766    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 767    SPtr<MMap> Map = new MMap(CacheF,0);
 768    pkgCache Cache(Map);
 769    if (_error->PendingError() == true || Map->Size() == 0)
 770    {
 771       if (Debug == true)
 772          std::clog << "Errors are pending or Map is empty()" << std::endl;
 773       _error->Discard();
 774       return false;
 775    }
 776
 777    /* Now we check every index file, see if it is in the cache,
 778       verify the IMS data and check that it is on the disk too.. */
 779    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 780    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 781    for (; Start != End; Start++)
 782    {
 783       if (Debug == true)
 784          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 785       if ((*Start)->HasPackages() == false)
 786       {
 787          if (Debug == true)
 788             std::clog << "Has NO packages" << std::endl;
 789          continue;
 790       }
 791
 792       if ((*Start)->Exists() == false)
 793       {
 794 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 795          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 796                           (*Start)->Describe().c_str());
 797 #endif
 798          if (Debug == true)
 799             std::clog << "file doesn't exist" << std::endl;
 800          continue;
 801       }
 802
 803       // FindInCache is also expected to do an IMS check.
 804       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 805       if (File.end() == true)
 806       {
 807          if (Debug == true)
 808             std::clog << "FindInCache returned end-Pointer" << std::endl;
 809          return false;
 810       }
 811
 812       Visited[File->ID] = true;
 813       if (Debug == true)
 814          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 815    }
 816
 817    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 818       if (Visited[I] == false)
 819       {
 820          if (Debug == true)
 821             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 822          return false;
 823       }
 824
 825    if (_error->PendingError() == true)
 826    {
 827       if (Debug == true)
 828       {
 829          std::clog << "Validity failed because of pending errors:" << std::endl;
 830          _error->DumpErrors();
 831       }
 832       _error->Discard();
 833       return false;
 834    }
 835
 836    if (OutMap != 0)
 837       *OutMap = Map.UnGuard();
 838    return true;
 839 }
 840                                                                         /*}}}*/
 841 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* Size is kind of an abstract notion that is only used for the progress
 844    meter */
 845 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 846 {
 847    unsigned long TotalSize = 0;
 848    for (; Start != End; Start++)
 849    {
 850       if ((*Start)->HasPackages() == false)
 851          continue;
 852       TotalSize += (*Start)->Size();
 853    }
 854    return TotalSize;
 855 }
 856                                                                         /*}}}*/
 857 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 858 // ---------------------------------------------------------------------
 859 /* */
 860 static bool BuildCache(pkgCacheGenerator &Gen,
 861                        OpProgress &Progress,
 862                        unsigned long &CurrentSize,unsigned long TotalSize,
 863                        FileIterator Start, FileIterator End)
 864 {
 865    FileIterator I;
 866    for (I = Start; I != End; I++)
 867    {
 868       if ((*I)->HasPackages() == false)
 869          continue;
 870
 871       if ((*I)->Exists() == false)
 872          continue;
 873
 874       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 875       {
 876          _error->Warning("Duplicate sources.list entry %s",
 877                          (*I)->Describe().c_str());
 878          continue;
 879       }
 880
 881       unsigned long Size = (*I)->Size();
 882       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 883       CurrentSize += Size;
 884
 885       if ((*I)->Merge(Gen,Progress) == false)
 886          return false;
 887    }
 888
 889    if (Gen.HasFileDeps() == true)
 890    {
 891       Progress.Done();
 892       TotalSize = ComputeSize(Start, End);
 893       CurrentSize = 0;
 894       for (I = Start; I != End; I++)
 895       {
 896          unsigned long Size = (*I)->Size();
 897          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 898          CurrentSize += Size;
 899          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 900             return false;
 901       }
 902    }
 903
 904    return true;
 905 }
 906                                                                         /*}}}*/
 907 // MakeStatusCache - Construct the status cache                         /*{{{*/
 908 // ---------------------------------------------------------------------
 909 /* This makes sure that the status cache (the cache that has all
 910    index files from the sources list and all local ones) is ready
 911    to be mmaped. If OutMap is not zero then a MMap object representing
 912    the cache will be stored there. This is pretty much mandetory if you
 913    are using AllowMem. AllowMem lets the function be run as non-root
 914    where it builds the cache 'fast' into a memory buffer. */
 915 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 916                         MMap **OutMap,bool AllowMem)
 917 {
 918    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 919    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 920
 921    vector<pkgIndexFile *> Files;
 922    for (vector<metaIndex *>::const_iterator i = List.begin();
 923         i != List.end();
 924         i++)
 925    {
 926       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 927       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 928            j != Indexes->end();
 929            j++)
 930          Files.push_back (*j);
 931    }
 932
 933    unsigned long const EndOfSource = Files.size();
 934    if (_system->AddStatusFiles(Files) == false)
 935       return false;
 936
 937    // Decide if we can write to the files..
 938    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 939    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 940
 941    // Decide if we can write to the cache
 942    bool Writeable = false;
 943    if (CacheFile.empty() == false)
 944       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 945    else
 946       if (SrcCacheFile.empty() == false)
 947          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 948    if (Debug == true)
 949       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
 950
 951    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 952       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 953
 954    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 955
 956    // Cache is OK, Fin.
 957    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 958    {
 959       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 960       if (Debug == true)
 961          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
 962       return true;
 963    }
 964    else if (Debug == true)
 965          std::clog << "pkgcache.bin is NOT valid" << std::endl;
 966
 967    /* At this point we know we need to reconstruct the package cache,
 968       begin. */
 969    SPtr<FileFd> CacheF;
 970    SPtr<DynamicMMap> Map;
 971    if (Writeable == true && CacheFile.empty() == false)
 972    {
 973       unlink(CacheFile.c_str());
 974       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 975       fchmod(CacheF->Fd(),0644);
 976       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 977       if (_error->PendingError() == true)
 978          return false;
 979       if (Debug == true)
 980          std::clog << "Open filebased MMap" << std::endl;
 981    }
 982    else
 983    {
 984       // Just build it in memory..
 985       Map = new DynamicMMap(0,MapSize);
 986       if (Debug == true)
 987          std::clog << "Open memory Map (not filebased)" << std::endl;
 988    }
 989
 990    // Lets try the source cache.
 991    unsigned long CurrentSize = 0;
 992    unsigned long TotalSize = 0;
 993    if (CheckValidity(SrcCacheFile,Files.begin(),
 994                      Files.begin()+EndOfSource) == true)
 995    {
 996       if (Debug == true)
 997          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
 998       // Preload the map with the source cache
 999       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1000       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1001       if ((alloc == 0 && _error->PendingError())
1002                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1003                                 SCacheF.Size()) == false)
1004          return false;
1005
1006       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1007
1008       // Build the status cache
1009       pkgCacheGenerator Gen(Map.Get(),&Progress);
1010       if (_error->PendingError() == true)
1011          return false;
1012       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1013                      Files.begin()+EndOfSource,Files.end()) == false)
1014          return false;
1015    }
1016    else
1017    {
1018       if (Debug == true)
1019          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1020       TotalSize = ComputeSize(Files.begin(),Files.end());
1021
1022       // Build the source cache
1023       pkgCacheGenerator Gen(Map.Get(),&Progress);
1024       if (_error->PendingError() == true)
1025          return false;
1026       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1027                      Files.begin(),Files.begin()+EndOfSource) == false)
1028          return false;
1029
1030       // Write it back
1031       if (Writeable == true && SrcCacheFile.empty() == false)
1032       {
1033          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
1034          if (_error->PendingError() == true)
1035             return false;
1036
1037          fchmod(SCacheF.Fd(),0644);
1038
1039          // Write out the main data
1040          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1041             return _error->Error(_("IO Error saving source cache"));
1042          SCacheF.Sync();
1043
1044          // Write out the proper header
1045          Gen.GetCache().HeaderP->Dirty = false;
1046          if (SCacheF.Seek(0) == false ||
1047              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1048             return _error->Error(_("IO Error saving source cache"));
1049          Gen.GetCache().HeaderP->Dirty = true;
1050          SCacheF.Sync();
1051       }
1052
1053       // Build the status cache
1054       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1055                      Files.begin()+EndOfSource,Files.end()) == false)
1056          return false;
1057    }
1058    if (Debug == true)
1059       std::clog << "Caches are ready for shipping" << std::endl;
1060
1061    if (_error->PendingError() == true)
1062       return false;
1063    if (OutMap != 0)
1064    {
1065       if (CacheF != 0)
1066       {
1067          delete Map.UnGuard();
1068          *OutMap = new MMap(*CacheF,0);
1069       }
1070       else
1071       {
1072          *OutMap = Map.UnGuard();
1073       }
1074    }
1075
1076    return true;
1077 }
1078                                                                         /*}}}*/
1079 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
1080 // ---------------------------------------------------------------------
1081 /* */
1082 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1083 {
1084    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1085    vector<pkgIndexFile *> Files;
1086    unsigned long EndOfSource = Files.size();
1087    if (_system->AddStatusFiles(Files) == false)
1088       return false;
1089
1090    SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
1091    unsigned long CurrentSize = 0;
1092    unsigned long TotalSize = 0;
1093
1094    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1095
1096    // Build the status cache
1097    Progress.OverallProgress(0,1,1,_("Reading package lists"));
1098    pkgCacheGenerator Gen(Map.Get(),&Progress);
1099    if (_error->PendingError() == true)
1100       return false;
1101    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1102                   Files.begin()+EndOfSource,Files.end()) == false)
1103       return false;
1104
1105    if (_error->PendingError() == true)
1106       return false;
1107    *OutMap = Map.UnGuard();
1108
1109    return true;
1110 }
1111                                                                         /*}}}*/