apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <vector>
  31
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36
  37 #include <apt-pkg/deblistparser.h>
  38                                                                         /*}}}*/
  39 typedef vector<pkgIndexFile *>::iterator FileIterator;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the diry flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  66       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  67       Cache.ReMap();
  68    }
  69    else
  70    {
  71       // Map directly from the existing file
  72       Cache.ReMap();
  73       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  74       if (Cache.VS != _system->VS)
  75       {
  76          _error->Error(_("Cache has an incompatible versioning system"));
  77          return;
  78       }
  79    }
  80
  81    Cache.HeaderP->Dirty = true;
  82    Map.Sync(0,sizeof(pkgCache::Header));
  83 }
  84                                                                         /*}}}*/
  85 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* We sync the data then unset the dirty flag in two steps so as to
  88    advoid a problem during a crash */
  89 pkgCacheGenerator::~pkgCacheGenerator()
  90 {
  91    if (_error->PendingError() == true)
  92       return;
  93    if (Map.Sync() == false)
  94       return;
  95
  96    Cache.HeaderP->Dirty = false;
  97    Map.Sync(0,sizeof(pkgCache::Header));
  98 }
  99                                                                         /*}}}*/
 100 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* This provides the generation of the entries in the cache. Each loop
 103    goes through a single package record from the underlying parse engine. */
 104 bool pkgCacheGenerator::MergeList(ListParser &List,
 105                                   pkgCache::VerIterator *OutVer)
 106 {
 107    List.Owner = this;
 108    debListParser *debian(dynamic_cast<debListParser *>(&List));
 109
 110    unsigned int Counter = 0;
 111   step:
 112    while (List.Step() == true)
 113    {
 114       // Get a pointer to the package structure
 115       srkString PackageName;
 116       if (debian != NULL)
 117          PackageName = debian->Find("Package");
 118       else
 119          PackageName = List.Package();
 120       if (PackageName.empty() == true)
 121          return false;
 122
 123       pkgCache::PkgIterator Pkg;
 124       if (NewPackage(Pkg,PackageName) == false) {
 125          _error->Warning(_("Error occurred while processing %s (NewPackage)"),std::string(PackageName).c_str());
 126          goto step;
 127       }
 128
 129       Counter++;
 130       if (Counter % 100 == 0 && Progress != 0)
 131          Progress->Progress(List.Offset());
 132
 133       string language(List.DescriptionLanguage());
 134
 135       /* Get a pointer to the version structure. We know the list is sorted
 136          so we use that fact in the search. Insertion of new versions is
 137          done with correct sorting */
 138       srkString Version;
 139       if (debian != NULL)
 140          Version = debian->Find("Version");
 141       else
 142          Version = List.Version();
 143       if (Version.empty() == true)
 144       {
 145          // we first process the package, then the descriptions
 146          // (this has the bonus that we get MMap error when we run out
 147          //  of MMap space)
 148          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false) {
 149             _error->Warning(_("Error occurred while processing %s (UsePackage1)"),
 150                                  std::string(PackageName).c_str());
 151             goto step;
 152          }
 153
 154          // Find the right version to write the description
 155          MD5SumValue CurMd5 = List.Description_md5();
 156          pkgCache::VerIterator Ver = Pkg.VersionList();
 157          map_ptrloc *LastVer = &Pkg->VersionList;
 158
 159          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 160          {
 161             pkgCache::DescIterator Desc = Ver.DescriptionList();
 162             map_ptrloc *LastDesc = &Ver->DescriptionList;
 163             bool duplicate=false;
 164
 165             // don't add a new description if we have one for the given
 166             // md5 && language
 167             for ( ; Desc.end() == false; Desc++)
 168                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 169                    Desc.LanguageCode() == language)
 170                   duplicate=true;
 171             if(duplicate)
 172                continue;
 173
 174             for (Desc = Ver.DescriptionList();
 175                  Desc.end() == false;
 176                  LastDesc = &Desc->NextDesc, Desc++)
 177             {
 178                if (MD5SumValue(Desc.md5()) == CurMd5)
 179                {
 180                   // Add new description
 181                   *LastDesc = NewDescription(Desc, language, CurMd5, *LastDesc);
 182                   Desc->ParentPkg = Pkg.Index();
 183
 184                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) {
 185                      _error->Warning(_("Error occurred while processing %s (NewFileDesc1)"),std::string(PackageName).c_str());
 186                      goto step;
 187                   }
 188                   break;
 189                }
 190             }
 191          }
 192
 193          continue;
 194       }
 195
 196       pkgCache::VerIterator Ver = Pkg.VersionList();
 197       map_ptrloc *LastVer = &Pkg->VersionList;
 198       int Res = 1;
 199       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 200       {
 201          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 202          if (Res >= 0)
 203             break;
 204       }
 205
 206       /* We already have a version for this item, record that we
 207          saw it */
 208       unsigned long Hash = List.VersionHash();
 209       if (Res == 0 && Ver->Hash == Hash)
 210       {
 211          if (List.UsePackage(Pkg,Ver) == false) {
 212             _error->Warning(_("Error occurred while processing %s (UsePackage2)"),
 213                                  std::string(PackageName).c_str());
 214             goto step;
 215          }
 216
 217          if (NewFileVer(Ver,List) == false) {
 218             _error->Warning(_("Error occurred while processing %s (NewFileVer1)"),
 219                                  std::string(PackageName).c_str());
 220             goto step;
 221          }
 222
 223          // Read only a single record and return
 224          if (OutVer != 0)
 225          {
 226             *OutVer = Ver;
 227             FoundFileDeps |= List.HasFileDeps();
 228             return true;
 229          }
 230
 231          continue;
 232       }
 233
 234       // Skip to the end of the same version set.
 235       if (Res == 0)
 236       {
 237          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 238          {
 239             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 240             if (Res != 0)
 241                break;
 242          }
 243       }
 244
 245       // Add a new version
 246       *LastVer = NewVersion(Ver,Version,*LastVer);
 247       Ver->ParentPkg = Pkg.Index();
 248       Ver->Hash = Hash;
 249
 250       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false) {
 251          _error->Warning(_("Error occurred while processing %s (NewVersion1)"),
 252                               std::string(PackageName).c_str());
 253          goto step;
 254       }
 255
 256       if (List.UsePackage(Pkg,Ver) == false) {
 257          _error->Warning(_("Error occurred while processing %s (UsePackage3)"),
 258                               std::string(PackageName).c_str());
 259          goto step;
 260       }
 261
 262       if (NewFileVer(Ver,List) == false) {
 263          _error->Warning(_("Error occurred while processing %s (NewVersion2)"),
 264                               std::string(PackageName).c_str());
 265          goto step;
 266       }
 267
 268       // Read only a single record and return
 269       if (OutVer != 0)
 270       {
 271          *OutVer = Ver;
 272          FoundFileDeps |= List.HasFileDeps();
 273          return true;
 274       }
 275
 276       /* Record the Description data. Description data always exist in
 277          Packages and Translation-* files. */
 278       pkgCache::DescIterator Desc = Ver.DescriptionList();
 279       map_ptrloc *LastDesc = &Ver->DescriptionList;
 280
 281       // Skip to the end of description set
 282       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 283
 284       // Add new description
 285       *LastDesc = NewDescription(Desc, language, List.Description_md5(), *LastDesc);
 286       Desc->ParentPkg = Pkg.Index();
 287
 288       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) {
 289          _error->Warning(_("Error occurred while processing %s (NewFileDesc2)"),std::string(PackageName).c_str());
 290          goto step;
 291       }
 292    }
 293
 294    FoundFileDeps |= List.HasFileDeps();
 295
 296    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 297       return _error->Error(_("Wow, you exceeded the number of package "
 298                              "names this APT is capable of."));
 299    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 300       return _error->Error(_("Wow, you exceeded the number of versions "
 301                              "this APT is capable of."));
 302    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 303       return _error->Error(_("Wow, you exceeded the number of descriptions "
 304                              "this APT is capable of."));
 305    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 306       return _error->Error(_("Wow, you exceeded the number of dependencies "
 307                              "this APT is capable of."));
 308    return true;
 309 }
 310                                                                         /*}}}*/
 311 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* If we found any file depends while parsing the main list we need to
 314    resolve them. Since it is undesired to load the entire list of files
 315    into the cache as virtual packages we do a two stage effort. MergeList
 316    identifies the file depends and this creates Provdies for them by
 317    re-parsing all the indexs. */
 318 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 319 {
 320    List.Owner = this;
 321
 322    unsigned int Counter = 0;
 323    while (List.Step() == true)
 324    {
 325       string PackageName = List.Package();
 326       if (PackageName.empty() == true)
 327          return false;
 328       string Version = List.Version();
 329       if (Version.empty() == true)
 330          continue;
 331
 332       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 333       if (Pkg.end() == true)
 334          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 335                                 PackageName.c_str());
 336       Counter++;
 337       if (Counter % 100 == 0 && Progress != 0)
 338          Progress->Progress(List.Offset());
 339
 340       unsigned long Hash = List.VersionHash();
 341       pkgCache::VerIterator Ver = Pkg.VersionList();
 342       for (; Ver.end() == false; Ver++)
 343       {
 344          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 345          {
 346             if (List.CollectFileProvides(Cache,Ver) == false)
 347                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 348             break;
 349          }
 350       }
 351
 352       if (Ver.end() == true)
 353          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 354    }
 355
 356    return true;
 357 }
 358                                                                         /*}}}*/
 359 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* This creates a new package structure and adds it to the hash table */
 362 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 363 {
 364    return NewPackage(Pkg, srkString(Name));
 365 }
 366
 367 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const srkString &Name)
 368 {
 369    Pkg = Cache.FindPkg(Name);
 370    if (Pkg.end() == false)
 371       return true;
 372
 373    // Get a structure
 374    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 375    if (Package == 0)
 376       return false;
 377
 378    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 379
 380    // Insert it into the hash table
 381    unsigned long Hash = Cache.Hash(Name);
 382    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 383    Cache.HeaderP->HashTable[Hash] = Package;
 384
 385    // Set the name and the ID
 386    Pkg->Name = Map.WriteString(Name.Start,Name.Size);
 387    if (Pkg->Name == 0)
 388       return false;
 389    Pkg->ID = Cache.HeaderP->PackageCount++;
 390
 391    return true;
 392 }
 393                                                                         /*}}}*/
 394 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 395 // ---------------------------------------------------------------------
 396 /* */
 397 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 398                                    ListParser &List)
 399 {
 400    if (CurrentFile == 0)
 401       return true;
 402
 403    // Get a structure
 404    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 405    if (VerFile == 0)
 406       return 0;
 407
 408    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 409    VF->File = CurrentFile - Cache.PkgFileP;
 410
 411    // Link it to the end of the list
 412    map_ptrloc *Last = &Ver->FileList;
 413    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 414       Last = &V->NextFile;
 415    VF->NextFile = *Last;
 416    *Last = VF.Index();
 417
 418    VF->Offset = List.Offset();
 419    VF->Size = List.Size();
 420    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 421       Cache.HeaderP->MaxVerFileSize = VF->Size;
 422    Cache.HeaderP->VerFileCount++;
 423
 424    return true;
 425 }
 426                                                                         /*}}}*/
 427 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 428 // ---------------------------------------------------------------------
 429 /* This puts a version structure in the linked list */
 430 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 431                                             const string &VerStr,
 432                                             unsigned long Next)
 433 {
 434    return NewVersion(Ver, srkString(VerStr), Next);
 435 }
 436
 437 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 438                                             const srkString &VerStr,
 439                                             unsigned long Next)
 440 {
 441    // Get a structure
 442    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 443    if (Version == 0)
 444       return 0;
 445
 446    // Fill it in
 447    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 448    Ver->NextVer = Next;
 449    Ver->ID = Cache.HeaderP->VersionCount++;
 450    Ver->VerStr = Map.WriteString(VerStr.Start, VerStr.Size);
 451    if (Ver->VerStr == 0)
 452       return 0;
 453
 454    return Version;
 455 }
 456                                                                         /*}}}*/
 457 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 458 // ---------------------------------------------------------------------
 459 /* */
 460 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 461                                    ListParser &List)
 462 {
 463    if (CurrentFile == 0)
 464       return true;
 465
 466    // Get a structure
 467    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 468    if (DescFile == 0)
 469       return false;
 470
 471    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 472    DF->File = CurrentFile - Cache.PkgFileP;
 473
 474    // Link it to the end of the list
 475    map_ptrloc *Last = &Desc->FileList;
 476    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 477       Last = &D->NextFile;
 478
 479    DF->NextFile = *Last;
 480    *Last = DF.Index();
 481
 482    DF->Offset = List.Offset();
 483    DF->Size = List.Size();
 484    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 485       Cache.HeaderP->MaxDescFileSize = DF->Size;
 486    Cache.HeaderP->DescFileCount++;
 487
 488    return true;
 489 }
 490                                                                         /*}}}*/
 491 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 492 // ---------------------------------------------------------------------
 493 /* This puts a description structure in the linked list */
 494 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 495                                             const string &Lang,
 496                                             const MD5SumValue &md5sum,
 497                                             map_ptrloc Next)
 498 {
 499    // Get a structure
 500    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 501    if (Description == 0)
 502       return 0;
 503
 504    // Fill it in
 505    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 506    Desc->NextDesc = Next;
 507    Desc->ID = Cache.HeaderP->DescriptionCount++;
 508    Desc->language_code = Map.WriteString(Lang);
 509    Desc->md5sum = Map.WriteString(md5sum.Value());
 510    if (Desc->language_code == 0 || Desc->md5sum == 0)
 511       return 0;
 512
 513    return Description;
 514 }
 515                                                                         /*}}}*/
 516 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 517 // ---------------------------------------------------------------------
 518 /* This creates a dependency element in the tree. It is linked to the
 519    version and to the package that it is pointing to. */
 520 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 521                                                const string &PackageName,
 522                                                const string &Version,
 523                                                unsigned int Op,
 524                                                unsigned int Type)
 525 {
 526     return NewDepends(Ver, srkString(PackageName), srkString(Version), Op, Type);
 527 }
 528
 529 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 530                                                const srkString &PackageName,
 531                                                const srkString &Version,
 532                                                unsigned int Op,
 533                                                unsigned int Type)
 534 {
 535    pkgCache &Cache = Owner->Cache;
 536
 537    // Get a structure
 538    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 539    if (Dependency == 0)
 540       return false;
 541
 542    // Fill it in
 543    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 544    Dep->ParentVer = Ver.Index();
 545    Dep->Type = Type;
 546    Dep->CompareOp = Op;
 547    Dep->ID = Cache.HeaderP->DependsCount++;
 548
 549    // Locate the target package
 550    pkgCache::PkgIterator Pkg;
 551    if (Owner->NewPackage(Pkg,PackageName) == false)
 552       return false;
 553
 554    // Probe the reverse dependency list for a version string that matches
 555    if (Version.empty() == false)
 556    {
 557 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 558          if (I->Version != 0 && I.TargetVer() == Version)
 559             Dep->Version = I->Version;*/
 560       if (Dep->Version == 0)
 561          if ((Dep->Version = WriteString(Version)) == 0)
 562             return false;
 563    }
 564
 565    // Link it to the package
 566    Dep->Package = Pkg.Index();
 567    Dep->NextRevDepends = Pkg->RevDepends;
 568    Pkg->RevDepends = Dep.Index();
 569
 570    /* Link it to the version (at the end of the list)
 571       Caching the old end point speeds up generation substantially */
 572    if (OldDepVer != Ver)
 573    {
 574       OldDepLast = &Ver->DependsList;
 575       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 576          OldDepLast = &D->NextDepends;
 577       OldDepVer = Ver;
 578    }
 579
 580    // Is it a file dependency?
 581    if (PackageName[0] == '/')
 582       FoundFileDeps = true;
 583
 584    Dep->NextDepends = *OldDepLast;
 585    *OldDepLast = Dep.Index();
 586    OldDepLast = &Dep->NextDepends;
 587
 588    return true;
 589 }
 590                                                                         /*}}}*/
 591 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* */
 594 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 595                                                 const string &PackageName,
 596                                                 const string &Version)
 597 {
 598    return NewProvides(Ver, srkString(PackageName), srkString(Version));
 599 }
 600
 601 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 602                                                 const srkString &PackageName,
 603                                                 const srkString &Version)
 604 {
 605    pkgCache &Cache = Owner->Cache;
 606
 607    // We do not add self referencing provides
 608    if (Ver.ParentPkg().Name() == PackageName)
 609       return true;
 610
 611    // Get a structure
 612    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 613    if (Provides == 0)
 614       return false;
 615    Cache.HeaderP->ProvidesCount++;
 616
 617    // Fill it in
 618    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 619    Prv->Version = Ver.Index();
 620    Prv->NextPkgProv = Ver->ProvidesList;
 621    Ver->ProvidesList = Prv.Index();
 622    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 623       return false;
 624
 625    // Locate the target package
 626    pkgCache::PkgIterator Pkg;
 627    if (Owner->NewPackage(Pkg,PackageName) == false)
 628       return false;
 629
 630    // Link it to the package
 631    Prv->ParentPkg = Pkg.Index();
 632    Prv->NextProvides = Pkg->ProvidesList;
 633    Pkg->ProvidesList = Prv.Index();
 634
 635    return true;
 636 }
 637                                                                         /*}}}*/
 638 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 639 // ---------------------------------------------------------------------
 640 /* */
 641 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 642                                            const char *NameStart,
 643                                            unsigned int NameSize)
 644 {
 645    pkgCache &Cache = Owner->Cache;
 646
 647    // Get a structure
 648    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 649    if (Tagg == 0)
 650       return false;
 651    Cache.HeaderP->TagCount++;
 652
 653    // Fill it in
 654    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 655    Tg->Name = WriteString(NameStart,NameSize);
 656    if (Tg->Name == 0)
 657       return false;
 658    Tg->NextTag = Pkg->TagList;
 659    Pkg->TagList = Tg.Index();
 660
 661    return true;
 662 }
 663                                                                         /*}}}*/
 664 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 665 // ---------------------------------------------------------------------
 666 /* This is used to select which file is to be associated with all newly
 667    added versions. The caller is responsible for setting the IMS fields. */
 668 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 669                                    const pkgIndexFile &Index,
 670                                    unsigned long Flags)
 671 {
 672    // Get some space for the structure
 673    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 674    if (CurrentFile == Cache.PkgFileP)
 675       return false;
 676
 677    // Fill it in
 678    CurrentFile->FileName = Map.WriteString(File);
 679    CurrentFile->Site = WriteUniqString(Site);
 680    CurrentFile->NextFile = Cache.HeaderP->FileList;
 681    CurrentFile->Flags = Flags;
 682    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 683    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 684    PkgFileName = File;
 685    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 686    Cache.HeaderP->PackageFileCount++;
 687
 688    if (CurrentFile->FileName == 0)
 689       return false;
 690
 691    if (Progress != 0)
 692       Progress->SubProgress(Index.Size());
 693    return true;
 694 }
 695                                                                         /*}}}*/
 696 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 697 // ---------------------------------------------------------------------
 698 /* This is used to create handles to strings. Given the same text it
 699    always returns the same number */
 700 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 701                                                  unsigned int Size)
 702 {
 703    /* We use a very small transient hash table here, this speeds up generation
 704       by a fair amount on slower machines */
 705    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 706    if (Bucket != 0 &&
 707        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 708       return Bucket->String;
 709
 710    // Search for an insertion point
 711    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 712    int Res = 1;
 713    map_ptrloc *Last = &Cache.HeaderP->StringList;
 714    for (; I != Cache.StringItemP; Last = &I->NextItem,
 715         I = Cache.StringItemP + I->NextItem)
 716    {
 717       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 718       if (Res >= 0)
 719          break;
 720    }
 721
 722    // Match
 723    if (Res == 0)
 724    {
 725       Bucket = I;
 726       return I->String;
 727    }
 728
 729    // Get a structure
 730    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 731    if (Item == 0)
 732       return 0;
 733
 734    // Fill in the structure
 735    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 736    ItemP->NextItem = I - Cache.StringItemP;
 737    *Last = Item;
 738    ItemP->String = Map.WriteString(S,Size);
 739    if (ItemP->String == 0)
 740       return 0;
 741
 742    Bucket = ItemP;
 743    return ItemP->String;
 744 }
 745                                                                         /*}}}*/
 746 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 747 // ---------------------------------------------------------------------
 748 /* This just verifies that each file in the list of index files exists,
 749    has matching attributes with the cache and the cache does not have
 750    any extra files. */
 751 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 752                           FileIterator End,MMap **OutMap = 0)
 753 {
 754    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 755    // No file, certainly invalid
 756    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 757    {
 758       if (Debug == true)
 759          std::clog << "CacheFile doesn't exist" << std::endl;
 760       return false;
 761    }
 762
 763    // Map it
 764    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 765    SPtr<MMap> Map = new MMap(CacheF,0);
 766    pkgCache Cache(Map);
 767    if (_error->PendingError() == true || Map->Size() == 0)
 768    {
 769       if (Debug == true)
 770          std::clog << "Errors are pending or Map is empty()" << std::endl;
 771       _error->Discard();
 772       return false;
 773    }
 774
 775    /* Now we check every index file, see if it is in the cache,
 776       verify the IMS data and check that it is on the disk too.. */
 777    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 778    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 779    for (; Start != End; Start++)
 780    {
 781       if (Debug == true)
 782          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 783       if ((*Start)->HasPackages() == false)
 784       {
 785          if (Debug == true)
 786             std::clog << "Has NO packages" << std::endl;
 787          continue;
 788       }
 789
 790       if ((*Start)->Exists() == false)
 791       {
 792 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 793          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 794                           (*Start)->Describe().c_str());
 795 #endif
 796          if (Debug == true)
 797             std::clog << "file doesn't exist" << std::endl;
 798          continue;
 799       }
 800
 801       // FindInCache is also expected to do an IMS check.
 802       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 803       if (File.end() == true)
 804       {
 805          if (Debug == true)
 806             std::clog << "FindInCache returned end-Pointer" << std::endl;
 807          return false;
 808       }
 809
 810       Visited[File->ID] = true;
 811       if (Debug == true)
 812          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 813    }
 814
 815    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 816       if (Visited[I] == false)
 817       {
 818          if (Debug == true)
 819             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 820          return false;
 821       }
 822
 823    if (_error->PendingError() == true)
 824    {
 825       if (Debug == true)
 826       {
 827          std::clog << "Validity failed because of pending errors:" << std::endl;
 828          _error->DumpErrors();
 829       }
 830       _error->Discard();
 831       return false;
 832    }
 833
 834    if (OutMap != 0)
 835       *OutMap = Map.UnGuard();
 836    return true;
 837 }
 838                                                                         /*}}}*/
 839 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 840 // ---------------------------------------------------------------------
 841 /* Size is kind of an abstract notion that is only used for the progress
 842    meter */
 843 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 844 {
 845    unsigned long TotalSize = 0;
 846    for (; Start != End; Start++)
 847    {
 848       if ((*Start)->HasPackages() == false)
 849          continue;
 850       TotalSize += (*Start)->Size();
 851    }
 852    return TotalSize;
 853 }
 854                                                                         /*}}}*/
 855 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 856 // ---------------------------------------------------------------------
 857 /* */
 858 static bool BuildCache(pkgCacheGenerator &Gen,
 859                        OpProgress &Progress,
 860                        unsigned long &CurrentSize,unsigned long TotalSize,
 861                        FileIterator Start, FileIterator End)
 862 {
 863    FileIterator I;
 864    for (I = Start; I != End; I++)
 865    {
 866       if ((*I)->HasPackages() == false)
 867          continue;
 868
 869       if ((*I)->Exists() == false)
 870          continue;
 871
 872       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 873       {
 874          _error->Warning("Duplicate sources.list entry %s",
 875                          (*I)->Describe().c_str());
 876          continue;
 877       }
 878
 879       unsigned long Size = (*I)->Size();
 880       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 881       CurrentSize += Size;
 882
 883       if ((*I)->Merge(Gen,Progress) == false)
 884          return false;
 885    }
 886
 887    if (Gen.HasFileDeps() == true)
 888    {
 889       Progress.Done();
 890       TotalSize = ComputeSize(Start, End);
 891       CurrentSize = 0;
 892       for (I = Start; I != End; I++)
 893       {
 894          unsigned long Size = (*I)->Size();
 895          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 896          CurrentSize += Size;
 897          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 898             return false;
 899       }
 900    }
 901
 902    return true;
 903 }
 904                                                                         /*}}}*/
 905 // MakeStatusCache - Construct the status cache                         /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* This makes sure that the status cache (the cache that has all
 908    index files from the sources list and all local ones) is ready
 909    to be mmaped. If OutMap is not zero then a MMap object representing
 910    the cache will be stored there. This is pretty much mandetory if you
 911    are using AllowMem. AllowMem lets the function be run as non-root
 912    where it builds the cache 'fast' into a memory buffer. */
 913 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 914                         MMap **OutMap,bool AllowMem)
 915 {
 916    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 917    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 918
 919    vector<pkgIndexFile *> Files;
 920    for (vector<metaIndex *>::const_iterator i = List.begin();
 921         i != List.end();
 922         i++)
 923    {
 924       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 925       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 926            j != Indexes->end();
 927            j++)
 928          Files.push_back (*j);
 929    }
 930
 931    unsigned long const EndOfSource = Files.size();
 932    if (_system->AddStatusFiles(Files) == false)
 933       return false;
 934
 935    // Decide if we can write to the files..
 936    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 937    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 938
 939    // Decide if we can write to the cache
 940    bool Writeable = false;
 941    if (CacheFile.empty() == false)
 942       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 943    else
 944       if (SrcCacheFile.empty() == false)
 945          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 946    if (Debug == true)
 947       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
 948
 949    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 950       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 951
 952    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 953
 954    // Cache is OK, Fin.
 955    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 956    {
 957       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 958       if (Debug == true)
 959          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
 960       return true;
 961    }
 962    else if (Debug == true)
 963          std::clog << "pkgcache.bin is NOT valid" << std::endl;
 964
 965    /* At this point we know we need to reconstruct the package cache,
 966       begin. */
 967    SPtr<FileFd> CacheF;
 968    SPtr<DynamicMMap> Map;
 969    if (Writeable == true && CacheFile.empty() == false)
 970    {
 971       unlink(CacheFile.c_str());
 972       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 973       fchmod(CacheF->Fd(),0644);
 974       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 975       if (_error->PendingError() == true)
 976          return false;
 977       if (Debug == true)
 978          std::clog << "Open filebased MMap" << std::endl;
 979    }
 980    else
 981    {
 982       // Just build it in memory..
 983       Map = new DynamicMMap(0,MapSize);
 984       if (Debug == true)
 985          std::clog << "Open memory Map (not filebased)" << std::endl;
 986    }
 987
 988    // Lets try the source cache.
 989    unsigned long CurrentSize = 0;
 990    unsigned long TotalSize = 0;
 991    if (CheckValidity(SrcCacheFile,Files.begin(),
 992                      Files.begin()+EndOfSource) == true)
 993    {
 994       if (Debug == true)
 995          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
 996       // Preload the map with the source cache
 997       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 998       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
 999       if ((alloc == 0 && _error->PendingError())
1000                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1001                                 SCacheF.Size()) == false)
1002          return false;
1003
1004       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1005
1006       // Build the status cache
1007       pkgCacheGenerator Gen(Map.Get(),&Progress);
1008       if (_error->PendingError() == true)
1009          return false;
1010       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1011                      Files.begin()+EndOfSource,Files.end()) == false)
1012          return false;
1013    }
1014    else
1015    {
1016       if (Debug == true)
1017          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1018       TotalSize = ComputeSize(Files.begin(),Files.end());
1019
1020       // Build the source cache
1021       pkgCacheGenerator Gen(Map.Get(),&Progress);
1022       if (_error->PendingError() == true)
1023          return false;
1024       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1025                      Files.begin(),Files.begin()+EndOfSource) == false)
1026          return false;
1027
1028       // Write it back
1029       if (Writeable == true && SrcCacheFile.empty() == false)
1030       {
1031          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
1032          if (_error->PendingError() == true)
1033             return false;
1034
1035          fchmod(SCacheF.Fd(),0644);
1036
1037          // Write out the main data
1038          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1039             return _error->Error(_("IO Error saving source cache"));
1040          SCacheF.Sync();
1041
1042          // Write out the proper header
1043          Gen.GetCache().HeaderP->Dirty = false;
1044          if (SCacheF.Seek(0) == false ||
1045              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1046             return _error->Error(_("IO Error saving source cache"));
1047          Gen.GetCache().HeaderP->Dirty = true;
1048          SCacheF.Sync();
1049       }
1050
1051       // Build the status cache
1052       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1053                      Files.begin()+EndOfSource,Files.end()) == false)
1054          return false;
1055    }
1056    if (Debug == true)
1057       std::clog << "Caches are ready for shipping" << std::endl;
1058
1059    if (_error->PendingError() == true)
1060       return false;
1061    if (OutMap != 0)
1062    {
1063       if (CacheF != 0)
1064       {
1065          delete Map.UnGuard();
1066          *OutMap = new MMap(*CacheF,0);
1067       }
1068       else
1069       {
1070          *OutMap = Map.UnGuard();
1071       }
1072    }
1073
1074    return true;
1075 }
1076                                                                         /*}}}*/
1077 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
1078 // ---------------------------------------------------------------------
1079 /* */
1080 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1081 {
1082    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1083    vector<pkgIndexFile *> Files;
1084    unsigned long EndOfSource = Files.size();
1085    if (_system->AddStatusFiles(Files) == false)
1086       return false;
1087
1088    SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
1089    unsigned long CurrentSize = 0;
1090    unsigned long TotalSize = 0;
1091
1092    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1093
1094    // Build the status cache
1095    Progress.OverallProgress(0,1,1,_("Reading package lists"));
1096    pkgCacheGenerator Gen(Map.Get(),&Progress);
1097    if (_error->PendingError() == true)
1098       return false;
1099    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1100                   Files.begin()+EndOfSource,Files.end()) == false)
1101       return false;
1102
1103    if (_error->PendingError() == true)
1104       return false;
1105    *OutMap = Map.UnGuard();
1106
1107    return true;
1108 }
1109                                                                         /*}}}*/