apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <vector>
  31
  32 #include <sys/stat.h>
  33 #include <unistd.h>
  34 #include <errno.h>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38 uint32_t hashlittle( const void *key, size_t length, uint32_t initval);
  39
  40 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* We set the diry flag and make sure that is written to the disk */
  43 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  44                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  45                     FoundFileDeps(0)
  46 {
  47    CurrentFile = 0;
  48    memset(UniqHash,0,sizeof(UniqHash));
  49
  50    if (_error->PendingError() == true)
  51       return;
  52
  53    if (Map.Size() == 0)
  54    {
  55       // Setup the map interface..
  56       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  57       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  58          return;
  59
  60       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  61
  62       // Starting header
  63       *Cache.HeaderP = pkgCache::Header();
  64       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  65       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  66       Cache.ReMap();
  67    }
  68    else
  69    {
  70       // Map directly from the existing file
  71       Cache.ReMap();
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73       if (Cache.VS != _system->VS)
  74       {
  75          _error->Error(_("Cache has an incompatible versioning system"));
  76          return;
  77       }
  78    }
  79
  80    Cache.HeaderP->Dirty = true;
  81    Map.Sync(0,sizeof(pkgCache::Header));
  82 }
  83                                                                         /*}}}*/
  84 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* We sync the data then unset the dirty flag in two steps so as to
  87    advoid a problem during a crash */
  88 pkgCacheGenerator::~pkgCacheGenerator()
  89 {
  90    if (_error->PendingError() == true)
  91       return;
  92    if (Map.Sync() == false)
  93       return;
  94
  95    Cache.HeaderP->Dirty = false;
  96    Map.Sync(0,sizeof(pkgCache::Header));
  97 }
  98                                                                         /*}}}*/
  99 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 100 // ---------------------------------------------------------------------
 101 /* This provides the generation of the entries in the cache. Each loop
 102    goes through a single package record from the underlying parse engine. */
 103 bool pkgCacheGenerator::MergeList(ListParser &List,
 104                                   pkgCache::VerIterator *OutVer)
 105 {
 106    List.Owner = this;
 107
 108    unsigned int Counter = 0;
 109    while (List.Step() == true)
 110    {
 111       // Get a pointer to the package structure
 112       string PackageName = List.Package();
 113       if (PackageName.empty() == true)
 114          return false;
 115
 116       pkgCache::PkgIterator Pkg;
 117       if (NewPackage(Pkg,PackageName) == false)
 118          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 119       Counter++;
 120       if (Counter % 100 == 0 && Progress != 0)
 121          Progress->Progress(List.Offset());
 122
 123       /* Get a pointer to the version structure. We know the list is sorted
 124          so we use that fact in the search. Insertion of new versions is
 125          done with correct sorting */
 126       string Version = List.Version();
 127       if (Version.empty() == true)
 128       {
 129          // we first process the package, then the descriptions
 130          // (this has the bonus that we get MMap error when we run out
 131          //  of MMap space)
 132          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 133             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 134                                  PackageName.c_str());
 135
 136          // Find the right version to write the description
 137          MD5SumValue CurMd5 = List.Description_md5();
 138          pkgCache::VerIterator Ver = Pkg.VersionList();
 139          map_ptrloc *LastVer = &Pkg->VersionList;
 140
 141          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 142          {
 143             pkgCache::DescIterator Desc = Ver.DescriptionList();
 144             map_ptrloc *LastDesc = &Ver->DescriptionList;
 145             bool duplicate=false;
 146
 147             // don't add a new description if we have one for the given
 148             // md5 && language
 149             for ( ; Desc.end() == false; Desc++)
 150                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 151                    Desc.LanguageCode() == List.DescriptionLanguage())
 152                   duplicate=true;
 153             if(duplicate)
 154                continue;
 155
 156             for (Desc = Ver.DescriptionList();
 157                  Desc.end() == false;
 158                  LastDesc = &Desc->NextDesc, Desc++)
 159             {
 160                if (MD5SumValue(Desc.md5()) == CurMd5)
 161                {
 162                   // Add new description
 163                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 164                   Desc->ParentPkg = Pkg.Index();
 165
 166                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 167                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 168                   break;
 169                }
 170             }
 171          }
 172
 173          continue;
 174       }
 175
 176       pkgCache::VerIterator Ver = Pkg.VersionList();
 177       map_ptrloc *LastVer = &Pkg->VersionList;
 178       int Res = 1;
 179       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 180       {
 181          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 182          if (Res >= 0)
 183             break;
 184       }
 185
 186       /* We already have a version for this item, record that we
 187          saw it */
 188       unsigned long Hash = List.VersionHash();
 189       if (Res == 0 && Ver->Hash == Hash)
 190       {
 191          if (List.UsePackage(Pkg,Ver) == false)
 192             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 193                                  PackageName.c_str());
 194
 195          if (NewFileVer(Ver,List) == false)
 196             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 197                                  PackageName.c_str());
 198
 199          // Read only a single record and return
 200          if (OutVer != 0)
 201          {
 202             *OutVer = Ver;
 203             FoundFileDeps |= List.HasFileDeps();
 204             return true;
 205          }
 206
 207          continue;
 208       }
 209
 210       // Skip to the end of the same version set.
 211       if (Res == 0)
 212       {
 213          for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 214          {
 215             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 216             if (Res != 0)
 217                break;
 218          }
 219       }
 220
 221       // Add a new version
 222       *LastVer = NewVersion(Ver,Version,*LastVer);
 223       Ver->ParentPkg = Pkg.Index();
 224       Ver->Hash = Hash;
 225
 226       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)
 227          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 228                               PackageName.c_str());
 229
 230       if (List.UsePackage(Pkg,Ver) == false)
 231          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 232                               PackageName.c_str());
 233
 234       if (NewFileVer(Ver,List) == false)
 235          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 236                               PackageName.c_str());
 237
 238       // Read only a single record and return
 239       if (OutVer != 0)
 240       {
 241          *OutVer = Ver;
 242          FoundFileDeps |= List.HasFileDeps();
 243          return true;
 244       }
 245
 246       /* Record the Description data. Description data always exist in
 247          Packages and Translation-* files. */
 248       pkgCache::DescIterator Desc = Ver.DescriptionList();
 249       map_ptrloc *LastDesc = &Ver->DescriptionList;
 250
 251       // Skip to the end of description set
 252       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 253
 254       // Add new description
 255       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 256       Desc->ParentPkg = Pkg.Index();
 257
 258       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 259          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 260    }
 261
 262    FoundFileDeps |= List.HasFileDeps();
 263
 264    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 265       return _error->Error(_("Wow, you exceeded the number of package "
 266                              "names this APT is capable of."));
 267    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 268       return _error->Error(_("Wow, you exceeded the number of versions "
 269                              "this APT is capable of."));
 270    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 271       return _error->Error(_("Wow, you exceeded the number of descriptions "
 272                              "this APT is capable of."));
 273    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 274       return _error->Error(_("Wow, you exceeded the number of dependencies "
 275                              "this APT is capable of."));
 276    return true;
 277 }
 278                                                                         /*}}}*/
 279 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 280 // ---------------------------------------------------------------------
 281 /* If we found any file depends while parsing the main list we need to
 282    resolve them. Since it is undesired to load the entire list of files
 283    into the cache as virtual packages we do a two stage effort. MergeList
 284    identifies the file depends and this creates Provdies for them by
 285    re-parsing all the indexs. */
 286 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 287 {
 288    List.Owner = this;
 289
 290    unsigned int Counter = 0;
 291    while (List.Step() == true)
 292    {
 293       string PackageName = List.Package();
 294       if (PackageName.empty() == true)
 295          return false;
 296       string Version = List.Version();
 297       if (Version.empty() == true)
 298          continue;
 299
 300       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 301       if (Pkg.end() == true)
 302          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 303                                 PackageName.c_str());
 304       Counter++;
 305       if (Counter % 100 == 0 && Progress != 0)
 306          Progress->Progress(List.Offset());
 307
 308       unsigned long Hash = List.VersionHash();
 309       pkgCache::VerIterator Ver = Pkg.VersionList();
 310       for (; Ver.end() == false; Ver++)
 311       {
 312          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 313          {
 314             if (List.CollectFileProvides(Cache,Ver) == false)
 315                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 316             break;
 317          }
 318       }
 319
 320       if (Ver.end() == true)
 321          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 322    }
 323
 324    return true;
 325 }
 326                                                                         /*}}}*/
 327 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 328 // ---------------------------------------------------------------------
 329 /* This creates a new package structure and adds it to the hash table */
 330 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 331 {
 332    Pkg = Cache.FindPkg(Name);
 333    if (Pkg.end() == false)
 334       return true;
 335
 336    // Get a structure
 337    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 338    if (Package == 0)
 339       return false;
 340
 341    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 342
 343    // Insert it into the hash table
 344    unsigned long Hash = Cache.Hash(Name);
 345    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 346    Cache.HeaderP->HashTable[Hash] = Package;
 347
 348    // Set the name and the ID
 349    Pkg->Name = Map.WriteString(Name);
 350    if (Pkg->Name == 0)
 351       return false;
 352    Pkg->ID = Cache.HeaderP->PackageCount++;
 353
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 358 // ---------------------------------------------------------------------
 359 /* */
 360 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 361                                    ListParser &List)
 362 {
 363    if (CurrentFile == 0)
 364       return true;
 365
 366    // Get a structure
 367    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 368    if (VerFile == 0)
 369       return 0;
 370
 371    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 372    VF->File = CurrentFile - Cache.PkgFileP;
 373
 374    // Link it to the end of the list
 375    map_ptrloc *Last = &Ver->FileList;
 376    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 377       Last = &V->NextFile;
 378    VF->NextFile = *Last;
 379    *Last = VF.Index();
 380
 381    VF->Offset = List.Offset();
 382    VF->Size = List.Size();
 383    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 384       Cache.HeaderP->MaxVerFileSize = VF->Size;
 385    Cache.HeaderP->VerFileCount++;
 386
 387    return true;
 388 }
 389                                                                         /*}}}*/
 390 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 391 // ---------------------------------------------------------------------
 392 /* This puts a version structure in the linked list */
 393 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 394                                             const string &VerStr,
 395                                             unsigned long Next)
 396 {
 397    // Get a structure
 398    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 399    if (Version == 0)
 400       return 0;
 401
 402    // Fill it in
 403    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 404    Ver->NextVer = Next;
 405    Ver->ID = Cache.HeaderP->VersionCount++;
 406    Ver->VerStr = Map.WriteString(VerStr);
 407    if (Ver->VerStr == 0)
 408       return 0;
 409
 410    return Version;
 411 }
 412                                                                         /*}}}*/
 413 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 414 // ---------------------------------------------------------------------
 415 /* */
 416 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 417                                    ListParser &List)
 418 {
 419    if (CurrentFile == 0)
 420       return true;
 421
 422    // Get a structure
 423    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 424    if (DescFile == 0)
 425       return false;
 426
 427    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 428    DF->File = CurrentFile - Cache.PkgFileP;
 429
 430    // Link it to the end of the list
 431    map_ptrloc *Last = &Desc->FileList;
 432    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 433       Last = &D->NextFile;
 434
 435    DF->NextFile = *Last;
 436    *Last = DF.Index();
 437
 438    DF->Offset = List.Offset();
 439    DF->Size = List.Size();
 440    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 441       Cache.HeaderP->MaxDescFileSize = DF->Size;
 442    Cache.HeaderP->DescFileCount++;
 443
 444    return true;
 445 }
 446                                                                         /*}}}*/
 447 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 448 // ---------------------------------------------------------------------
 449 /* This puts a description structure in the linked list */
 450 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 451                                             const string &Lang,
 452                                             const MD5SumValue &md5sum,
 453                                             map_ptrloc Next)
 454 {
 455    // Get a structure
 456    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 457    if (Description == 0)
 458       return 0;
 459
 460    // Fill it in
 461    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 462    Desc->NextDesc = Next;
 463    Desc->ID = Cache.HeaderP->DescriptionCount++;
 464    Desc->language_code = Map.WriteString(Lang);
 465    Desc->md5sum = Map.WriteString(md5sum.Value());
 466    if (Desc->language_code == 0 || Desc->md5sum == 0)
 467       return 0;
 468
 469    return Description;
 470 }
 471                                                                         /*}}}*/
 472 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* This creates a dependency element in the tree. It is linked to the
 475    version and to the package that it is pointing to. */
 476 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 477                                                const string &PackageName,
 478                                                const string &Version,
 479                                                unsigned int Op,
 480                                                unsigned int Type)
 481 {
 482    pkgCache &Cache = Owner->Cache;
 483
 484    // Get a structure
 485    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 486    if (Dependency == 0)
 487       return false;
 488
 489    // Fill it in
 490    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 491    Dep->ParentVer = Ver.Index();
 492    Dep->Type = Type;
 493    Dep->CompareOp = Op;
 494    Dep->ID = Cache.HeaderP->DependsCount++;
 495
 496    // Locate the target package
 497    pkgCache::PkgIterator Pkg;
 498    if (Owner->NewPackage(Pkg,PackageName) == false)
 499       return false;
 500
 501    // Probe the reverse dependency list for a version string that matches
 502    if (Version.empty() == false)
 503    {
 504 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 505          if (I->Version != 0 && I.TargetVer() == Version)
 506             Dep->Version = I->Version;*/
 507       if (Dep->Version == 0)
 508          if ((Dep->Version = WriteString(Version)) == 0)
 509             return false;
 510    }
 511
 512    // Link it to the package
 513    Dep->Package = Pkg.Index();
 514    Dep->NextRevDepends = Pkg->RevDepends;
 515    Pkg->RevDepends = Dep.Index();
 516
 517    /* Link it to the version (at the end of the list)
 518       Caching the old end point speeds up generation substantially */
 519    if (OldDepVer != Ver)
 520    {
 521       OldDepLast = &Ver->DependsList;
 522       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 523          OldDepLast = &D->NextDepends;
 524       OldDepVer = Ver;
 525    }
 526
 527    // Is it a file dependency?
 528    if (PackageName[0] == '/')
 529       FoundFileDeps = true;
 530
 531    Dep->NextDepends = *OldDepLast;
 532    *OldDepLast = Dep.Index();
 533    OldDepLast = &Dep->NextDepends;
 534
 535    return true;
 536 }
 537                                                                         /*}}}*/
 538 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 539 // ---------------------------------------------------------------------
 540 /* */
 541 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 542                                                 const string &PackageName,
 543                                                 const string &Version)
 544 {
 545    pkgCache &Cache = Owner->Cache;
 546
 547    // We do not add self referencing provides
 548    if (Ver.ParentPkg().Name() == PackageName)
 549       return true;
 550
 551    // Get a structure
 552    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 553    if (Provides == 0)
 554       return false;
 555    Cache.HeaderP->ProvidesCount++;
 556
 557    // Fill it in
 558    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 559    Prv->Version = Ver.Index();
 560    Prv->NextPkgProv = Ver->ProvidesList;
 561    Ver->ProvidesList = Prv.Index();
 562    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 563       return false;
 564
 565    // Locate the target package
 566    pkgCache::PkgIterator Pkg;
 567    if (Owner->NewPackage(Pkg,PackageName) == false)
 568       return false;
 569
 570    // Link it to the package
 571    Prv->ParentPkg = Pkg.Index();
 572    Prv->NextProvides = Pkg->ProvidesList;
 573    Pkg->ProvidesList = Prv.Index();
 574
 575    return true;
 576 }
 577                                                                         /*}}}*/
 578 // ListParser::NewTag - Create a Tag element                            /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 bool pkgCacheGenerator::ListParser::NewTag(pkgCache::PkgIterator Pkg,
 582                                            const char *NameStart,
 583                                            unsigned int NameSize)
 584 {
 585    pkgCache &Cache = Owner->Cache;
 586
 587    // Get a structure
 588    unsigned long Tagg = Owner->Map.Allocate(sizeof(pkgCache::Tag));
 589    if (Tagg == 0)
 590       return false;
 591    Cache.HeaderP->TagCount++;
 592
 593    // Fill it in
 594    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
 595    Tg->Name = WriteString(NameStart,NameSize);
 596    if (Tg->Name == 0)
 597       return false;
 598    Tg->NextTag = Pkg->TagList;
 599    Pkg->TagList = Tg.Index();
 600
 601    return true;
 602 }
 603                                                                         /*}}}*/
 604 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 605 // ---------------------------------------------------------------------
 606 /* This is used to select which file is to be associated with all newly
 607    added versions. The caller is responsible for setting the IMS fields. */
 608 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 609                                    const pkgIndexFile &Index,
 610                                    unsigned long Flags)
 611 {
 612    // Get some space for the structure
 613    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 614    if (CurrentFile == Cache.PkgFileP)
 615       return false;
 616
 617    // Fill it in
 618    CurrentFile->FileName = Map.WriteString(File);
 619    CurrentFile->Site = WriteUniqString(Site);
 620    CurrentFile->NextFile = Cache.HeaderP->FileList;
 621    CurrentFile->Flags = Flags;
 622    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 623    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 624    PkgFileName = File;
 625    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 626    Cache.HeaderP->PackageFileCount++;
 627
 628    if (CurrentFile->FileName == 0)
 629       return false;
 630
 631    if (Progress != 0)
 632       Progress->SubProgress(Index.Size());
 633    return true;
 634 }
 635                                                                         /*}}}*/
 636 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 637 // ---------------------------------------------------------------------
 638 /* This is used to create handles to strings. Given the same text it
 639    always returns the same number */
 640 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 641                                                  unsigned int Size)
 642 {
 643    uint32_t hash = hashlittle(S, Size, 0xdeadbeef);
 644
 645    /* We use a VERY LARGE INTRANSIENT hash table here, this speeds up generation
 646       by AN INSANE amount on ALL machines */
 647    pkgCache::StringItem **Bucket2;
 648    while (true) {
 649       Bucket2 = &UniqHash[hash % _count(UniqHash)];
 650       if (*Bucket2 == NULL)
 651          break;
 652       if (stringcmp(S,S+Size,Cache.StrP + (*Bucket2)->String) == 0)
 653          return (*Bucket2)->String;
 654       hash += 7;
 655    }
 656
 657    pkgCache::StringItem *&Bucket = *Bucket2;
 658    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 659    map_ptrloc *Last = &Cache.HeaderP->StringList;
 660
 661    // Get a structure
 662    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 663    if (Item == 0)
 664       return 0;
 665
 666    // Fill in the structure
 667    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 668    ItemP->NextItem = I - Cache.StringItemP;
 669    *Last = Item;
 670    ItemP->String = Map.WriteString(S,Size);
 671    if (ItemP->String == 0)
 672       return 0;
 673
 674    Bucket = ItemP;
 675    return ItemP->String;
 676 }
 677                                                                         /*}}}*/
 678 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 679 // ---------------------------------------------------------------------
 680 /* This just verifies that each file in the list of index files exists,
 681    has matching attributes with the cache and the cache does not have
 682    any extra files. */
 683 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 684                           FileIterator End,MMap **OutMap = 0)
 685 {
 686    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 687    // No file, certainly invalid
 688    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 689    {
 690       if (Debug == true)
 691          std::clog << "CacheFile doesn't exist" << std::endl;
 692       return false;
 693    }
 694
 695    // Map it
 696    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 697    SPtr<MMap> Map = new MMap(CacheF,0);
 698    pkgCache Cache(Map);
 699    if (_error->PendingError() == true || Map->Size() == 0)
 700    {
 701       if (Debug == true)
 702          std::clog << "Errors are pending or Map is empty()" << std::endl;
 703       _error->Discard();
 704       return false;
 705    }
 706
 707    /* Now we check every index file, see if it is in the cache,
 708       verify the IMS data and check that it is on the disk too.. */
 709    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 710    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 711    for (; Start != End; Start++)
 712    {
 713       if (Debug == true)
 714          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 715       if ((*Start)->HasPackages() == false)
 716       {
 717          if (Debug == true)
 718             std::clog << "Has NO packages" << std::endl;
 719          continue;
 720       }
 721
 722       if ((*Start)->Exists() == false)
 723       {
 724 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 725          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 726                           (*Start)->Describe().c_str());
 727 #endif
 728          if (Debug == true)
 729             std::clog << "file doesn't exist" << std::endl;
 730          continue;
 731       }
 732
 733       // FindInCache is also expected to do an IMS check.
 734       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 735       if (File.end() == true)
 736       {
 737          if (Debug == true)
 738             std::clog << "FindInCache returned end-Pointer" << std::endl;
 739          return false;
 740       }
 741
 742       Visited[File->ID] = true;
 743       if (Debug == true)
 744          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 745    }
 746
 747    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 748       if (Visited[I] == false)
 749       {
 750          if (Debug == true)
 751             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 752          return false;
 753       }
 754
 755    if (_error->PendingError() == true)
 756    {
 757       if (Debug == true)
 758       {
 759          std::clog << "Validity failed because of pending errors:" << std::endl;
 760          _error->DumpErrors();
 761       }
 762       _error->Discard();
 763       return false;
 764    }
 765
 766    if (OutMap != 0)
 767       *OutMap = Map.UnGuard();
 768    return true;
 769 }
 770                                                                         /*}}}*/
 771 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 772 // ---------------------------------------------------------------------
 773 /* Size is kind of an abstract notion that is only used for the progress
 774    meter */
 775 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 776 {
 777    unsigned long TotalSize = 0;
 778    for (; Start != End; Start++)
 779    {
 780       if ((*Start)->HasPackages() == false)
 781          continue;
 782       TotalSize += (*Start)->Size();
 783    }
 784    return TotalSize;
 785 }
 786                                                                         /*}}}*/
 787 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 788 // ---------------------------------------------------------------------
 789 /* */
 790 static bool BuildCache(pkgCacheGenerator &Gen,
 791                        OpProgress &Progress,
 792                        unsigned long &CurrentSize,unsigned long TotalSize,
 793                        FileIterator Start, FileIterator End)
 794 {
 795    FileIterator I;
 796    for (I = Start; I != End; I++)
 797    {
 798       if ((*I)->HasPackages() == false)
 799          continue;
 800
 801       if ((*I)->Exists() == false)
 802          continue;
 803
 804       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 805       {
 806          _error->Warning("Duplicate sources.list entry %s",
 807                          (*I)->Describe().c_str());
 808          continue;
 809       }
 810
 811       unsigned long Size = (*I)->Size();
 812       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 813       CurrentSize += Size;
 814
 815       if ((*I)->Merge(Gen,Progress) == false)
 816          return false;
 817    }
 818
 819    if (Gen.HasFileDeps() == true)
 820    {
 821       Progress.Done();
 822       TotalSize = ComputeSize(Start, End);
 823       CurrentSize = 0;
 824       for (I = Start; I != End; I++)
 825       {
 826          unsigned long Size = (*I)->Size();
 827          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 828          CurrentSize += Size;
 829          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 830             return false;
 831       }
 832    }
 833
 834    return true;
 835 }
 836                                                                         /*}}}*/
 837 // MakeStatusCache - Construct the status cache                         /*{{{*/
 838 // ---------------------------------------------------------------------
 839 /* This makes sure that the status cache (the cache that has all
 840    index files from the sources list and all local ones) is ready
 841    to be mmaped. If OutMap is not zero then a MMap object representing
 842    the cache will be stored there. This is pretty much mandetory if you
 843    are using AllowMem. AllowMem lets the function be run as non-root
 844    where it builds the cache 'fast' into a memory buffer. */
 845 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 846                         MMap **OutMap,bool AllowMem)
 847 {
 848    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 849    unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 850
 851    vector<pkgIndexFile *> Files;
 852    for (vector<metaIndex *>::const_iterator i = List.begin();
 853         i != List.end();
 854         i++)
 855    {
 856       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 857       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 858            j != Indexes->end();
 859            j++)
 860          Files.push_back (*j);
 861    }
 862
 863    unsigned long const EndOfSource = Files.size();
 864    if (_system->AddStatusFiles(Files) == false)
 865       return false;
 866
 867    // Decide if we can write to the files..
 868    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 869    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 870
 871    // Decide if we can write to the cache
 872    bool Writeable = false;
 873    if (CacheFile.empty() == false)
 874       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 875    else
 876       if (SrcCacheFile.empty() == false)
 877          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 878    if (Debug == true)
 879       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
 880
 881    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 882       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 883
 884    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 885
 886    // Cache is OK, Fin.
 887    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 888    {
 889       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 890       if (Debug == true)
 891          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
 892       return true;
 893    }
 894    else if (Debug == true)
 895          std::clog << "pkgcache.bin is NOT valid" << std::endl;
 896
 897    /* At this point we know we need to reconstruct the package cache,
 898       begin. */
 899    SPtr<FileFd> CacheF;
 900    SPtr<DynamicMMap> Map;
 901    if (Writeable == true && CacheFile.empty() == false)
 902    {
 903       unlink(CacheFile.c_str());
 904       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 905       fchmod(CacheF->Fd(),0644);
 906       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 907       if (_error->PendingError() == true)
 908          return false;
 909       if (Debug == true)
 910          std::clog << "Open filebased MMap" << std::endl;
 911    }
 912    else
 913    {
 914       // Just build it in memory..
 915       Map = new DynamicMMap(0,MapSize);
 916       if (Debug == true)
 917          std::clog << "Open memory Map (not filebased)" << std::endl;
 918    }
 919
 920    // Lets try the source cache.
 921    unsigned long CurrentSize = 0;
 922    unsigned long TotalSize = 0;
 923    if (CheckValidity(SrcCacheFile,Files.begin(),
 924                      Files.begin()+EndOfSource) == true)
 925    {
 926       if (Debug == true)
 927          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
 928       // Preload the map with the source cache
 929       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 930       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
 931       if ((alloc == 0 && _error->PendingError())
 932                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
 933                                 SCacheF.Size()) == false)
 934          return false;
 935
 936       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 937
 938       // Build the status cache
 939       pkgCacheGenerator Gen(Map.Get(),&Progress);
 940       if (_error->PendingError() == true)
 941          return false;
 942       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 943                      Files.begin()+EndOfSource,Files.end()) == false)
 944          return false;
 945    }
 946    else
 947    {
 948       if (Debug == true)
 949          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
 950       TotalSize = ComputeSize(Files.begin(),Files.end());
 951
 952       // Build the source cache
 953       pkgCacheGenerator Gen(Map.Get(),&Progress);
 954       if (_error->PendingError() == true)
 955          return false;
 956       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 957                      Files.begin(),Files.begin()+EndOfSource) == false)
 958          return false;
 959
 960       // Write it back
 961       if (Writeable == true && SrcCacheFile.empty() == false)
 962       {
 963          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 964          if (_error->PendingError() == true)
 965             return false;
 966
 967          fchmod(SCacheF.Fd(),0644);
 968
 969          // Write out the main data
 970          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 971             return _error->Error(_("IO Error saving source cache"));
 972          SCacheF.Sync();
 973
 974          // Write out the proper header
 975          Gen.GetCache().HeaderP->Dirty = false;
 976          if (SCacheF.Seek(0) == false ||
 977              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 978             return _error->Error(_("IO Error saving source cache"));
 979          Gen.GetCache().HeaderP->Dirty = true;
 980          SCacheF.Sync();
 981       }
 982
 983       // Build the status cache
 984       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 985                      Files.begin()+EndOfSource,Files.end()) == false)
 986          return false;
 987    }
 988    if (Debug == true)
 989       std::clog << "Caches are ready for shipping" << std::endl;
 990
 991    if (_error->PendingError() == true)
 992       return false;
 993    if (OutMap != 0)
 994    {
 995       if (CacheF != 0)
 996       {
 997          delete Map.UnGuard();
 998          *OutMap = new MMap(*CacheF,0);
 999       }
1000       else
1001       {
1002          *OutMap = Map.UnGuard();
1003       }
1004    }
1005
1006    return true;
1007 }
1008                                                                         /*}}}*/
1009 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
1010 // ---------------------------------------------------------------------
1011 /* */
1012 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1013 {
1014    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
1015    vector<pkgIndexFile *> Files;
1016    unsigned long EndOfSource = Files.size();
1017    if (_system->AddStatusFiles(Files) == false)
1018       return false;
1019
1020    SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);
1021    unsigned long CurrentSize = 0;
1022    unsigned long TotalSize = 0;
1023
1024    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1025
1026    // Build the status cache
1027    Progress.OverallProgress(0,1,1,_("Reading package lists"));
1028    pkgCacheGenerator Gen(Map.Get(),&Progress);
1029    if (_error->PendingError() == true)
1030       return false;
1031    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1032                   Files.begin()+EndOfSource,Files.end()) == false)
1033       return false;
1034
1035    if (_error->PendingError() == true)
1036       return false;
1037    *OutMap = Map.UnGuard();
1038
1039    return true;
1040 }
1041                                                                         /*}}}*/