apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/metaindex.h>
  28 #include <apt-pkg/fileutl.h>
  29
  30 #include <vector>
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35
  36 #include <apti18n.h>
  37                                                                         /*}}}*/
  38 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  42                             MD5SumValue const &CurMd5, std::string const &CurLang);
  43
  44 using std::string;
  45
  46 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  47 // ---------------------------------------------------------------------
  48 /* We set the dirty flag and make sure that is written to the disk */
  49 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  50                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  51                     FoundFileDeps(0)
  52 {
  53    CurrentFile = 0;
  54    memset(UniqHash,0,sizeof(UniqHash));
  55
  56    if (_error->PendingError() == true)
  57       return;
  58
  59    if (Map.Size() == 0)
  60    {
  61       // Setup the map interface..
  62       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  63       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  64          return;
  65
  66       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  67
  68       // Starting header
  69       *Cache.HeaderP = pkgCache::Header();
  70       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  71       Cache.HeaderP->VerSysName = idxVerSysName;
  72       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  73       Cache.HeaderP->Architecture = idxArchitecture;
  74       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  75          return;
  76       Cache.ReMap();
  77    }
  78    else
  79    {
  80       // Map directly from the existing file
  81       Cache.ReMap();
  82       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  83       if (Cache.VS != _system->VS)
  84       {
  85          _error->Error(_("Cache has an incompatible versioning system"));
  86          return;
  87       }
  88    }
  89
  90    Cache.HeaderP->Dirty = true;
  91    Map.Sync(0,sizeof(pkgCache::Header));
  92 }
  93                                                                         /*}}}*/
  94 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* We sync the data then unset the dirty flag in two steps so as to
  97    advoid a problem during a crash */
  98 pkgCacheGenerator::~pkgCacheGenerator()
  99 {
 100    if (_error->PendingError() == true)
 101       return;
 102    if (Map.Sync() == false)
 103       return;
 104
 105    Cache.HeaderP->Dirty = false;
 106    Cache.HeaderP->CacheFileSize = Map.Size();
 107    Map.Sync(0,sizeof(pkgCache::Header));
 108 }
 109                                                                         /*}}}*/
 110 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 111    if (oldMap == newMap)
 112       return;
 113
 114    if (_config->FindB("Debug::pkgCacheGen", false))
 115       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 116
 117    Cache.ReMap(false);
 118
 119    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 120
 121    for (size_t i = 0; i < _count(UniqHash); ++i)
 122       if (UniqHash[i] != 0)
 123          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 124
 125    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 126         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 127       (*i)->ReMap(oldMap, newMap);
 128    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 129         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 130       (*i)->ReMap(oldMap, newMap);
 131    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 132         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 133       (*i)->ReMap(oldMap, newMap);
 134    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 135         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 136       (*i)->ReMap(oldMap, newMap);
 137    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 138         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 139       (*i)->ReMap(oldMap, newMap);
 140    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 141         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 142       (*i)->ReMap(oldMap, newMap);
 143    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 144         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 145       (*i)->ReMap(oldMap, newMap);
 146 }                                                                       /*}}}*/
 147 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 148 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 149                                         const unsigned long &Len) {
 150    void const * const oldMap = Map.Data();
 151    map_ptrloc const index = Map.WriteString(String, Len);
 152    if (index != 0)
 153       ReMap(oldMap, Map.Data());
 154    return index;
 155 }
 156                                                                         /*}}}*/
 157 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 158 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 159    void const * const oldMap = Map.Data();
 160    map_ptrloc const index = Map.WriteString(String);
 161    if (index != 0)
 162       ReMap(oldMap, Map.Data());
 163    return index;
 164 }
 165                                                                         /*}}}*/
 166 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 167    void const * const oldMap = Map.Data();
 168    map_ptrloc const index = Map.Allocate(size);
 169    if (index != 0)
 170       ReMap(oldMap, Map.Data());
 171    return index;
 172 }
 173                                                                         /*}}}*/
 174 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 175 // ---------------------------------------------------------------------
 176 /* This provides the generation of the entries in the cache. Each loop
 177    goes through a single package record from the underlying parse engine. */
 178 bool pkgCacheGenerator::MergeList(ListParser &List,
 179                                   pkgCache::VerIterator *OutVer)
 180 {
 181    List.Owner = this;
 182
 183    unsigned int Counter = 0;
 184    while (List.Step() == true)
 185    {
 186       string const PackageName = List.Package();
 187       if (PackageName.empty() == true)
 188          return false;
 189
 190       Counter++;
 191       if (Counter % 100 == 0 && Progress != 0)
 192          Progress->Progress(List.Offset());
 193
 194       string Arch = List.Architecture();
 195       string const Version = List.Version();
 196       if (Version.empty() == true && Arch.empty() == true)
 197       {
 198          if (MergeListGroup(List, PackageName) == false)
 199             return false;
 200       }
 201
 202       if (Arch.empty() == true)
 203          Arch = _config->Find("APT::Architecture");
 204
 205       // Get a pointer to the package structure
 206       pkgCache::PkgIterator Pkg;
 207       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 208       if (NewPackage(Pkg, PackageName, Arch) == false)
 209          // TRANSLATOR: The first placeholder is a package name,
 210          // the other two should be copied verbatim as they include debug info
 211          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 212                               PackageName.c_str(), "NewPackage", 1);
 213
 214
 215       if (Version.empty() == true)
 216       {
 217          if (MergeListPackage(List, Pkg) == false)
 218             return false;
 219       }
 220       else
 221       {
 222          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 223             return false;
 224       }
 225
 226       if (OutVer != 0)
 227       {
 228          FoundFileDeps |= List.HasFileDeps();
 229          return true;
 230       }
 231    }
 232
 233    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 234       return _error->Error(_("Wow, you exceeded the number of package "
 235                              "names this APT is capable of."));
 236    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 237       return _error->Error(_("Wow, you exceeded the number of versions "
 238                              "this APT is capable of."));
 239    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 240       return _error->Error(_("Wow, you exceeded the number of descriptions "
 241                              "this APT is capable of."));
 242    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 243       return _error->Error(_("Wow, you exceeded the number of dependencies "
 244                              "this APT is capable of."));
 245
 246    FoundFileDeps |= List.HasFileDeps();
 247    return true;
 248 }
 249 // CacheGenerator::MergeListGroup                                       /*{{{*/
 250 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 251 {
 252    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 253    // a group has no data on it's own, only packages have it but these
 254    // stanzas like this come from Translation- files to add descriptions,
 255    // but without a version we don't need a description for it…
 256    if (Grp.end() == true)
 257       return true;
 258    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 259
 260    pkgCache::PkgIterator Pkg;
 261    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 262    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 263       if (MergeListPackage(List, Pkg) == false)
 264          return false;
 265
 266    return true;
 267 }
 268                                                                         /*}}}*/
 269 // CacheGenerator::MergeListPackage                                     /*{{{*/
 270 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 271 {
 272    // we first process the package, then the descriptions
 273    // (for deb this package processing is in fact a no-op)
 274    pkgCache::VerIterator Ver(Cache);
 275    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 276    if (List.UsePackage(Pkg, Ver) == false)
 277       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 278                            Pkg.Name(), "UsePackage", 1);
 279
 280    // Find the right version to write the description
 281    MD5SumValue CurMd5 = List.Description_md5();
 282    std::string CurLang = List.DescriptionLanguage();
 283
 284    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 285    {
 286       pkgCache::DescIterator Desc = Ver.DescriptionList();
 287
 288       // a version can only have one md5 describing it
 289       if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
 290          continue;
 291
 292       // don't add a new description if we have one for the given
 293       // md5 && language
 294       if (IsDuplicateDescription(Desc, CurMd5, CurLang) == true)
 295          continue;
 296
 297       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 298       // we add at the end, so that the start is constant as we need
 299       // that to be able to efficiently share these lists
 300       map_ptrloc *LastDesc = &Ver->DescriptionList;
 301       for (;Desc.end() == false && Desc->NextDesc != 0; ++Desc);
 302       if (Desc.end() == false)
 303          LastDesc = &Desc->NextDesc;
 304
 305       void const * const oldMap = Map.Data();
 306       map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, *LastDesc);
 307       if (unlikely(descindex == 0 && _error->PendingError()))
 308          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 309                               Pkg.Name(), "NewDescription", 1);
 310       if (oldMap != Map.Data())
 311          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 312       *LastDesc = descindex;
 313       Desc->ParentPkg = Pkg.Index();
 314
 315       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 316          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 317                               Pkg.Name(), "NewFileDesc", 1);
 318
 319       // we can stop here as all "same" versions will share the description
 320       break;
 321    }
 322
 323    return true;
 324 }
 325                                                                         /*}}}*/
 326 // CacheGenerator::MergeListVersion                                     /*{{{*/
 327 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 328                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 329 {
 330    pkgCache::VerIterator Ver = Pkg.VersionList();
 331    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 332    map_ptrloc *LastVer = &Pkg->VersionList;
 333    void const * oldMap = Map.Data();
 334
 335    unsigned long const Hash = List.VersionHash();
 336    if (Ver.end() == false)
 337    {
 338       /* We know the list is sorted so we use that fact in the search.
 339          Insertion of new versions is done with correct sorting */
 340       int Res = 1;
 341       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 342       {
 343          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 344          // Version is higher as current version - insert here
 345          if (Res > 0)
 346             break;
 347          // Versionstrings are equal - is hash also equal?
 348          if (Res == 0 && Ver->Hash == Hash)
 349             break;
 350          // proceed with the next till we have either the right
 351          // or we found another version (which will be lower)
 352       }
 353
 354       /* We already have a version for this item, record that we saw it */
 355       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 356       {
 357          if (List.UsePackage(Pkg,Ver) == false)
 358             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 359                                  Pkg.Name(), "UsePackage", 2);
 360
 361          if (NewFileVer(Ver,List) == false)
 362             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 363                                  Pkg.Name(), "NewFileVer", 1);
 364
 365          // Read only a single record and return
 366          if (OutVer != 0)
 367          {
 368             *OutVer = Ver;
 369             return true;
 370          }
 371
 372          return true;
 373       }
 374    }
 375
 376    // Add a new version
 377    map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 378    if (verindex == 0 && _error->PendingError())
 379       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 380                            Pkg.Name(), "NewVersion", 1);
 381
 382    if (oldMap != Map.Data())
 383          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 384    *LastVer = verindex;
 385    Ver->ParentPkg = Pkg.Index();
 386    Ver->Hash = Hash;
 387
 388    if (unlikely(List.NewVersion(Ver) == false))
 389       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 390                            Pkg.Name(), "NewVersion", 2);
 391
 392    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 393       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 394                            Pkg.Name(), "UsePackage", 3);
 395
 396    if (unlikely(NewFileVer(Ver,List) == false))
 397       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 398                            Pkg.Name(), "NewFileVer", 2);
 399
 400    pkgCache::GrpIterator Grp = Pkg.Group();
 401    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 402
 403    /* If it is the first version of this package we need to add implicit
 404       Multi-Arch dependencies to all other package versions in the group now -
 405       otherwise we just add them for this new version */
 406    if (Pkg.VersionList()->NextVer == 0)
 407    {
 408       pkgCache::PkgIterator P = Grp.PackageList();
 409       Dynamic<pkgCache::PkgIterator> DynP(P);
 410       for (; P.end() != true; P = Grp.NextPkg(P))
 411       {
 412          if (P->ID == Pkg->ID)
 413             continue;
 414          pkgCache::VerIterator V = P.VersionList();
 415          Dynamic<pkgCache::VerIterator> DynV(V);
 416          for (; V.end() != true; ++V)
 417             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 418                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                                     Pkg.Name(), "AddImplicitDepends", 1);
 420       }
 421    }
 422    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 423       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 424                            Pkg.Name(), "AddImplicitDepends", 2);
 425
 426    // Read only a single record and return
 427    if (OutVer != 0)
 428    {
 429       *OutVer = Ver;
 430       return true;
 431    }
 432
 433    /* Record the Description (it is not translated) */
 434    MD5SumValue CurMd5 = List.Description_md5();
 435    if (CurMd5.Value().empty() == true)
 436       return true;
 437    std::string CurLang = List.DescriptionLanguage();
 438
 439    /* Before we add a new description we first search in the group for
 440       a version with a description of the same MD5 - if so we reuse this
 441       description group instead of creating our own for this version */
 442    for (pkgCache::PkgIterator P = Grp.PackageList();
 443         P.end() == false; P = Grp.NextPkg(P))
 444    {
 445       for (pkgCache::VerIterator V = P.VersionList();
 446            V.end() == false; ++V)
 447       {
 448          if (IsDuplicateDescription(V.DescriptionList(), CurMd5, "") == false)
 449             continue;
 450          Ver->DescriptionList = V->DescriptionList;
 451          return true;
 452       }
 453    }
 454
 455    // We haven't found reusable descriptions, so add the first description
 456    pkgCache::DescIterator Desc = Ver.DescriptionList();
 457    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 458    map_ptrloc *LastDesc = &Ver->DescriptionList;
 459
 460    oldMap = Map.Data();
 461    map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, *LastDesc);
 462    if (unlikely(descindex == 0 && _error->PendingError()))
 463       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 464                            Pkg.Name(), "NewDescription", 2);
 465    if (oldMap != Map.Data())
 466        LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 467    *LastDesc = descindex;
 468    Desc->ParentPkg = Pkg.Index();
 469
 470    if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 471       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 472                            Pkg.Name(), "NewFileDesc", 2);
 473
 474    return true;
 475 }
 476                                                                         /*}}}*/
 477                                                                         /*}}}*/
 478 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 479 // ---------------------------------------------------------------------
 480 /* If we found any file depends while parsing the main list we need to
 481    resolve them. Since it is undesired to load the entire list of files
 482    into the cache as virtual packages we do a two stage effort. MergeList
 483    identifies the file depends and this creates Provdies for them by
 484    re-parsing all the indexs. */
 485 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 486 {
 487    List.Owner = this;
 488
 489    unsigned int Counter = 0;
 490    while (List.Step() == true)
 491    {
 492       string PackageName = List.Package();
 493       if (PackageName.empty() == true)
 494          return false;
 495       string Version = List.Version();
 496       if (Version.empty() == true)
 497          continue;
 498
 499       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 500       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 501       if (Pkg.end() == true)
 502          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 503                                 PackageName.c_str(), "FindPkg", 1);
 504       Counter++;
 505       if (Counter % 100 == 0 && Progress != 0)
 506          Progress->Progress(List.Offset());
 507
 508       unsigned long Hash = List.VersionHash();
 509       pkgCache::VerIterator Ver = Pkg.VersionList();
 510       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 511       for (; Ver.end() == false; ++Ver)
 512       {
 513          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 514          {
 515             if (List.CollectFileProvides(Cache,Ver) == false)
 516                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 517                                     PackageName.c_str(), "CollectFileProvides", 1);
 518             break;
 519          }
 520       }
 521
 522       if (Ver.end() == true)
 523          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 524    }
 525
 526    return true;
 527 }
 528                                                                         /*}}}*/
 529 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* This creates a new group structure and adds it to the hash table */
 532 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 533 {
 534    Grp = Cache.FindGrp(Name);
 535    if (Grp.end() == false)
 536       return true;
 537
 538    // Get a structure
 539    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 540    if (unlikely(Group == 0))
 541       return false;
 542
 543    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 544    map_ptrloc const idxName = WriteStringInMap(Name);
 545    if (unlikely(idxName == 0))
 546       return false;
 547    Grp->Name = idxName;
 548
 549    // Insert it into the hash table
 550    unsigned long const Hash = Cache.Hash(Name);
 551    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 552    Cache.HeaderP->GrpHashTable[Hash] = Group;
 553
 554    Grp->ID = Cache.HeaderP->GroupCount++;
 555    return true;
 556 }
 557                                                                         /*}}}*/
 558 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 559 // ---------------------------------------------------------------------
 560 /* This creates a new package structure and adds it to the hash table */
 561 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 562                                         const string &Arch) {
 563    pkgCache::GrpIterator Grp;
 564    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 565    if (unlikely(NewGroup(Grp, Name) == false))
 566       return false;
 567
 568    Pkg = Grp.FindPkg(Arch);
 569       if (Pkg.end() == false)
 570          return true;
 571
 572    // Get a structure
 573    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 574    if (unlikely(Package == 0))
 575       return false;
 576    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 577
 578    // Insert the package into our package list
 579    if (Grp->FirstPackage == 0) // the group is new
 580    {
 581       // Insert it into the hash table
 582       unsigned long const Hash = Cache.Hash(Name);
 583       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 584       Cache.HeaderP->PkgHashTable[Hash] = Package;
 585       Grp->FirstPackage = Package;
 586    }
 587    else // Group the Packages together
 588    {
 589       // this package is the new last package
 590       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 591       Pkg->NextPackage = LastPkg->NextPackage;
 592       LastPkg->NextPackage = Package;
 593    }
 594    Grp->LastPackage = Package;
 595
 596    // Set the name, arch and the ID
 597    Pkg->Name = Grp->Name;
 598    Pkg->Group = Grp.Index();
 599    // all is mapped to the native architecture
 600    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 601    if (unlikely(idxArch == 0))
 602       return false;
 603    Pkg->Arch = idxArch;
 604    Pkg->ID = Cache.HeaderP->PackageCount++;
 605
 606    return true;
 607 }
 608                                                                         /*}}}*/
 609 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 610 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 611                                            pkgCache::PkgIterator &P,
 612                                            pkgCache::VerIterator &V)
 613 {
 614    // copy P.Arch() into a string here as a cache remap
 615    // in NewDepends() later may alter the pointer location
 616    string Arch = P.Arch() == NULL ? "" : P.Arch();
 617    map_ptrloc *OldDepLast = NULL;
 618    /* MultiArch handling introduces a lot of implicit Dependencies:
 619       - MultiArch: same → Co-Installable if they have the same version
 620       - All others conflict with all other group members */
 621    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 622    pkgCache::PkgIterator D = G.PackageList();
 623    Dynamic<pkgCache::PkgIterator> DynD(D);
 624    for (; D.end() != true; D = G.NextPkg(D))
 625    {
 626       if (Arch == D.Arch() || D->VersionList == 0)
 627          continue;
 628       /* We allow only one installed arch at the time
 629          per group, therefore each group member conflicts
 630          with all other group members */
 631       if (coInstall == true)
 632       {
 633          // Replaces: ${self}:other ( << ${binary:Version})
 634          NewDepends(D, V, V.VerStr(),
 635                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 636                     OldDepLast);
 637          // Breaks: ${self}:other (!= ${binary:Version})
 638          NewDepends(D, V, V.VerStr(),
 639                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 640                     OldDepLast);
 641       } else {
 642          // Conflicts: ${self}:other
 643          NewDepends(D, V, "",
 644                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 645                     OldDepLast);
 646       }
 647    }
 648    return true;
 649 }
 650 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 651                                            pkgCache::PkgIterator &D)
 652 {
 653    /* MultiArch handling introduces a lot of implicit Dependencies:
 654       - MultiArch: same → Co-Installable if they have the same version
 655       - All others conflict with all other group members */
 656    map_ptrloc *OldDepLast = NULL;
 657    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 658    if (coInstall == true)
 659    {
 660       // Replaces: ${self}:other ( << ${binary:Version})
 661       NewDepends(D, V, V.VerStr(),
 662                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 663                  OldDepLast);
 664       // Breaks: ${self}:other (!= ${binary:Version})
 665       NewDepends(D, V, V.VerStr(),
 666                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 667                  OldDepLast);
 668    } else {
 669       // Conflicts: ${self}:other
 670       NewDepends(D, V, "",
 671                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 672                  OldDepLast);
 673    }
 674    return true;
 675 }
 676
 677                                                                         /*}}}*/
 678 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 679 // ---------------------------------------------------------------------
 680 /* */
 681 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 682                                    ListParser &List)
 683 {
 684    if (CurrentFile == 0)
 685       return true;
 686
 687    // Get a structure
 688    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 689    if (VerFile == 0)
 690       return 0;
 691
 692    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 693    VF->File = CurrentFile - Cache.PkgFileP;
 694
 695    // Link it to the end of the list
 696    map_ptrloc *Last = &Ver->FileList;
 697    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 698       Last = &V->NextFile;
 699    VF->NextFile = *Last;
 700    *Last = VF.Index();
 701
 702    VF->Offset = List.Offset();
 703    VF->Size = List.Size();
 704    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 705       Cache.HeaderP->MaxVerFileSize = VF->Size;
 706    Cache.HeaderP->VerFileCount++;
 707
 708    return true;
 709 }
 710                                                                         /*}}}*/
 711 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 712 // ---------------------------------------------------------------------
 713 /* This puts a version structure in the linked list */
 714 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 715                                             const string &VerStr,
 716                                             unsigned long Next)
 717 {
 718    // Get a structure
 719    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 720    if (Version == 0)
 721       return 0;
 722
 723    // Fill it in
 724    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 725    Ver->NextVer = Next;
 726    Ver->ID = Cache.HeaderP->VersionCount++;
 727    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 728    if (unlikely(idxVerStr == 0))
 729       return 0;
 730    Ver->VerStr = idxVerStr;
 731
 732    return Version;
 733 }
 734                                                                         /*}}}*/
 735 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 736 // ---------------------------------------------------------------------
 737 /* */
 738 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 739                                    ListParser &List)
 740 {
 741    if (CurrentFile == 0)
 742       return true;
 743
 744    // Get a structure
 745    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 746    if (DescFile == 0)
 747       return false;
 748
 749    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 750    DF->File = CurrentFile - Cache.PkgFileP;
 751
 752    // Link it to the end of the list
 753    map_ptrloc *Last = &Desc->FileList;
 754    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 755       Last = &D->NextFile;
 756
 757    DF->NextFile = *Last;
 758    *Last = DF.Index();
 759
 760    DF->Offset = List.Offset();
 761    DF->Size = List.Size();
 762    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 763       Cache.HeaderP->MaxDescFileSize = DF->Size;
 764    Cache.HeaderP->DescFileCount++;
 765
 766    return true;
 767 }
 768                                                                         /*}}}*/
 769 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 770 // ---------------------------------------------------------------------
 771 /* This puts a description structure in the linked list */
 772 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 773                                             const string &Lang,
 774                                             const MD5SumValue &md5sum,
 775                                             map_ptrloc Next)
 776 {
 777    // Get a structure
 778    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 779    if (Description == 0)
 780       return 0;
 781
 782    // Fill it in
 783    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 784    Desc->NextDesc = Next;
 785    Desc->ID = Cache.HeaderP->DescriptionCount++;
 786    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 787    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 788    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 789       return 0;
 790    Desc->language_code = idxlanguage_code;
 791    Desc->md5sum = idxmd5sum;
 792
 793    return Description;
 794 }
 795                                                                         /*}}}*/
 796 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 797 // ---------------------------------------------------------------------
 798 /* This creates a dependency element in the tree. It is linked to the
 799    version and to the package that it is pointing to. */
 800 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 801                                    pkgCache::VerIterator &Ver,
 802                                    string const &Version,
 803                                    unsigned int const &Op,
 804                                    unsigned int const &Type,
 805                                    map_ptrloc* &OldDepLast)
 806 {
 807    void const * const oldMap = Map.Data();
 808    // Get a structure
 809    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 810    if (unlikely(Dependency == 0))
 811       return false;
 812
 813    // Fill it in
 814    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 815    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 816    Dep->ParentVer = Ver.Index();
 817    Dep->Type = Type;
 818    Dep->CompareOp = Op;
 819    Dep->ID = Cache.HeaderP->DependsCount++;
 820
 821    // Probe the reverse dependency list for a version string that matches
 822    if (Version.empty() == false)
 823    {
 824 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 825          if (I->Version != 0 && I.TargetVer() == Version)
 826             Dep->Version = I->Version;*/
 827       if (Dep->Version == 0) {
 828          map_ptrloc const index = WriteStringInMap(Version);
 829          if (unlikely(index == 0))
 830             return false;
 831          Dep->Version = index;
 832       }
 833    }
 834
 835    // Link it to the package
 836    Dep->Package = Pkg.Index();
 837    Dep->NextRevDepends = Pkg->RevDepends;
 838    Pkg->RevDepends = Dep.Index();
 839
 840    // Do we know where to link the Dependency to?
 841    if (OldDepLast == NULL)
 842    {
 843       OldDepLast = &Ver->DependsList;
 844       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 845          OldDepLast = &D->NextDepends;
 846    } else if (oldMap != Map.Data())
 847       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 848
 849    Dep->NextDepends = *OldDepLast;
 850    *OldDepLast = Dep.Index();
 851    OldDepLast = &Dep->NextDepends;
 852
 853    return true;
 854 }
 855                                                                         /*}}}*/
 856 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 857 // ---------------------------------------------------------------------
 858 /* This creates a Group and the Package to link this dependency to if
 859    needed and handles also the caching of the old endpoint */
 860 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 861                                                const string &PackageName,
 862                                                const string &Arch,
 863                                                const string &Version,
 864                                                unsigned int Op,
 865                                                unsigned int Type)
 866 {
 867    pkgCache::GrpIterator Grp;
 868    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 869    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 870       return false;
 871
 872    // Locate the target package
 873    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 874    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 875    if (Pkg.end() == true) {
 876       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 877          return false;
 878    }
 879
 880    // Is it a file dependency?
 881    if (unlikely(PackageName[0] == '/'))
 882       FoundFileDeps = true;
 883
 884    /* Caching the old end point speeds up generation substantially */
 885    if (OldDepVer != Ver) {
 886       OldDepLast = NULL;
 887       OldDepVer = Ver;
 888    }
 889
 890    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 891 }
 892                                                                         /*}}}*/
 893 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 894 // ---------------------------------------------------------------------
 895 /* */
 896 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 897                                                 const string &PkgName,
 898                                                 const string &PkgArch,
 899                                                 const string &Version)
 900 {
 901    pkgCache &Cache = Owner->Cache;
 902
 903    // We do not add self referencing provides
 904    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 905         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 906       return true;
 907
 908    // Get a structure
 909    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 910    if (unlikely(Provides == 0))
 911       return false;
 912    Cache.HeaderP->ProvidesCount++;
 913
 914    // Fill it in
 915    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 916    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 917    Prv->Version = Ver.Index();
 918    Prv->NextPkgProv = Ver->ProvidesList;
 919    Ver->ProvidesList = Prv.Index();
 920    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 921       return false;
 922
 923    // Locate the target package
 924    pkgCache::PkgIterator Pkg;
 925    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 926    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 927       return false;
 928
 929    // Link it to the package
 930    Prv->ParentPkg = Pkg.Index();
 931    Prv->NextProvides = Pkg->ProvidesList;
 932    Pkg->ProvidesList = Prv.Index();
 933
 934    return true;
 935 }
 936                                                                         /*}}}*/
 937 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 938 // ---------------------------------------------------------------------
 939 /* This is used to select which file is to be associated with all newly
 940    added versions. The caller is responsible for setting the IMS fields. */
 941 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 942                                    const pkgIndexFile &Index,
 943                                    unsigned long Flags)
 944 {
 945    // Get some space for the structure
 946    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 947    if (unlikely(idxFile == 0))
 948       return false;
 949    CurrentFile = Cache.PkgFileP + idxFile;
 950
 951    // Fill it in
 952    map_ptrloc const idxFileName = WriteStringInMap(File);
 953    map_ptrloc const idxSite = WriteUniqString(Site);
 954    if (unlikely(idxFileName == 0 || idxSite == 0))
 955       return false;
 956    CurrentFile->FileName = idxFileName;
 957    CurrentFile->Site = idxSite;
 958    CurrentFile->NextFile = Cache.HeaderP->FileList;
 959    CurrentFile->Flags = Flags;
 960    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 961    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 962    if (unlikely(idxIndexType == 0))
 963       return false;
 964    CurrentFile->IndexType = idxIndexType;
 965    PkgFileName = File;
 966    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 967    Cache.HeaderP->PackageFileCount++;
 968
 969    if (Progress != 0)
 970       Progress->SubProgress(Index.Size());
 971    return true;
 972 }
 973                                                                         /*}}}*/
 974 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 975 // ---------------------------------------------------------------------
 976 /* This is used to create handles to strings. Given the same text it
 977    always returns the same number */
 978 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 979                                                  unsigned int Size)
 980 {
 981    /* We use a very small transient hash table here, this speeds up generation
 982       by a fair amount on slower machines */
 983    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 984    if (Bucket != 0 &&
 985        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 986       return Bucket->String;
 987
 988    // Search for an insertion point
 989    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 990    int Res = 1;
 991    map_ptrloc *Last = &Cache.HeaderP->StringList;
 992    for (; I != Cache.StringItemP; Last = &I->NextItem,
 993         I = Cache.StringItemP + I->NextItem)
 994    {
 995       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 996       if (Res >= 0)
 997          break;
 998    }
 999
1000    // Match
1001    if (Res == 0)
1002    {
1003       Bucket = I;
1004       return I->String;
1005    }
1006
1007    // Get a structure
1008    void const * const oldMap = Map.Data();
1009    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
1010    if (Item == 0)
1011       return 0;
1012
1013    map_ptrloc const idxString = WriteStringInMap(S,Size);
1014    if (unlikely(idxString == 0))
1015       return 0;
1016    if (oldMap != Map.Data()) {
1017       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
1018       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
1019    }
1020    *Last = Item;
1021
1022    // Fill in the structure
1023    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
1024    ItemP->NextItem = I - Cache.StringItemP;
1025    ItemP->String = idxString;
1026
1027    Bucket = ItemP;
1028    return ItemP->String;
1029 }
1030                                                                         /*}}}*/
1031 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1032 // ---------------------------------------------------------------------
1033 /* This just verifies that each file in the list of index files exists,
1034    has matching attributes with the cache and the cache does not have
1035    any extra files. */
1036 static bool CheckValidity(const string &CacheFile,
1037                           pkgSourceList &List,
1038                           FileIterator Start,
1039                           FileIterator End,
1040                           MMap **OutMap = 0)
1041 {
1042    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1043    // No file, certainly invalid
1044    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1045    {
1046       if (Debug == true)
1047          std::clog << "CacheFile doesn't exist" << std::endl;
1048       return false;
1049    }
1050
1051    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1052    {
1053       if (Debug == true)
1054          std::clog << "sources.list is newer than the cache" << std::endl;
1055       return false;
1056    }
1057
1058    // Map it
1059    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1060    SPtr<MMap> Map = new MMap(CacheF,0);
1061    pkgCache Cache(Map);
1062    if (_error->PendingError() == true || Map->Size() == 0)
1063    {
1064       if (Debug == true)
1065          std::clog << "Errors are pending or Map is empty()" << std::endl;
1066       _error->Discard();
1067       return false;
1068    }
1069
1070    /* Now we check every index file, see if it is in the cache,
1071       verify the IMS data and check that it is on the disk too.. */
1072    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1073    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1074    for (; Start != End; ++Start)
1075    {
1076       if (Debug == true)
1077          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1078       if ((*Start)->HasPackages() == false)
1079       {
1080          if (Debug == true)
1081             std::clog << "Has NO packages" << std::endl;
1082          continue;
1083       }
1084
1085       if ((*Start)->Exists() == false)
1086       {
1087 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1088          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1089                           (*Start)->Describe().c_str());
1090 #endif
1091          if (Debug == true)
1092             std::clog << "file doesn't exist" << std::endl;
1093          continue;
1094       }
1095
1096       // FindInCache is also expected to do an IMS check.
1097       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1098       if (File.end() == true)
1099       {
1100          if (Debug == true)
1101             std::clog << "FindInCache returned end-Pointer" << std::endl;
1102          return false;
1103       }
1104
1105       Visited[File->ID] = true;
1106       if (Debug == true)
1107          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1108    }
1109
1110    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1111       if (Visited[I] == false)
1112       {
1113          if (Debug == true)
1114             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1115          return false;
1116       }
1117
1118    if (_error->PendingError() == true)
1119    {
1120       if (Debug == true)
1121       {
1122          std::clog << "Validity failed because of pending errors:" << std::endl;
1123          _error->DumpErrors();
1124       }
1125       _error->Discard();
1126       return false;
1127    }
1128
1129    if (OutMap != 0)
1130       *OutMap = Map.UnGuard();
1131    return true;
1132 }
1133                                                                         /*}}}*/
1134 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1135 // ---------------------------------------------------------------------
1136 /* Size is kind of an abstract notion that is only used for the progress
1137    meter */
1138 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1139 {
1140    unsigned long TotalSize = 0;
1141    for (; Start != End; ++Start)
1142    {
1143       if ((*Start)->HasPackages() == false)
1144          continue;
1145       TotalSize += (*Start)->Size();
1146    }
1147    return TotalSize;
1148 }
1149                                                                         /*}}}*/
1150 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1151 // ---------------------------------------------------------------------
1152 /* */
1153 static bool BuildCache(pkgCacheGenerator &Gen,
1154                        OpProgress *Progress,
1155                        unsigned long &CurrentSize,unsigned long TotalSize,
1156                        FileIterator Start, FileIterator End)
1157 {
1158    FileIterator I;
1159    for (I = Start; I != End; ++I)
1160    {
1161       if ((*I)->HasPackages() == false)
1162          continue;
1163
1164       if ((*I)->Exists() == false)
1165          continue;
1166
1167       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1168       {
1169          _error->Warning("Duplicate sources.list entry %s",
1170                          (*I)->Describe().c_str());
1171          continue;
1172       }
1173
1174       unsigned long Size = (*I)->Size();
1175       if (Progress != NULL)
1176          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1177       CurrentSize += Size;
1178
1179       if ((*I)->Merge(Gen,Progress) == false)
1180          return false;
1181    }
1182
1183    if (Gen.HasFileDeps() == true)
1184    {
1185       if (Progress != NULL)
1186          Progress->Done();
1187       TotalSize = ComputeSize(Start, End);
1188       CurrentSize = 0;
1189       for (I = Start; I != End; ++I)
1190       {
1191          unsigned long Size = (*I)->Size();
1192          if (Progress != NULL)
1193             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1194          CurrentSize += Size;
1195          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1196             return false;
1197       }
1198    }
1199
1200    return true;
1201 }
1202                                                                         /*}}}*/
1203 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1204 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1205    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1206    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1207    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1208    Flags |= MMap::Moveable;
1209    if (_config->FindB("APT::Cache-Fallback", false) == true)
1210       Flags |= MMap::Fallback;
1211    if (CacheF != NULL)
1212       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1213    else
1214       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1215 }
1216                                                                         /*}}}*/
1217 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1218 // ---------------------------------------------------------------------
1219 /* This makes sure that the status cache (the cache that has all
1220    index files from the sources list and all local ones) is ready
1221    to be mmaped. If OutMap is not zero then a MMap object representing
1222    the cache will be stored there. This is pretty much mandetory if you
1223    are using AllowMem. AllowMem lets the function be run as non-root
1224    where it builds the cache 'fast' into a memory buffer. */
1225 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1226                         MMap **OutMap, bool AllowMem)
1227    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1228 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1229                         MMap **OutMap,bool AllowMem)
1230 {
1231    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1232
1233    std::vector<pkgIndexFile *> Files;
1234    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1235         i != List.end();
1236         ++i)
1237    {
1238       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1239       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1240            j != Indexes->end();
1241            ++j)
1242          Files.push_back (*j);
1243    }
1244
1245    unsigned long const EndOfSource = Files.size();
1246    if (_system->AddStatusFiles(Files) == false)
1247       return false;
1248
1249    // Decide if we can write to the files..
1250    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1251    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1252
1253    // ensure the cache directory exists
1254    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1255    {
1256       string dir = _config->FindDir("Dir::Cache");
1257       size_t const len = dir.size();
1258       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1259          dir = dir.substr(0, len - 5);
1260       if (CacheFile.empty() == false)
1261          CreateDirectory(dir, flNotFile(CacheFile));
1262       if (SrcCacheFile.empty() == false)
1263          CreateDirectory(dir, flNotFile(SrcCacheFile));
1264    }
1265
1266    // Decide if we can write to the cache
1267    bool Writeable = false;
1268    if (CacheFile.empty() == false)
1269       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1270    else
1271       if (SrcCacheFile.empty() == false)
1272          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1273    if (Debug == true)
1274       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1275
1276    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1277       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1278
1279    if (Progress != NULL)
1280       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1281
1282    // Cache is OK, Fin.
1283    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1284    {
1285       if (Progress != NULL)
1286          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1287       if (Debug == true)
1288          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1289       return true;
1290    }
1291    else if (Debug == true)
1292          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1293
1294    /* At this point we know we need to reconstruct the package cache,
1295       begin. */
1296    SPtr<FileFd> CacheF;
1297    SPtr<DynamicMMap> Map;
1298    if (Writeable == true && CacheFile.empty() == false)
1299    {
1300       _error->PushToStack();
1301       unlink(CacheFile.c_str());
1302       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1303       fchmod(CacheF->Fd(),0644);
1304       Map = CreateDynamicMMap(CacheF, MMap::Public);
1305       if (_error->PendingError() == true)
1306       {
1307          delete CacheF.UnGuard();
1308          delete Map.UnGuard();
1309          if (Debug == true)
1310             std::clog << "Open filebased MMap FAILED" << std::endl;
1311          Writeable = false;
1312          if (AllowMem == false)
1313          {
1314             _error->MergeWithStack();
1315             return false;
1316          }
1317          _error->RevertToStack();
1318       }
1319       else
1320       {
1321          _error->MergeWithStack();
1322          if (Debug == true)
1323             std::clog << "Open filebased MMap" << std::endl;
1324       }
1325    }
1326    if (Writeable == false || CacheFile.empty() == true)
1327    {
1328       // Just build it in memory..
1329       Map = CreateDynamicMMap(NULL);
1330       if (Debug == true)
1331          std::clog << "Open memory Map (not filebased)" << std::endl;
1332    }
1333
1334    // Lets try the source cache.
1335    unsigned long CurrentSize = 0;
1336    unsigned long TotalSize = 0;
1337    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1338                      Files.begin()+EndOfSource) == true)
1339    {
1340       if (Debug == true)
1341          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1342       // Preload the map with the source cache
1343       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1344       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1345       if ((alloc == 0 && _error->PendingError())
1346                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1347                                 SCacheF.Size()) == false)
1348          return false;
1349
1350       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1351
1352       // Build the status cache
1353       pkgCacheGenerator Gen(Map.Get(),Progress);
1354       if (_error->PendingError() == true)
1355          return false;
1356       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1357                      Files.begin()+EndOfSource,Files.end()) == false)
1358          return false;
1359    }
1360    else
1361    {
1362       if (Debug == true)
1363          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1364       TotalSize = ComputeSize(Files.begin(),Files.end());
1365
1366       // Build the source cache
1367       pkgCacheGenerator Gen(Map.Get(),Progress);
1368       if (_error->PendingError() == true)
1369          return false;
1370       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1371                      Files.begin(),Files.begin()+EndOfSource) == false)
1372          return false;
1373
1374       // Write it back
1375       if (Writeable == true && SrcCacheFile.empty() == false)
1376       {
1377          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1378          if (_error->PendingError() == true)
1379             return false;
1380
1381          fchmod(SCacheF.Fd(),0644);
1382
1383          // Write out the main data
1384          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1385             return _error->Error(_("IO Error saving source cache"));
1386          SCacheF.Sync();
1387
1388          // Write out the proper header
1389          Gen.GetCache().HeaderP->Dirty = false;
1390          if (SCacheF.Seek(0) == false ||
1391              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1392             return _error->Error(_("IO Error saving source cache"));
1393          Gen.GetCache().HeaderP->Dirty = true;
1394          SCacheF.Sync();
1395       }
1396
1397       // Build the status cache
1398       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1399                      Files.begin()+EndOfSource,Files.end()) == false)
1400          return false;
1401    }
1402    if (Debug == true)
1403       std::clog << "Caches are ready for shipping" << std::endl;
1404
1405    if (_error->PendingError() == true)
1406       return false;
1407    if (OutMap != 0)
1408    {
1409       if (CacheF != 0)
1410       {
1411          delete Map.UnGuard();
1412          *OutMap = new MMap(*CacheF,0);
1413       }
1414       else
1415       {
1416          *OutMap = Map.UnGuard();
1417       }
1418    }
1419
1420    return true;
1421 }
1422                                                                         /*}}}*/
1423 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1424 // ---------------------------------------------------------------------
1425 /* */
1426 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1427    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1428 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1429 {
1430    std::vector<pkgIndexFile *> Files;
1431    unsigned long EndOfSource = Files.size();
1432    if (_system->AddStatusFiles(Files) == false)
1433       return false;
1434
1435    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1436    unsigned long CurrentSize = 0;
1437    unsigned long TotalSize = 0;
1438
1439    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1440
1441    // Build the status cache
1442    if (Progress != NULL)
1443       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1444    pkgCacheGenerator Gen(Map.Get(),Progress);
1445    if (_error->PendingError() == true)
1446       return false;
1447    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1448                   Files.begin()+EndOfSource,Files.end()) == false)
1449       return false;
1450
1451    if (_error->PendingError() == true)
1452       return false;
1453    *OutMap = Map.UnGuard();
1454
1455    return true;
1456 }
1457                                                                         /*}}}*/
1458 // IsDuplicateDescription                                               /*{{{*/
1459 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1460                             MD5SumValue const &CurMd5, std::string const &CurLang)
1461 {
1462    // Descriptions in the same link-list have all the same md5
1463    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1464       return false;
1465    for (; Desc.end() == false; ++Desc)
1466       if (Desc.LanguageCode() == CurLang)
1467          return true;
1468    return false;
1469 }
1470                                                                         /*}}}*/
1471 // CacheGenerator::FinishCache                                          /*{{{*/
1472 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
1473 {
1474    return true;
1475 }
1476                                                                         /*}}}*/