apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 181       // Ignore views outside of the cache.
 182       if (ViewP->data() < static_cast<const char*>(oldMap)
 183          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 184          continue;
 185       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 186       *ViewP = StringView(data , ViewP->size());
 187    }
 188 }                                                                       /*}}}*/
 189 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 190 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 191                                         const unsigned long &Len) {
 192    size_t oldSize = Map.Size();
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String, Len);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data(), oldSize);
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 201 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 202    size_t oldSize = Map.Size();
 203    void const * const oldMap = Map.Data();
 204    map_stringitem_t const index = Map.WriteString(String);
 205    if (index != 0)
 206       ReMap(oldMap, Map.Data(), oldSize);
 207    return index;
 208 }
 209                                                                         /*}}}*/
 210 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 211    size_t oldSize = Map.Size();
 212    void const * const oldMap = Map.Data();
 213    map_pointer_t const index = Map.Allocate(size);
 214    if (index != 0)
 215       ReMap(oldMap, Map.Data(), oldSize);
 216    return index;
 217 }
 218                                                                         /*}}}*/
 219 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 220 // ---------------------------------------------------------------------
 221 /* This provides the generation of the entries in the cache. Each loop
 222    goes through a single package record from the underlying parse engine. */
 223 bool pkgCacheGenerator::MergeList(ListParser &List,
 224                                   pkgCache::VerIterator *OutVer)
 225 {
 226    List.Owner = this;
 227
 228    unsigned int Counter = 0;
 229    while (List.Step() == true)
 230    {
 231       string const PackageName = List.Package();
 232       if (PackageName.empty() == true)
 233          return false;
 234
 235       Counter++;
 236       if (Counter % 100 == 0 && Progress != 0)
 237          Progress->Progress(List.Offset());
 238
 239       string Arch = List.Architecture();
 240       string const Version = List.Version();
 241       if (Version.empty() == true && Arch.empty() == true)
 242       {
 243          // package descriptions
 244          if (MergeListGroup(List, PackageName) == false)
 245             return false;
 246          continue;
 247       }
 248
 249       // Get a pointer to the package structure
 250       pkgCache::PkgIterator Pkg;
 251       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 252       if (NewPackage(Pkg, PackageName, Arch) == false)
 253          // TRANSLATOR: The first placeholder is a package name,
 254          // the other two should be copied verbatim as they include debug info
 255          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 256                               PackageName.c_str(), "NewPackage", 1);
 257
 258
 259       if (Version.empty() == true)
 260       {
 261          if (MergeListPackage(List, Pkg) == false)
 262             return false;
 263       }
 264       else
 265       {
 266          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 267             return false;
 268       }
 269
 270       if (OutVer != 0)
 271          return true;
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    return true;
 288 }
 289 // CacheGenerator::MergeListGroup                                       /*{{{*/
 290 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 291 {
 292    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 293    // a group has no data on it's own, only packages have it but these
 294    // stanzas like this come from Translation- files to add descriptions,
 295    // but without a version we don't need a description for it…
 296    if (Grp.end() == true)
 297       return true;
 298    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 299
 300    pkgCache::PkgIterator Pkg;
 301    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 302    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 303       if (MergeListPackage(List, Pkg) == false)
 304          return false;
 305
 306    return true;
 307 }
 308                                                                         /*}}}*/
 309 // CacheGenerator::MergeListPackage                                     /*{{{*/
 310 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 311 {
 312    // we first process the package, then the descriptions
 313    // (for deb this package processing is in fact a no-op)
 314    pkgCache::VerIterator Ver(Cache);
 315    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 316    if (List.UsePackage(Pkg, Ver) == false)
 317       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 318                            Pkg.Name(), "UsePackage", 1);
 319
 320    // Find the right version to write the description
 321    MD5SumValue CurMd5 = List.Description_md5();
 322    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 323    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 324    {
 325       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 326
 327       // a version can only have one md5 describing it
 328       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 329          continue;
 330
 331       map_stringitem_t md5idx = VerDesc->md5sum;
 332       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 333       {
 334          // don't add a new description if we have one for the given
 335          // md5 && language
 336          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 337             continue;
 338
 339          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 340       }
 341
 342       // we can stop here as all "same" versions will share the description
 343       break;
 344    }
 345
 346    return true;
 347 }
 348                                                                         /*}}}*/
 349 // CacheGenerator::MergeListVersion                                     /*{{{*/
 350 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 351                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 352 {
 353    pkgCache::VerIterator Ver = Pkg.VersionList();
 354    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 355    map_pointer_t *LastVer = &Pkg->VersionList;
 356    void const * oldMap = Map.Data();
 357
 358    unsigned short const Hash = List.VersionHash();
 359    if (Ver.end() == false)
 360    {
 361       /* We know the list is sorted so we use that fact in the search.
 362          Insertion of new versions is done with correct sorting */
 363       int Res = 1;
 364       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 365       {
 366          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 367          // Version is higher as current version - insert here
 368          if (Res > 0)
 369             break;
 370          // Versionstrings are equal - is hash also equal?
 371          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 372             break;
 373          // proceed with the next till we have either the right
 374          // or we found another version (which will be lower)
 375       }
 376
 377       /* We already have a version for this item, record that we saw it */
 378       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 379       {
 380          if (List.UsePackage(Pkg,Ver) == false)
 381             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 382                                  Pkg.Name(), "UsePackage", 2);
 383
 384          if (NewFileVer(Ver,List) == false)
 385             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 386                                  Pkg.Name(), "NewFileVer", 1);
 387
 388          // Read only a single record and return
 389          if (OutVer != 0)
 390          {
 391             *OutVer = Ver;
 392             return true;
 393          }
 394
 395          return true;
 396       }
 397    }
 398
 399    // Add a new version
 400    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 401    if (unlikely(verindex == 0))
 402       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 403                            Pkg.Name(), "NewVersion", 1);
 404
 405    if (oldMap != Map.Data())
 406          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 407    *LastVer = verindex;
 408
 409    if (unlikely(List.NewVersion(Ver) == false))
 410       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 411                            Pkg.Name(), "NewVersion", 2);
 412
 413    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 414       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 415                            Pkg.Name(), "UsePackage", 3);
 416
 417    if (unlikely(NewFileVer(Ver,List) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "NewFileVer", 2);
 420
 421    pkgCache::GrpIterator Grp = Pkg.Group();
 422    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 423
 424    /* If it is the first version of this package we need to add implicit
 425       Multi-Arch dependencies to all other package versions in the group now -
 426       otherwise we just add them for this new version */
 427    if (Pkg.VersionList()->NextVer == 0)
 428    {
 429       pkgCache::PkgIterator P = Grp.PackageList();
 430       Dynamic<pkgCache::PkgIterator> DynP(P);
 431       for (; P.end() != true; P = Grp.NextPkg(P))
 432       {
 433          if (P->ID == Pkg->ID)
 434             continue;
 435          pkgCache::VerIterator V = P.VersionList();
 436          Dynamic<pkgCache::VerIterator> DynV(V);
 437          for (; V.end() != true; ++V)
 438             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 439                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 440                                     Pkg.Name(), "AddImplicitDepends", 1);
 441       }
 442    }
 443    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 444       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 445                            Pkg.Name(), "AddImplicitDepends", 2);
 446
 447    // Read only a single record and return
 448    if (OutVer != 0)
 449    {
 450       *OutVer = Ver;
 451       return true;
 452    }
 453
 454    /* Record the Description(s) based on their master md5sum */
 455    MD5SumValue CurMd5 = List.Description_md5();
 456
 457    /* Before we add a new description we first search in the group for
 458       a version with a description of the same MD5 - if so we reuse this
 459       description group instead of creating our own for this version */
 460    for (pkgCache::PkgIterator P = Grp.PackageList();
 461         P.end() == false; P = Grp.NextPkg(P))
 462    {
 463       for (pkgCache::VerIterator V = P.VersionList();
 464            V.end() == false; ++V)
 465       {
 466          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 467             continue;
 468          Ver->DescriptionList = V->DescriptionList;
 469       }
 470    }
 471
 472    // We haven't found reusable descriptions, so add the first description(s)
 473    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 474    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 475    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 476       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 477          return false;
 478    return true;
 479 }
 480                                                                         /*}}}*/
 481 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 482 {
 483    pkgCache::DescIterator Desc;
 484    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 485
 486    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 487    if (unlikely(descindex == 0))
 488       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 489             Ver.ParentPkg().Name(), "NewDescription", 1);
 490
 491    md5idx = Desc->md5sum;
 492    Desc->ParentPkg = Ver.ParentPkg().Index();
 493
 494    // we add at the end, so that the start is constant as we need
 495    // that to be able to efficiently share these lists
 496    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 497    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 498    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 499    *LastNextDesc = descindex;
 500
 501    if (NewFileDesc(Desc,List) == false)
 502       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 503             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 504
 505    return true;
 506 }
 507                                                                         /*}}}*/
 508                                                                         /*}}}*/
 509 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* This creates a new group structure and adds it to the hash table */
 512 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 513 {
 514    Dynamic<StringView> DName(Name);
 515    Grp = Cache.FindGrp(Name);
 516    if (Grp.end() == false)
 517       return true;
 518
 519    // Get a structure
 520    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 521    if (unlikely(Group == 0))
 522       return false;
 523
 524    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 525    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 526    if (unlikely(idxName == 0))
 527       return false;
 528    Grp->Name = idxName;
 529
 530    // Insert it into the hash table
 531    unsigned long const Hash = Cache.Hash(Name);
 532    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 533
 534    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 535       insertAt = &(Cache.GrpP + *insertAt)->Next;
 536    Grp->Next = *insertAt;
 537    *insertAt = Group;
 538
 539    Grp->ID = Cache.HeaderP->GroupCount++;
 540    return true;
 541 }
 542                                                                         /*}}}*/
 543 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* This creates a new package structure and adds it to the hash table */
 546 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 547                                         StringView Arch) {
 548    pkgCache::GrpIterator Grp;
 549    Dynamic<StringView> DName(Name);
 550    Dynamic<StringView> DArch(Arch);
 551    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 552    if (unlikely(NewGroup(Grp, Name) == false))
 553       return false;
 554
 555    Pkg = Grp.FindPkg(Arch);
 556       if (Pkg.end() == false)
 557          return true;
 558
 559    // Get a structure
 560    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 561    if (unlikely(Package == 0))
 562       return false;
 563    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 564
 565    // Set the name, arch and the ID
 566    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 567    Pkg->Group = Grp.Index();
 568    // all is mapped to the native architecture
 569    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 570    if (unlikely(idxArch == 0))
 571       return false;
 572    Pkg->Arch = idxArch;
 573    Pkg->ID = Cache.HeaderP->PackageCount++;
 574
 575    // Insert the package into our package list
 576    if (Grp->FirstPackage == 0) // the group is new
 577    {
 578       Grp->FirstPackage = Package;
 579       // Insert it into the hash table
 580       map_id_t const Hash = Cache.Hash(Name);
 581       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 582       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 583          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 584       Pkg->NextPackage = *insertAt;
 585       *insertAt = Package;
 586    }
 587    else // Group the Packages together
 588    {
 589       // if sibling is provided by another package, this one is too
 590       {
 591          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 592          if (M.end() == false) {
 593             pkgCache::PrvIterator Prv;
 594             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 595             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 596             {
 597                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 598                   continue;
 599                pkgCache::VerIterator Ver = Prv.OwnerVer();
 600                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 601                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 602                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 603                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 604                {
 605                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 606                      continue;
 607                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 608                      return false;
 609                }
 610             }
 611          }
 612       }
 613       // let M-A:foreign package siblings provide this package
 614       {
 615          pkgCache::PkgIterator P;
 616          pkgCache::VerIterator Ver;
 617          Dynamic<pkgCache::PkgIterator> DynP(P);
 618          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 619
 620          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 621          {
 622             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 623                continue;
 624             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 625                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 626                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 627                      return false;
 628          }
 629       }
 630       // and negative dependencies, don't forget negative dependencies
 631       {
 632          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 633          if (M.end() == false) {
 634             pkgCache::DepIterator Dep;
 635             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 636             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 637             {
 638                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 639                   continue;
 640                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 641                      Dep->Type != pkgCache::Dep::Replaces)
 642                   continue;
 643                pkgCache::VerIterator Ver = Dep.ParentVer();
 644                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 645                map_pointer_t * unused = NULL;
 646                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 647                   return false;
 648             }
 649          }
 650       }
 651
 652       // this package is the new last package
 653       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 654       Pkg->NextPackage = LastPkg->NextPackage;
 655       LastPkg->NextPackage = Package;
 656    }
 657    Grp->LastPackage = Package;
 658
 659    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 660    if (Arch == "any")
 661    {
 662       size_t const found = Name.find(':');
 663       StringView const NameA = Name.substr(0, found);
 664       StringView ArchA = Name.substr(found + 1);
 665       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 666       if (PkgA.end() == false)
 667       {
 668          // ArchA is used inside the loop which might remap (NameA is not used)
 669          Dynamic<StringView> DynArchA(ArchA);
 670          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 671          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 672          for (; Prv.end() == false; ++Prv)
 673          {
 674             if (Prv.IsMultiArchImplicit())
 675                continue;
 676             pkgCache::VerIterator V = Prv.OwnerVer();
 677             if (ArchA != V.ParentPkg().Arch())
 678                continue;
 679             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 680                return false;
 681          }
 682          pkgCache::VerIterator V = PkgA.VersionList();
 683          Dynamic<pkgCache::VerIterator> DynV(V);
 684          for (; V.end() == false; ++V)
 685          {
 686             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 687                return false;
 688          }
 689       }
 690    }
 691    return true;
 692 }
 693                                                                         /*}}}*/
 694 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 695 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 696                                            pkgCache::PkgIterator &P,
 697                                            pkgCache::VerIterator &V)
 698 {
 699    // copy P.Arch() into a string here as a cache remap
 700    // in NewDepends() later may alter the pointer location
 701    string Arch = P.Arch() == NULL ? "" : P.Arch();
 702    map_pointer_t *OldDepLast = NULL;
 703    /* MultiArch handling introduces a lot of implicit Dependencies:
 704       - MultiArch: same → Co-Installable if they have the same version
 705       - All others conflict with all other group members */
 706    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 707    pkgCache::PkgIterator D = G.PackageList();
 708    Dynamic<pkgCache::PkgIterator> DynD(D);
 709    map_stringitem_t const VerStrIdx = V->VerStr;
 710    for (; D.end() != true; D = G.NextPkg(D))
 711    {
 712       if (Arch == D.Arch() || D->VersionList == 0)
 713          continue;
 714       /* We allow only one installed arch at the time
 715          per group, therefore each group member conflicts
 716          with all other group members */
 717       if (coInstall == true)
 718       {
 719          // Replaces: ${self}:other ( << ${binary:Version})
 720          NewDepends(D, V, VerStrIdx,
 721                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 722                     OldDepLast);
 723          // Breaks: ${self}:other (!= ${binary:Version})
 724          NewDepends(D, V, VerStrIdx,
 725                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 726                     OldDepLast);
 727       } else {
 728          // Conflicts: ${self}:other
 729          NewDepends(D, V, 0,
 730                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 731                     OldDepLast);
 732       }
 733    }
 734    return true;
 735 }
 736 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 737                                            pkgCache::PkgIterator &D)
 738 {
 739    /* MultiArch handling introduces a lot of implicit Dependencies:
 740       - MultiArch: same → Co-Installable if they have the same version
 741       - All others conflict with all other group members */
 742    map_pointer_t *OldDepLast = NULL;
 743    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 744    if (coInstall == true)
 745    {
 746       map_stringitem_t const VerStrIdx = V->VerStr;
 747       // Replaces: ${self}:other ( << ${binary:Version})
 748       NewDepends(D, V, VerStrIdx,
 749                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 750                  OldDepLast);
 751       // Breaks: ${self}:other (!= ${binary:Version})
 752       NewDepends(D, V, VerStrIdx,
 753                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 754                  OldDepLast);
 755    } else {
 756       // Conflicts: ${self}:other
 757       NewDepends(D, V, 0,
 758                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 759                  OldDepLast);
 760    }
 761    return true;
 762 }
 763
 764                                                                         /*}}}*/
 765 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 766 // ---------------------------------------------------------------------
 767 /* */
 768 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 769                                    ListParser &List)
 770 {
 771    if (CurrentFile == 0)
 772       return true;
 773
 774    // Get a structure
 775    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 776    if (VerFile == 0)
 777       return false;
 778
 779    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 780    VF->File = CurrentFile - Cache.PkgFileP;
 781
 782    // Link it to the end of the list
 783    map_pointer_t *Last = &Ver->FileList;
 784    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 785       Last = &V->NextFile;
 786    VF->NextFile = *Last;
 787    *Last = VF.Index();
 788
 789    VF->Offset = List.Offset();
 790    VF->Size = List.Size();
 791    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 792       Cache.HeaderP->MaxVerFileSize = VF->Size;
 793    Cache.HeaderP->VerFileCount++;
 794
 795    return true;
 796 }
 797                                                                         /*}}}*/
 798 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 799 // ---------------------------------------------------------------------
 800 /* This puts a version structure in the linked list */
 801 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 802                                             const string &VerStr,
 803                                             map_pointer_t const ParentPkg,
 804                                             unsigned short const Hash,
 805                                             map_pointer_t const Next)
 806 {
 807    // Get a structure
 808    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 809    if (Version == 0)
 810       return 0;
 811
 812    // Fill it in
 813    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 814    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 815    Ver->NextVer = Next;
 816    Ver->ParentPkg = ParentPkg;
 817    Ver->Hash = Hash;
 818    Ver->ID = Cache.HeaderP->VersionCount++;
 819
 820    // try to find the version string in the group for reuse
 821    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 822    pkgCache::GrpIterator Grp = Pkg.Group();
 823    if (Pkg.end() == false && Grp.end() == false)
 824    {
 825       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 826       {
 827          if (Pkg == P)
 828             continue;
 829          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 830          {
 831             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 832             if (cmp == 0)
 833             {
 834                Ver->VerStr = V->VerStr;
 835                return Version;
 836             }
 837             else if (cmp < 0)
 838                break;
 839          }
 840       }
 841    }
 842    // haven't found the version string, so create
 843    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 844    if (unlikely(idxVerStr == 0))
 845       return 0;
 846    Ver->VerStr = idxVerStr;
 847    return Version;
 848 }
 849                                                                         /*}}}*/
 850 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 851 // ---------------------------------------------------------------------
 852 /* */
 853 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 854                                    ListParser &List)
 855 {
 856    if (CurrentFile == 0)
 857       return true;
 858
 859    // Get a structure
 860    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 861    if (DescFile == 0)
 862       return false;
 863
 864    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 865    DF->File = CurrentFile - Cache.PkgFileP;
 866
 867    // Link it to the end of the list
 868    map_pointer_t *Last = &Desc->FileList;
 869    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 870       Last = &D->NextFile;
 871
 872    DF->NextFile = *Last;
 873    *Last = DF.Index();
 874
 875    DF->Offset = List.Offset();
 876    DF->Size = List.Size();
 877    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 878       Cache.HeaderP->MaxDescFileSize = DF->Size;
 879    Cache.HeaderP->DescFileCount++;
 880
 881    return true;
 882 }
 883                                                                         /*}}}*/
 884 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 885 // ---------------------------------------------------------------------
 886 /* This puts a description structure in the linked list */
 887 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 888                                             const string &Lang,
 889                                             const MD5SumValue &md5sum,
 890                                             map_stringitem_t const idxmd5str)
 891 {
 892    // Get a structure
 893    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 894    if (Description == 0)
 895       return 0;
 896
 897    // Fill it in
 898    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 899    Desc->ID = Cache.HeaderP->DescriptionCount++;
 900    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 901    if (unlikely(idxlanguage_code == 0))
 902       return 0;
 903    Desc->language_code = idxlanguage_code;
 904
 905    if (idxmd5str != 0)
 906       Desc->md5sum = idxmd5str;
 907    else
 908    {
 909       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 910       if (unlikely(idxmd5sum == 0))
 911          return 0;
 912       Desc->md5sum = idxmd5sum;
 913    }
 914
 915    return Description;
 916 }
 917                                                                         /*}}}*/
 918 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 919 // ---------------------------------------------------------------------
 920 /* This creates a dependency element in the tree. It is linked to the
 921    version and to the package that it is pointing to. */
 922 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 923                                    pkgCache::VerIterator &Ver,
 924                                    map_pointer_t const Version,
 925                                    uint8_t const Op,
 926                                    uint8_t const Type,
 927                                    map_pointer_t* &OldDepLast)
 928 {
 929    void const * const oldMap = Map.Data();
 930    // Get a structure
 931    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 932    if (unlikely(Dependency == 0))
 933       return false;
 934
 935    bool isDuplicate = false;
 936    map_pointer_t DependencyData = 0;
 937    map_pointer_t PreviousData = 0;
 938    if (Pkg->RevDepends != 0)
 939    {
 940       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 941       DependencyData = L->DependencyData;
 942       do {
 943          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 944          if (Version > D->Version)
 945             break;
 946          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 947          {
 948             isDuplicate = true;
 949             break;
 950          }
 951          PreviousData = DependencyData;
 952          DependencyData = D->NextData;
 953       } while (DependencyData != 0);
 954    }
 955
 956    if (isDuplicate == false)
 957    {
 958       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 959       if (unlikely(DependencyData == 0))
 960         return false;
 961    }
 962
 963    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 964    Link->ParentVer = Ver.Index();
 965    Link->DependencyData = DependencyData;
 966    Link->ID = Cache.HeaderP->DependsCount++;
 967
 968    pkgCache::DepIterator Dep(Cache, Link);
 969    if (isDuplicate == false)
 970    {
 971       Dep->Type = Type;
 972       Dep->CompareOp = Op;
 973       Dep->Version = Version;
 974       Dep->Package = Pkg.Index();
 975       ++Cache.HeaderP->DependsDataCount;
 976       if (PreviousData != 0)
 977       {
 978          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 979          Dep->NextData = D->NextData;
 980          D->NextData = DependencyData;
 981       }
 982       else if (Pkg->RevDepends != 0)
 983       {
 984          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 985          Dep->NextData = D->DependencyData;
 986       }
 987    }
 988
 989    if (isDuplicate == true || PreviousData != 0)
 990    {
 991       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 992       Link->NextRevDepends = L->NextRevDepends;
 993       L->NextRevDepends = Dependency;
 994    }
 995    else
 996    {
 997       Link->NextRevDepends = Pkg->RevDepends;
 998       Pkg->RevDepends = Dependency;
 999    }
1000
1001
1002    // Do we know where to link the Dependency to?
1003    if (OldDepLast == NULL)
1004    {
1005       OldDepLast = &Ver->DependsList;
1006       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1007          OldDepLast = &D->NextDepends;
1008    } else if (oldMap != Map.Data())
1009       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1010
1011    Dep->NextDepends = *OldDepLast;
1012    *OldDepLast = Dependency;
1013    OldDepLast = &Dep->NextDepends;
1014    return true;
1015 }
1016                                                                         /*}}}*/
1017 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1018 // ---------------------------------------------------------------------
1019 /* This creates a Group and the Package to link this dependency to if
1020    needed and handles also the caching of the old endpoint */
1021 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1022                                                StringView PackageName,
1023                                                StringView Arch,
1024                                                StringView Version,
1025                                                uint8_t const Op,
1026                                                uint8_t const Type)
1027 {
1028    pkgCache::GrpIterator Grp;
1029    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1030    Dynamic<StringView> DynPackageName(PackageName);
1031    Dynamic<StringView> DynArch(Arch);
1032    Dynamic<StringView> DynVersion(Version);
1033    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1034       return false;
1035
1036    map_stringitem_t idxVersion = 0;
1037    if (Version.empty() == false)
1038    {
1039       int const CmpOp = Op & 0x0F;
1040       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1041       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1042          idxVersion = Ver->VerStr;
1043
1044       if (idxVersion == 0)
1045       {
1046          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1047          if (unlikely(idxVersion == 0))
1048             return false;
1049       }
1050    }
1051
1052    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1053          Type == pkgCache::Dep::Conflicts ||
1054          Type == pkgCache::Dep::Replaces);
1055
1056    pkgCache::PkgIterator Pkg;
1057    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1058    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1059    {
1060       // Locate the target package
1061       Pkg = Grp.FindPkg(Arch);
1062       if (Pkg.end() == true) {
1063          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1064             return false;
1065       }
1066
1067       /* Caching the old end point speeds up generation substantially */
1068       if (OldDepVer != Ver) {
1069          OldDepLast = NULL;
1070          OldDepVer = Ver;
1071       }
1072
1073       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1074    }
1075    else
1076    {
1077       /* Caching the old end point speeds up generation substantially */
1078       if (OldDepVer != Ver) {
1079          OldDepLast = NULL;
1080          OldDepVer = Ver;
1081       }
1082
1083       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1084       {
1085          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1086                return false;
1087       }
1088    }
1089    return true;
1090 }
1091                                                                         /*}}}*/
1092 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1093 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1094                                                 StringView PkgName,
1095                                                 StringView PkgArch,
1096                                                 StringView Version,
1097                                                 uint8_t const Flags)
1098 {
1099    pkgCache const &Cache = Owner->Cache;
1100    Dynamic<StringView> DynPkgName(PkgName);
1101    Dynamic<StringView> DynArch(PkgArch);
1102    Dynamic<StringView> DynVersion(Version);
1103
1104    // We do not add self referencing provides
1105    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1106         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1107          (Version.empty() || Version == Ver.VerStr()))
1108       return true;
1109
1110    // Locate the target package
1111    pkgCache::PkgIterator Pkg;
1112    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1113    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1114       return false;
1115
1116    map_stringitem_t idxProvideVersion = 0;
1117    if (Version.empty() == false) {
1118       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1119       if (unlikely(idxProvideVersion == 0))
1120          return false;
1121    }
1122    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1123 }
1124 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1125                                     pkgCache::PkgIterator &Pkg,
1126                                     map_pointer_t const ProvideVersion,
1127                                     uint8_t const Flags)
1128 {
1129    // Get a structure
1130    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1131    if (unlikely(Provides == 0))
1132       return false;
1133    ++Cache.HeaderP->ProvidesCount;
1134
1135    // Fill it in
1136    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1137    Prv->Version = Ver.Index();
1138    Prv->ProvideVersion = ProvideVersion;
1139    Prv->Flags = Flags;
1140    Prv->NextPkgProv = Ver->ProvidesList;
1141    Ver->ProvidesList = Prv.Index();
1142
1143    // Link it to the package
1144    Prv->ParentPkg = Pkg.Index();
1145    Prv->NextProvides = Pkg->ProvidesList;
1146    Pkg->ProvidesList = Prv.Index();
1147    return true;
1148 }
1149                                                                         /*}}}*/
1150 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1151 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1152                                 StringView Version, uint8_t const Flags) {
1153    pkgCache &Cache = Owner->Cache;
1154    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1155    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1156    Dynamic<StringView> DynPackage(Package);
1157    Dynamic<StringView> DynVersion(Version);
1158
1159    if (Grp.end() == true)
1160       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1161    else
1162    {
1163       map_stringitem_t idxProvideVersion = 0;
1164       if (Version.empty() == false) {
1165          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1166          if (unlikely(idxProvideVersion == 0))
1167             return false;
1168       }
1169
1170       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1171       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1172       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1173       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1174       pkgCache::PkgIterator Pkg;
1175       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1176       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1177       {
1178          if (isImplicit && OwnerPkg == Pkg)
1179             continue;
1180          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1181             continue;
1182          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1183             return false;
1184       }
1185    }
1186    return true;
1187 }
1188                                                                         /*}}}*/
1189 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1190       pkgCache::VerIterator const &Ver)
1191 {
1192    return Hash == Ver->Hash;
1193 }
1194                                                                         /*}}}*/
1195 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1196 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1197                                    unsigned long Flags)
1198 {
1199    if (File.empty() && Site.empty())
1200    {
1201       CurrentRlsFile = NULL;
1202       return true;
1203    }
1204
1205    // Get some space for the structure
1206    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1207    if (unlikely(idxFile == 0))
1208       return false;
1209    CurrentRlsFile = Cache.RlsFileP + idxFile;
1210
1211    // Fill it in
1212    map_stringitem_t const idxFileName = WriteStringInMap(File);
1213    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1214    if (unlikely(idxFileName == 0 || idxSite == 0))
1215       return false;
1216    CurrentRlsFile->FileName = idxFileName;
1217    CurrentRlsFile->Site = idxSite;
1218    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1219    CurrentRlsFile->Flags = Flags;
1220    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1221    RlsFileName = File;
1222    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1223    Cache.HeaderP->ReleaseFileCount++;
1224
1225    return true;
1226 }
1227                                                                         /*}}}*/
1228 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1229 // ---------------------------------------------------------------------
1230 /* This is used to select which file is to be associated with all newly
1231    added versions. The caller is responsible for setting the IMS fields. */
1232 bool pkgCacheGenerator::SelectFile(std::string const &File,
1233                                    pkgIndexFile const &Index,
1234                                    std::string const &Architecture,
1235                                    std::string const &Component,
1236                                    unsigned long const Flags)
1237 {
1238    // Get some space for the structure
1239    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1240    if (unlikely(idxFile == 0))
1241       return false;
1242    CurrentFile = Cache.PkgFileP + idxFile;
1243
1244    // Fill it in
1245    map_stringitem_t const idxFileName = WriteStringInMap(File);
1246    if (unlikely(idxFileName == 0))
1247       return false;
1248    CurrentFile->FileName = idxFileName;
1249    CurrentFile->NextFile = Cache.HeaderP->FileList;
1250    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1251    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1252    if (unlikely(idxIndexType == 0))
1253       return false;
1254    CurrentFile->IndexType = idxIndexType;
1255    if (Architecture.empty())
1256       CurrentFile->Architecture = 0;
1257    else
1258    {
1259       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1260       if (unlikely(arch == 0))
1261          return false;
1262       CurrentFile->Architecture = arch;
1263    }
1264    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1265    if (unlikely(component == 0))
1266       return false;
1267    CurrentFile->Component = component;
1268    CurrentFile->Flags = Flags;
1269    if (CurrentRlsFile != NULL)
1270       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1271    else
1272       CurrentFile->Release = 0;
1273    PkgFileName = File;
1274    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1275    Cache.HeaderP->PackageFileCount++;
1276
1277    if (Progress != 0)
1278       Progress->SubProgress(Index.Size());
1279    return true;
1280 }
1281                                                                         /*}}}*/
1282 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1283 // ---------------------------------------------------------------------
1284 /* This is used to create handles to strings. Given the same text it
1285    always returns the same number */
1286 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1287                                                  unsigned int Size)
1288 {
1289    auto strings = &strMixed;
1290    switch(type) {
1291       case MIXED: strings = &strMixed; break;
1292       case PKGNAME: strings = &strPkgNames; break;
1293       case VERSIONNUMBER: strings = &strVersions; break;
1294       case SECTION: strings = &strSections; break;
1295       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1296    }
1297
1298    auto const item = strings->find({S, Size, nullptr, 0});
1299    if (item != strings->end())
1300       return item->item;
1301
1302    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1303    strings->insert({nullptr, Size, this, idxString});
1304    return idxString;
1305 }
1306                                                                         /*}}}*/
1307 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1308 // ---------------------------------------------------------------------
1309 /* This just verifies that each file in the list of index files exists,
1310    has matching attributes with the cache and the cache does not have
1311    any extra files. */
1312 class APT_HIDDEN ScopedErrorRevert {
1313 public:
1314    ScopedErrorRevert() { _error->PushToStack(); }
1315    ~ScopedErrorRevert() { _error->RevertToStack(); }
1316 };
1317 static bool CheckValidity(const string &CacheFile,
1318                           pkgSourceList &List,
1319                           FileIterator const Start,
1320                           FileIterator const End,
1321                           MMap **OutMap = 0,
1322                           pkgCache **OutCache = 0)
1323 {
1324    ScopedErrorRevert ser;
1325    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1326    // No file, certainly invalid
1327    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1328    {
1329       if (Debug == true)
1330          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1331       return false;
1332    }
1333
1334    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1335    {
1336       if (Debug == true)
1337          std::clog << "sources.list is newer than the cache" << std::endl;
1338       return false;
1339    }
1340
1341    // Map it
1342    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1343    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1344    if (unlikely(Map->validData()) == false)
1345       return false;
1346    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1347    pkgCache &Cache = *CacheP.get();
1348    if (_error->PendingError() || Map->Size() == 0)
1349    {
1350       if (Debug == true)
1351          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1352       return false;
1353    }
1354
1355    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1356    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1357    std::vector<pkgIndexFile *> Files;
1358    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1359    {
1360       if (Debug == true)
1361          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1362       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1363       if (RlsFile.end() == true)
1364       {
1365          if (Debug == true)
1366             std::clog << "FindInCache returned end-Pointer" << std::endl;
1367          return false;
1368       }
1369
1370       RlsVisited[RlsFile->ID] = true;
1371       if (Debug == true)
1372          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1373
1374       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1375       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1376             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1377    }
1378    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1379       if (RlsVisited[I] == false)
1380       {
1381          if (Debug == true)
1382             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1383          return false;
1384       }
1385
1386    std::copy(Start, End, std::back_inserter(Files));
1387
1388    /* Now we check every index file, see if it is in the cache,
1389       verify the IMS data and check that it is on the disk too.. */
1390    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1391    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1392    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1393    {
1394       if (Debug == true)
1395          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1396       if ((*PkgFile)->Exists() == false)
1397       {
1398          if (Debug == true)
1399             std::clog << "file doesn't exist" << std::endl;
1400          continue;
1401       }
1402
1403       // FindInCache is also expected to do an IMS check.
1404       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1405       if (File.end() == true)
1406       {
1407          if (Debug == true)
1408             std::clog << "FindInCache returned end-Pointer" << std::endl;
1409          return false;
1410       }
1411
1412       Visited[File->ID] = true;
1413       if (Debug == true)
1414          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1415    }
1416
1417    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1418       if (Visited[I] == false)
1419       {
1420          if (Debug == true)
1421             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1422          return false;
1423       }
1424
1425    if (_error->PendingError() == true)
1426    {
1427       if (Debug == true)
1428       {
1429          std::clog << "Validity failed because of pending errors:" << std::endl;
1430          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1431       }
1432       return false;
1433    }
1434
1435    if (OutMap != 0)
1436       *OutMap = Map.release();
1437    if (OutCache != 0)
1438       *OutCache = CacheP.release();
1439    return true;
1440 }
1441                                                                         /*}}}*/
1442 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1443 // ---------------------------------------------------------------------
1444 /* Size is kind of an abstract notion that is only used for the progress
1445    meter */
1446 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1447 {
1448    map_filesize_t TotalSize = 0;
1449    if (List !=  NULL)
1450    {
1451       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1452       {
1453          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1454          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1455             if ((*j)->HasPackages() == true)
1456                TotalSize += (*j)->Size();
1457       }
1458    }
1459
1460    for (; Start < End; ++Start)
1461    {
1462       if ((*Start)->HasPackages() == false)
1463          continue;
1464       TotalSize += (*Start)->Size();
1465    }
1466    return TotalSize;
1467 }
1468                                                                         /*}}}*/
1469 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1470 static bool BuildCache(pkgCacheGenerator &Gen,
1471                        OpProgress * const Progress,
1472                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1473                        pkgSourceList const * const List,
1474                        FileIterator const Start, FileIterator const End)
1475 {
1476    bool mergeFailure = false;
1477
1478    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1479       if (I->HasPackages() == false || mergeFailure)
1480          return;
1481
1482       if (I->Exists() == false)
1483          return;
1484
1485       if (I->FindInCache(Gen.GetCache()).end() == false)
1486       {
1487          _error->Warning("Duplicate sources.list entry %s",
1488                I->Describe().c_str());
1489          return;
1490       }
1491
1492       map_filesize_t const Size = I->Size();
1493       if (Progress != NULL)
1494          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1495       CurrentSize += Size;
1496
1497       if (I->Merge(Gen,Progress) == false)
1498          mergeFailure = true;
1499    };
1500
1501    if (List !=  NULL)
1502    {
1503       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1504       {
1505          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1506          {
1507             _error->Warning("Duplicate sources.list entry %s",
1508                   (*i)->Describe().c_str());
1509             continue;
1510          }
1511
1512          if ((*i)->Merge(Gen, Progress) == false)
1513             return false;
1514
1515          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1516          if (Indexes != NULL)
1517             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1518          if (mergeFailure)
1519             return false;
1520       }
1521    }
1522
1523    if (Start != End)
1524    {
1525       Gen.SelectReleaseFile("", "");
1526       std::for_each(Start, End, indexFileMerge);
1527       if (mergeFailure)
1528          return false;
1529    }
1530    return true;
1531 }
1532                                                                         /*}}}*/
1533 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1534 // ---------------------------------------------------------------------
1535 /* This makes sure that the status cache (the cache that has all
1536    index files from the sources list and all local ones) is ready
1537    to be mmaped. If OutMap is not zero then a MMap object representing
1538    the cache will be stored there. This is pretty much mandetory if you
1539    are using AllowMem. AllowMem lets the function be run as non-root
1540    where it builds the cache 'fast' into a memory buffer. */
1541 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1542 {
1543    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1544    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1545    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1546    Flags |= MMap::Moveable;
1547    if (_config->FindB("APT::Cache-Fallback", false) == true)
1548       Flags |= MMap::Fallback;
1549    if (CacheF != NULL)
1550       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1551    else
1552       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1553 }
1554 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1555       std::string const &FileName)
1556 {
1557    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1558    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1559       return false;
1560
1561    fchmod(SCacheF.Fd(),0644);
1562
1563    // Write out the main data
1564    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1565       return _error->Error(_("IO Error saving source cache"));
1566
1567    // Write out the proper header
1568    Gen->GetCache().HeaderP->Dirty = false;
1569    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1570    if (SCacheF.Seek(0) == false ||
1571          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1572       return _error->Error(_("IO Error saving source cache"));
1573    Gen->GetCache().HeaderP->Dirty = true;
1574    return true;
1575 }
1576 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1577       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1578 {
1579    Map.reset(CreateDynamicMMap(NULL, 0));
1580    if (unlikely(Map->validData()) == false)
1581       return false;
1582    FileFd CacheF(FileName, FileFd::ReadOnly);
1583    if (CacheF.IsOpen() == false || CacheF.Failed())
1584       return false;
1585    _error->PushToStack();
1586    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1587    bool const newError = _error->PendingError();
1588    _error->MergeWithStack();
1589    if (alloc == 0 && newError)
1590       return false;
1591    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1592       return false;
1593    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1594    return true;
1595 }
1596 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1597                         MMap **OutMap, bool AllowMem)
1598    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1599 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1600                         MMap **OutMap,bool)
1601 {
1602    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1603 }
1604 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1605                         MMap **OutMap,pkgCache **OutCache, bool)
1606 {
1607    // FIXME: deprecate the ignored AllowMem parameter
1608    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1609
1610    std::vector<pkgIndexFile *> Files;
1611    if (_system->AddStatusFiles(Files) == false)
1612       return false;
1613
1614    // Decide if we can write to the files..
1615    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1616    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1617
1618    // ensure the cache directory exists
1619    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1620    {
1621       string dir = _config->FindDir("Dir::Cache");
1622       size_t const len = dir.size();
1623       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1624          dir = dir.substr(0, len - 5);
1625       if (CacheFile.empty() == false)
1626          CreateDirectory(dir, flNotFile(CacheFile));
1627       if (SrcCacheFile.empty() == false)
1628          CreateDirectory(dir, flNotFile(SrcCacheFile));
1629    }
1630
1631    if (Progress != NULL)
1632       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1633
1634    bool pkgcache_fine = false;
1635    bool srcpkgcache_fine = false;
1636    bool volatile_fine = List.GetVolatileFiles().empty();
1637
1638    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1639                      volatile_fine ? OutCache : NULL) == true)
1640    {
1641       if (Debug == true)
1642          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1643       pkgcache_fine = true;
1644       srcpkgcache_fine = true;
1645    }
1646    if (pkgcache_fine == false)
1647    {
1648       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1649       {
1650          if (Debug == true)
1651             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1652          srcpkgcache_fine = true;
1653       }
1654    }
1655
1656    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1657    {
1658       if (Progress != NULL)
1659          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1660       return true;
1661    }
1662
1663    bool Writeable = false;
1664    if (srcpkgcache_fine == false || pkgcache_fine == false)
1665    {
1666       if (CacheFile.empty() == false)
1667          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1668       else if (SrcCacheFile.empty() == false)
1669          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1670
1671       if (Debug == true)
1672          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1673    }
1674
1675    // At this point we know we need to construct something, so get storage ready
1676    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1677    if (unlikely(Map->validData()) == false)
1678       return false;
1679    if (Debug == true)
1680       std::clog << "Open memory Map (not filebased)" << std::endl;
1681
1682    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1683    map_filesize_t CurrentSize = 0;
1684    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1685    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1686    if (srcpkgcache_fine == true && pkgcache_fine == false)
1687    {
1688       if (Debug == true)
1689          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1690       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1691          return false;
1692       srcpkgcache_fine = true;
1693       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1694    }
1695    else if (srcpkgcache_fine == false)
1696    {
1697       if (Debug == true)
1698          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1699       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1700
1701       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1702       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1703                Files.end(),Files.end()) == false)
1704          return false;
1705
1706       if (Writeable == true && SrcCacheFile.empty() == false)
1707          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1708             return false;
1709    }
1710
1711    if (pkgcache_fine == false)
1712    {
1713       if (Debug == true)
1714          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1715       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1716                Files.begin(), Files.end()) == false)
1717          return false;
1718
1719       if (Writeable == true && CacheFile.empty() == false)
1720          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1721             return false;
1722    }
1723
1724    if (Debug == true)
1725       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1726
1727    if (volatile_fine == false)
1728    {
1729       if (Gen == nullptr)
1730       {
1731          if (Debug == true)
1732             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1733          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1734             return false;
1735       }
1736
1737       Files = List.GetVolatileFiles();
1738       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1739                Files.begin(), Files.end()) == false)
1740          return false;
1741    }
1742
1743    if (OutMap != nullptr)
1744       *OutMap = Map.release();
1745
1746    if (Debug == true)
1747       std::clog << "Everything is ready for shipping" << std::endl;
1748    return true;
1749 }
1750                                                                         /*}}}*/
1751 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1752 class APT_HIDDEN ScopedErrorMerge {
1753 public:
1754    ScopedErrorMerge() { _error->PushToStack(); }
1755    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1756 };
1757 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1758    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1759 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1760 {
1761    std::vector<pkgIndexFile *> Files;
1762    if (_system->AddStatusFiles(Files) == false)
1763       return false;
1764
1765    ScopedErrorMerge sem;
1766    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1767    if (unlikely(Map->validData()) == false)
1768       return false;
1769    map_filesize_t CurrentSize = 0;
1770    map_filesize_t TotalSize = 0;
1771    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1772
1773    // Build the status cache
1774    if (Progress != NULL)
1775       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1776    pkgCacheGenerator Gen(Map.get(),Progress);
1777    if (_error->PendingError() == true)
1778       return false;
1779    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1780                   Files.begin(), Files.end()) == false)
1781       return false;
1782
1783    if (_error->PendingError() == true)
1784       return false;
1785    *OutMap = Map.release();
1786
1787    return true;
1788 }
1789                                                                         /*}}}*/
1790 // IsDuplicateDescription                                               /*{{{*/
1791 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1792                             MD5SumValue const &CurMd5, std::string const &CurLang)
1793 {
1794    // Descriptions in the same link-list have all the same md5
1795    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1796       return false;
1797    for (; Desc.end() == false; ++Desc)
1798       if (Desc.LanguageCode() == CurLang)
1799          return true;
1800    return false;
1801 }
1802                                                                         /*}}}*/
1803
1804 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1805 pkgCacheListParser::~pkgCacheListParser() {}