apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return false;
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (_error->PendingError() == true || Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    if (oldMap == newMap)
 152       return;
 153
 154    if (_config->FindB("Debug::pkgCacheGen", false))
 155       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 156
 157    Cache.ReMap(false);
 158
 159    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 160    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 161
 162    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 181         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 182       (*i)->ReMap(oldMap, newMap);
 183    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 184         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 185       (*i)->ReMap(oldMap, newMap);
 186    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 187       // Ignore views outside of the cache.
 188       if (ViewP->data() < static_cast<const char*>(oldMap)
 189          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 190          continue;
 191       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 192       *ViewP = StringView(data , ViewP->size());
 193    }
 194 }                                                                       /*}}}*/
 195 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 196 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 197                                         const unsigned long &Len) {
 198    size_t oldSize = Map.Size();
 199    void const * const oldMap = Map.Data();
 200    map_stringitem_t const index = Map.WriteString(String, Len);
 201    if (index != 0)
 202       ReMap(oldMap, Map.Data(), oldSize);
 203    return index;
 204 }
 205                                                                         /*}}}*/
 206 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 207 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 208    size_t oldSize = Map.Size();
 209    void const * const oldMap = Map.Data();
 210    map_stringitem_t const index = Map.WriteString(String);
 211    if (index != 0)
 212       ReMap(oldMap, Map.Data(), oldSize);
 213    return index;
 214 }
 215                                                                         /*}}}*/
 216 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 217    size_t oldSize = Map.Size();
 218    void const * const oldMap = Map.Data();
 219    map_pointer_t const index = Map.Allocate(size);
 220    if (index != 0)
 221       ReMap(oldMap, Map.Data(), oldSize);
 222    return index;
 223 }
 224                                                                         /*}}}*/
 225 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* This provides the generation of the entries in the cache. Each loop
 228    goes through a single package record from the underlying parse engine. */
 229 bool pkgCacheGenerator::MergeList(ListParser &List,
 230                                   pkgCache::VerIterator *OutVer)
 231 {
 232    List.Owner = this;
 233
 234    unsigned int Counter = 0;
 235    while (List.Step() == true)
 236    {
 237       string const PackageName = List.Package();
 238       if (PackageName.empty() == true)
 239          return false;
 240
 241       Counter++;
 242       if (Counter % 100 == 0 && Progress != 0)
 243          Progress->Progress(List.Offset());
 244
 245       APT::StringView Arch = List.Architecture();
 246       Dynamic<APT::StringView> DynArch(Arch);
 247       APT::StringView Version = List.Version();
 248       Dynamic<APT::StringView> DynVersion(Version);
 249       if (Version.empty() == true && Arch.empty() == true)
 250       {
 251          // package descriptions
 252          if (MergeListGroup(List, PackageName) == false)
 253             return false;
 254          continue;
 255       }
 256
 257       // Get a pointer to the package structure
 258       pkgCache::PkgIterator Pkg;
 259       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 260       if (NewPackage(Pkg, PackageName, Arch) == false)
 261          // TRANSLATOR: The first placeholder is a package name,
 262          // the other two should be copied verbatim as they include debug info
 263          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 264                               PackageName.c_str(), "NewPackage", 1);
 265
 266
 267       if (Version.empty() == true)
 268       {
 269          if (MergeListPackage(List, Pkg) == false)
 270             return false;
 271       }
 272       else
 273       {
 274          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 275             return false;
 276       }
 277
 278       if (OutVer != 0)
 279          return true;
 280    }
 281
 282    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 283       return _error->Error(_("Wow, you exceeded the number of package "
 284                              "names this APT is capable of."));
 285    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 286       return _error->Error(_("Wow, you exceeded the number of versions "
 287                              "this APT is capable of."));
 288    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 289       return _error->Error(_("Wow, you exceeded the number of descriptions "
 290                              "this APT is capable of."));
 291    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 292       return _error->Error(_("Wow, you exceeded the number of dependencies "
 293                              "this APT is capable of."));
 294
 295    return true;
 296 }
 297 // CacheGenerator::MergeListGroup                                       /*{{{*/
 298 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 299 {
 300    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 301    // a group has no data on it's own, only packages have it but these
 302    // stanzas like this come from Translation- files to add descriptions,
 303    // but without a version we don't need a description for it…
 304    if (Grp.end() == true)
 305       return true;
 306    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 307
 308    pkgCache::PkgIterator Pkg;
 309    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 310    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 311       if (MergeListPackage(List, Pkg) == false)
 312          return false;
 313
 314    return true;
 315 }
 316                                                                         /*}}}*/
 317 // CacheGenerator::MergeListPackage                                     /*{{{*/
 318 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 319 {
 320    // we first process the package, then the descriptions
 321    // (for deb this package processing is in fact a no-op)
 322    pkgCache::VerIterator Ver(Cache);
 323    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 324    if (List.UsePackage(Pkg, Ver) == false)
 325       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 326                            Pkg.Name(), "UsePackage", 1);
 327
 328    // Find the right version to write the description
 329    MD5SumValue CurMd5 = List.Description_md5();
 330    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 331    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 332    {
 333       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 334
 335       // a version can only have one md5 describing it
 336       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 337          continue;
 338
 339       map_stringitem_t md5idx = VerDesc->md5sum;
 340       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 341       {
 342          // don't add a new description if we have one for the given
 343          // md5 && language
 344          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 345             continue;
 346
 347          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 348       }
 349
 350       // we can stop here as all "same" versions will share the description
 351       break;
 352    }
 353
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357 // CacheGenerator::MergeListVersion                                     /*{{{*/
 358 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 359                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 360 {
 361    pkgCache::VerIterator Ver = Pkg.VersionList();
 362    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 363    map_pointer_t *LastVer = &Pkg->VersionList;
 364    void const * oldMap = Map.Data();
 365
 366    unsigned short const Hash = List.VersionHash();
 367    if (Ver.end() == false)
 368    {
 369       /* We know the list is sorted so we use that fact in the search.
 370          Insertion of new versions is done with correct sorting */
 371       int Res = 1;
 372       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 373       {
 374          char const * const VerStr = Ver.VerStr();
 375          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 376                VerStr, VerStr + strlen(VerStr));
 377          // Version is higher as current version - insert here
 378          if (Res > 0)
 379             break;
 380          // Versionstrings are equal - is hash also equal?
 381          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 382             break;
 383          // proceed with the next till we have either the right
 384          // or we found another version (which will be lower)
 385       }
 386
 387       /* We already have a version for this item, record that we saw it */
 388       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 389       {
 390          if (List.UsePackage(Pkg,Ver) == false)
 391             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 392                                  Pkg.Name(), "UsePackage", 2);
 393
 394          if (NewFileVer(Ver,List) == false)
 395             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 396                                  Pkg.Name(), "NewFileVer", 1);
 397
 398          // Read only a single record and return
 399          if (OutVer != 0)
 400          {
 401             *OutVer = Ver;
 402             return true;
 403          }
 404
 405          return true;
 406       }
 407    }
 408
 409    // Add a new version
 410    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 411    if (unlikely(verindex == 0))
 412       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 413                            Pkg.Name(), "NewVersion", 1);
 414
 415    if (oldMap != Map.Data())
 416          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 417    *LastVer = verindex;
 418
 419    if (unlikely(List.NewVersion(Ver) == false))
 420       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 421                            Pkg.Name(), "NewVersion", 2);
 422
 423    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 424       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                            Pkg.Name(), "UsePackage", 3);
 426
 427    if (unlikely(NewFileVer(Ver,List) == false))
 428       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 429                            Pkg.Name(), "NewFileVer", 2);
 430
 431    pkgCache::GrpIterator Grp = Pkg.Group();
 432    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 433
 434    /* If it is the first version of this package we need to add implicit
 435       Multi-Arch dependencies to all other package versions in the group now -
 436       otherwise we just add them for this new version */
 437    if (Pkg.VersionList()->NextVer == 0)
 438    {
 439       pkgCache::PkgIterator P = Grp.PackageList();
 440       Dynamic<pkgCache::PkgIterator> DynP(P);
 441       for (; P.end() != true; P = Grp.NextPkg(P))
 442       {
 443          if (P->ID == Pkg->ID)
 444             continue;
 445          pkgCache::VerIterator V = P.VersionList();
 446          Dynamic<pkgCache::VerIterator> DynV(V);
 447          for (; V.end() != true; ++V)
 448             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 449                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 450                                     Pkg.Name(), "AddImplicitDepends", 1);
 451       }
 452    }
 453    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 454       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 455                            Pkg.Name(), "AddImplicitDepends", 2);
 456
 457    // Read only a single record and return
 458    if (OutVer != 0)
 459    {
 460       *OutVer = Ver;
 461       return true;
 462    }
 463
 464    /* Record the Description(s) based on their master md5sum */
 465    MD5SumValue CurMd5 = List.Description_md5();
 466
 467    /* Before we add a new description we first search in the group for
 468       a version with a description of the same MD5 - if so we reuse this
 469       description group instead of creating our own for this version */
 470    for (pkgCache::PkgIterator P = Grp.PackageList();
 471         P.end() == false; P = Grp.NextPkg(P))
 472    {
 473       for (pkgCache::VerIterator V = P.VersionList();
 474            V.end() == false; ++V)
 475       {
 476          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 477             continue;
 478          Ver->DescriptionList = V->DescriptionList;
 479       }
 480    }
 481
 482    // We haven't found reusable descriptions, so add the first description(s)
 483    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 484    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 485    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 486       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 487          return false;
 488    return true;
 489 }
 490                                                                         /*}}}*/
 491 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 492 {
 493    pkgCache::DescIterator Desc;
 494    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 495
 496    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 497    if (unlikely(descindex == 0))
 498       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 499             Ver.ParentPkg().Name(), "NewDescription", 1);
 500
 501    md5idx = Desc->md5sum;
 502    Desc->ParentPkg = Ver.ParentPkg().Index();
 503
 504    // we add at the end, so that the start is constant as we need
 505    // that to be able to efficiently share these lists
 506    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 507    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 508    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 509    *LastNextDesc = descindex;
 510
 511    if (NewFileDesc(Desc,List) == false)
 512       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 513             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 514
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518                                                                         /*}}}*/
 519 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 520 // ---------------------------------------------------------------------
 521 /* This creates a new group structure and adds it to the hash table */
 522 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 523 {
 524    Dynamic<StringView> DName(Name);
 525    Grp = Cache.FindGrp(Name);
 526    if (Grp.end() == false)
 527       return true;
 528
 529    // Get a structure
 530    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 531    if (unlikely(Group == 0))
 532       return false;
 533
 534    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 535    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 536    if (unlikely(idxName == 0))
 537       return false;
 538    Grp->Name = idxName;
 539
 540    // Insert it into the hash table
 541    unsigned long const Hash = Cache.Hash(Name);
 542    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 543
 544    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 545       insertAt = &(Cache.GrpP + *insertAt)->Next;
 546    Grp->Next = *insertAt;
 547    *insertAt = Group;
 548
 549    Grp->ID = Cache.HeaderP->GroupCount++;
 550    return true;
 551 }
 552                                                                         /*}}}*/
 553 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* This creates a new package structure and adds it to the hash table */
 556 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 557                                         StringView Arch) {
 558    pkgCache::GrpIterator Grp;
 559    Dynamic<StringView> DName(Name);
 560    Dynamic<StringView> DArch(Arch);
 561    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 562    if (unlikely(NewGroup(Grp, Name) == false))
 563       return false;
 564
 565    Pkg = Grp.FindPkg(Arch);
 566       if (Pkg.end() == false)
 567          return true;
 568
 569    // Get a structure
 570    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 571    if (unlikely(Package == 0))
 572       return false;
 573    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 574
 575    // Set the name, arch and the ID
 576    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 577    Pkg->Group = Grp.Index();
 578    // all is mapped to the native architecture
 579    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 580    if (unlikely(idxArch == 0))
 581       return false;
 582    Pkg->Arch = idxArch;
 583    Pkg->ID = Cache.HeaderP->PackageCount++;
 584
 585    // Insert the package into our package list
 586    if (Grp->FirstPackage == 0) // the group is new
 587    {
 588       Grp->FirstPackage = Package;
 589       // Insert it into the hash table
 590       map_id_t const Hash = Cache.Hash(Name);
 591       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 592       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 593          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 594       Pkg->NextPackage = *insertAt;
 595       *insertAt = Package;
 596    }
 597    else // Group the Packages together
 598    {
 599       // if sibling is provided by another package, this one is too
 600       {
 601          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 602          if (M.end() == false) {
 603             pkgCache::PrvIterator Prv;
 604             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 605             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 606             {
 607                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 608                   continue;
 609                pkgCache::VerIterator Ver = Prv.OwnerVer();
 610                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 611                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 612                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 613                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 614                {
 615                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 616                      continue;
 617                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 618                      return false;
 619                }
 620             }
 621          }
 622       }
 623       // let M-A:foreign package siblings provide this package
 624       {
 625          pkgCache::PkgIterator P;
 626          pkgCache::VerIterator Ver;
 627          Dynamic<pkgCache::PkgIterator> DynP(P);
 628          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 629
 630          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 631          {
 632             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 633                continue;
 634             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 635                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 636                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 637                      return false;
 638          }
 639       }
 640       // and negative dependencies, don't forget negative dependencies
 641       {
 642          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 643          if (M.end() == false) {
 644             pkgCache::DepIterator Dep;
 645             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 646             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 647             {
 648                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 649                   continue;
 650                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 651                      Dep->Type != pkgCache::Dep::Replaces)
 652                   continue;
 653                pkgCache::VerIterator Ver = Dep.ParentVer();
 654                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 655                map_pointer_t * unused = NULL;
 656                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 657                   return false;
 658             }
 659          }
 660       }
 661
 662       // this package is the new last package
 663       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 664       Pkg->NextPackage = LastPkg->NextPackage;
 665       LastPkg->NextPackage = Package;
 666    }
 667    Grp->LastPackage = Package;
 668
 669    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 670    if (Arch == "any")
 671    {
 672       size_t const found = Name.rfind(':');
 673       StringView ArchA = Name.substr(found + 1);
 674       if (ArchA != "any")
 675       {
 676          // ArchA is used inside the loop which might remap (NameA is not used)
 677          Dynamic<StringView> DynArchA(ArchA);
 678          StringView NameA = Name.substr(0, found);
 679          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 680          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 681          if (PkgA.end())
 682          {
 683             Dynamic<StringView> DynNameA(NameA);
 684             if (NewPackage(PkgA, NameA, ArchA) == false)
 685                return false;
 686          }
 687          if (unlikely(PkgA.end()))
 688             return _error->Fatal("NewPackage was successful for %s:%s,"
 689                   "but the package doesn't exist anyhow!",
 690                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 691          else
 692          {
 693             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 694             for (; Prv.end() == false; ++Prv)
 695             {
 696                if (Prv.IsMultiArchImplicit())
 697                   continue;
 698                pkgCache::VerIterator V = Prv.OwnerVer();
 699                if (ArchA != V.ParentPkg().Arch())
 700                   continue;
 701                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 702                   return false;
 703             }
 704             pkgCache::VerIterator V = PkgA.VersionList();
 705             Dynamic<pkgCache::VerIterator> DynV(V);
 706             for (; V.end() == false; ++V)
 707             {
 708                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 709                   return false;
 710             }
 711          }
 712       }
 713    }
 714    return true;
 715 }
 716                                                                         /*}}}*/
 717 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 718 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 719                                            pkgCache::PkgIterator &P,
 720                                            pkgCache::VerIterator &V)
 721 {
 722    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 723    Dynamic<APT::StringView> DynArch(Arch);
 724    map_pointer_t *OldDepLast = NULL;
 725    /* MultiArch handling introduces a lot of implicit Dependencies:
 726       - MultiArch: same → Co-Installable if they have the same version
 727       - All others conflict with all other group members */
 728    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 729    pkgCache::PkgIterator D = G.PackageList();
 730    Dynamic<pkgCache::PkgIterator> DynD(D);
 731    map_stringitem_t const VerStrIdx = V->VerStr;
 732    for (; D.end() != true; D = G.NextPkg(D))
 733    {
 734       if (Arch == D.Arch() || D->VersionList == 0)
 735          continue;
 736       /* We allow only one installed arch at the time
 737          per group, therefore each group member conflicts
 738          with all other group members */
 739       if (coInstall == true)
 740       {
 741          // Replaces: ${self}:other ( << ${binary:Version})
 742          NewDepends(D, V, VerStrIdx,
 743                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 744                     OldDepLast);
 745          // Breaks: ${self}:other (!= ${binary:Version})
 746          NewDepends(D, V, VerStrIdx,
 747                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 748                     OldDepLast);
 749       } else {
 750          // Conflicts: ${self}:other
 751          NewDepends(D, V, 0,
 752                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 753                     OldDepLast);
 754       }
 755    }
 756    return true;
 757 }
 758 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 759                                            pkgCache::PkgIterator &D)
 760 {
 761    /* MultiArch handling introduces a lot of implicit Dependencies:
 762       - MultiArch: same → Co-Installable if they have the same version
 763       - All others conflict with all other group members */
 764    map_pointer_t *OldDepLast = NULL;
 765    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 766    if (coInstall == true)
 767    {
 768       map_stringitem_t const VerStrIdx = V->VerStr;
 769       // Replaces: ${self}:other ( << ${binary:Version})
 770       NewDepends(D, V, VerStrIdx,
 771                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 772                  OldDepLast);
 773       // Breaks: ${self}:other (!= ${binary:Version})
 774       NewDepends(D, V, VerStrIdx,
 775                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 776                  OldDepLast);
 777    } else {
 778       // Conflicts: ${self}:other
 779       NewDepends(D, V, 0,
 780                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 781                  OldDepLast);
 782    }
 783    return true;
 784 }
 785
 786                                                                         /*}}}*/
 787 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 788 // ---------------------------------------------------------------------
 789 /* */
 790 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 791                                    ListParser &List)
 792 {
 793    if (CurrentFile == 0)
 794       return true;
 795
 796    // Get a structure
 797    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 798    if (VerFile == 0)
 799       return false;
 800
 801    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 802    VF->File = CurrentFile - Cache.PkgFileP;
 803
 804    // Link it to the end of the list
 805    map_pointer_t *Last = &Ver->FileList;
 806    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 807       Last = &V->NextFile;
 808    VF->NextFile = *Last;
 809    *Last = VF.Index();
 810
 811    VF->Offset = List.Offset();
 812    VF->Size = List.Size();
 813    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 814       Cache.HeaderP->MaxVerFileSize = VF->Size;
 815    Cache.HeaderP->VerFileCount++;
 816
 817    return true;
 818 }
 819                                                                         /*}}}*/
 820 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 821 // ---------------------------------------------------------------------
 822 /* This puts a version structure in the linked list */
 823 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 824                                             APT::StringView const &VerStr,
 825                                             map_pointer_t const ParentPkg,
 826                                             unsigned short const Hash,
 827                                             map_pointer_t const Next)
 828 {
 829    // Get a structure
 830    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 831    if (Version == 0)
 832       return 0;
 833
 834    // Fill it in
 835    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 836    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 837    Ver->NextVer = Next;
 838    Ver->ParentPkg = ParentPkg;
 839    Ver->Hash = Hash;
 840    Ver->ID = Cache.HeaderP->VersionCount++;
 841
 842    // try to find the version string in the group for reuse
 843    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 844    pkgCache::GrpIterator Grp = Pkg.Group();
 845    if (Pkg.end() == false && Grp.end() == false)
 846    {
 847       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 848       {
 849          if (Pkg == P)
 850             continue;
 851          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 852          {
 853             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 854             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 855             {
 856                Ver->VerStr = V->VerStr;
 857                return Version;
 858             }
 859             else if (cmp < 0)
 860                break;
 861          }
 862       }
 863    }
 864    // haven't found the version string, so create
 865    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 866    if (unlikely(idxVerStr == 0))
 867       return 0;
 868    Ver->VerStr = idxVerStr;
 869    return Version;
 870 }
 871                                                                         /*}}}*/
 872 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 873 // ---------------------------------------------------------------------
 874 /* */
 875 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 876                                    ListParser &List)
 877 {
 878    if (CurrentFile == 0)
 879       return true;
 880
 881    // Get a structure
 882    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 883    if (DescFile == 0)
 884       return false;
 885
 886    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 887    DF->File = CurrentFile - Cache.PkgFileP;
 888
 889    // Link it to the end of the list
 890    map_pointer_t *Last = &Desc->FileList;
 891    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 892       Last = &D->NextFile;
 893
 894    DF->NextFile = *Last;
 895    *Last = DF.Index();
 896
 897    DF->Offset = List.Offset();
 898    DF->Size = List.Size();
 899    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 900       Cache.HeaderP->MaxDescFileSize = DF->Size;
 901    Cache.HeaderP->DescFileCount++;
 902
 903    return true;
 904 }
 905                                                                         /*}}}*/
 906 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 907 // ---------------------------------------------------------------------
 908 /* This puts a description structure in the linked list */
 909 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 910                                             const string &Lang,
 911                                             const MD5SumValue &md5sum,
 912                                             map_stringitem_t const idxmd5str)
 913 {
 914    // Get a structure
 915    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 916    if (Description == 0)
 917       return 0;
 918
 919    // Fill it in
 920    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 921    Desc->ID = Cache.HeaderP->DescriptionCount++;
 922    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 923    if (unlikely(idxlanguage_code == 0))
 924       return 0;
 925    Desc->language_code = idxlanguage_code;
 926
 927    if (idxmd5str != 0)
 928       Desc->md5sum = idxmd5str;
 929    else
 930    {
 931       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 932       if (unlikely(idxmd5sum == 0))
 933          return 0;
 934       Desc->md5sum = idxmd5sum;
 935    }
 936
 937    return Description;
 938 }
 939                                                                         /*}}}*/
 940 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 941 // ---------------------------------------------------------------------
 942 /* This creates a dependency element in the tree. It is linked to the
 943    version and to the package that it is pointing to. */
 944 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 945                                    pkgCache::VerIterator &Ver,
 946                                    map_pointer_t const Version,
 947                                    uint8_t const Op,
 948                                    uint8_t const Type,
 949                                    map_pointer_t* &OldDepLast)
 950 {
 951    void const * const oldMap = Map.Data();
 952    // Get a structure
 953    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 954    if (unlikely(Dependency == 0))
 955       return false;
 956
 957    bool isDuplicate = false;
 958    map_pointer_t DependencyData = 0;
 959    map_pointer_t PreviousData = 0;
 960    if (Pkg->RevDepends != 0)
 961    {
 962       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 963       DependencyData = L->DependencyData;
 964       do {
 965          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 966          if (Version > D->Version)
 967             break;
 968          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 969          {
 970             isDuplicate = true;
 971             break;
 972          }
 973          PreviousData = DependencyData;
 974          DependencyData = D->NextData;
 975       } while (DependencyData != 0);
 976    }
 977
 978    if (isDuplicate == false)
 979    {
 980       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 981       if (unlikely(DependencyData == 0))
 982         return false;
 983    }
 984
 985    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 986    Link->ParentVer = Ver.Index();
 987    Link->DependencyData = DependencyData;
 988    Link->ID = Cache.HeaderP->DependsCount++;
 989
 990    pkgCache::DepIterator Dep(Cache, Link);
 991    if (isDuplicate == false)
 992    {
 993       Dep->Type = Type;
 994       Dep->CompareOp = Op;
 995       Dep->Version = Version;
 996       Dep->Package = Pkg.Index();
 997       ++Cache.HeaderP->DependsDataCount;
 998       if (PreviousData != 0)
 999       {
1000          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1001          Dep->NextData = D->NextData;
1002          D->NextData = DependencyData;
1003       }
1004       else if (Pkg->RevDepends != 0)
1005       {
1006          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1007          Dep->NextData = D->DependencyData;
1008       }
1009    }
1010
1011    if (isDuplicate == true || PreviousData != 0)
1012    {
1013       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1014       Link->NextRevDepends = L->NextRevDepends;
1015       L->NextRevDepends = Dependency;
1016    }
1017    else
1018    {
1019       Link->NextRevDepends = Pkg->RevDepends;
1020       Pkg->RevDepends = Dependency;
1021    }
1022
1023
1024    // Do we know where to link the Dependency to?
1025    if (OldDepLast == NULL)
1026    {
1027       OldDepLast = &Ver->DependsList;
1028       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1029          OldDepLast = &D->NextDepends;
1030    } else if (oldMap != Map.Data())
1031       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1032
1033    Dep->NextDepends = *OldDepLast;
1034    *OldDepLast = Dependency;
1035    OldDepLast = &Dep->NextDepends;
1036    return true;
1037 }
1038                                                                         /*}}}*/
1039 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1040 // ---------------------------------------------------------------------
1041 /* This creates a Group and the Package to link this dependency to if
1042    needed and handles also the caching of the old endpoint */
1043 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1044                                                StringView PackageName,
1045                                                StringView Arch,
1046                                                StringView Version,
1047                                                uint8_t const Op,
1048                                                uint8_t const Type)
1049 {
1050    pkgCache::GrpIterator Grp;
1051    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1052    Dynamic<StringView> DynPackageName(PackageName);
1053    Dynamic<StringView> DynArch(Arch);
1054    Dynamic<StringView> DynVersion(Version);
1055    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1056       return false;
1057
1058    map_stringitem_t idxVersion = 0;
1059    if (Version.empty() == false)
1060    {
1061       int const CmpOp = Op & 0x0F;
1062       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1063       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1064          idxVersion = Ver->VerStr;
1065
1066       if (idxVersion == 0)
1067       {
1068          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1069          if (unlikely(idxVersion == 0))
1070             return false;
1071       }
1072    }
1073
1074    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1075          Type == pkgCache::Dep::Conflicts ||
1076          Type == pkgCache::Dep::Replaces);
1077
1078    pkgCache::PkgIterator Pkg;
1079    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1080    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1081    {
1082       // Locate the target package
1083       Pkg = Grp.FindPkg(Arch);
1084       if (Pkg.end() == true) {
1085          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1086             return false;
1087       }
1088
1089       /* Caching the old end point speeds up generation substantially */
1090       if (OldDepVer != Ver) {
1091          OldDepLast = NULL;
1092          OldDepVer = Ver;
1093       }
1094
1095       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1096    }
1097    else
1098    {
1099       /* Caching the old end point speeds up generation substantially */
1100       if (OldDepVer != Ver) {
1101          OldDepLast = NULL;
1102          OldDepVer = Ver;
1103       }
1104
1105       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1106       {
1107          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1108                return false;
1109       }
1110    }
1111    return true;
1112 }
1113                                                                         /*}}}*/
1114 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1115 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1116                                                 StringView PkgName,
1117                                                 StringView PkgArch,
1118                                                 StringView Version,
1119                                                 uint8_t const Flags)
1120 {
1121    pkgCache const &Cache = Owner->Cache;
1122    Dynamic<StringView> DynPkgName(PkgName);
1123    Dynamic<StringView> DynArch(PkgArch);
1124    Dynamic<StringView> DynVersion(Version);
1125
1126    // We do not add self referencing provides
1127    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1128         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1129          (Version.empty() || Version == Ver.VerStr()))
1130       return true;
1131
1132    // Locate the target package
1133    pkgCache::PkgIterator Pkg;
1134    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1135    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1136       return false;
1137
1138    map_stringitem_t idxProvideVersion = 0;
1139    if (Version.empty() == false) {
1140       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1141       if (unlikely(idxProvideVersion == 0))
1142          return false;
1143    }
1144    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1145 }
1146 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1147                                     pkgCache::PkgIterator &Pkg,
1148                                     map_pointer_t const ProvideVersion,
1149                                     uint8_t const Flags)
1150 {
1151    // Get a structure
1152    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1153    if (unlikely(Provides == 0))
1154       return false;
1155    ++Cache.HeaderP->ProvidesCount;
1156
1157    // Fill it in
1158    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1159    Prv->Version = Ver.Index();
1160    Prv->ProvideVersion = ProvideVersion;
1161    Prv->Flags = Flags;
1162    Prv->NextPkgProv = Ver->ProvidesList;
1163    Ver->ProvidesList = Prv.Index();
1164
1165    // Link it to the package
1166    Prv->ParentPkg = Pkg.Index();
1167    Prv->NextProvides = Pkg->ProvidesList;
1168    Pkg->ProvidesList = Prv.Index();
1169    return true;
1170 }
1171                                                                         /*}}}*/
1172 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1173 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1174                                 StringView Version, uint8_t const Flags) {
1175    pkgCache &Cache = Owner->Cache;
1176    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1177    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1178    Dynamic<StringView> DynPackage(Package);
1179    Dynamic<StringView> DynVersion(Version);
1180
1181    if (Grp.end() == true)
1182       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1183    else
1184    {
1185       map_stringitem_t idxProvideVersion = 0;
1186       if (Version.empty() == false) {
1187          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1188          if (unlikely(idxProvideVersion == 0))
1189             return false;
1190       }
1191
1192       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1193       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1194       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1195       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1196       pkgCache::PkgIterator Pkg;
1197       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1198       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1199       {
1200          if (isImplicit && OwnerPkg == Pkg)
1201             continue;
1202          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1203             continue;
1204          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1205             return false;
1206       }
1207    }
1208    return true;
1209 }
1210                                                                         /*}}}*/
1211 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1212       pkgCache::VerIterator const &Ver)
1213 {
1214    return Hash == Ver->Hash;
1215 }
1216                                                                         /*}}}*/
1217 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1218 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1219                                    unsigned long Flags)
1220 {
1221    if (File.empty() && Site.empty())
1222    {
1223       CurrentRlsFile = NULL;
1224       return true;
1225    }
1226
1227    // Get some space for the structure
1228    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1229    if (unlikely(idxFile == 0))
1230       return false;
1231    CurrentRlsFile = Cache.RlsFileP + idxFile;
1232
1233    // Fill it in
1234    map_stringitem_t const idxFileName = WriteStringInMap(File);
1235    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1236    if (unlikely(idxFileName == 0 || idxSite == 0))
1237       return false;
1238    CurrentRlsFile->FileName = idxFileName;
1239    CurrentRlsFile->Site = idxSite;
1240    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1241    CurrentRlsFile->Flags = Flags;
1242    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1243    RlsFileName = File;
1244    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1245    Cache.HeaderP->ReleaseFileCount++;
1246
1247    return true;
1248 }
1249                                                                         /*}}}*/
1250 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1251 // ---------------------------------------------------------------------
1252 /* This is used to select which file is to be associated with all newly
1253    added versions. The caller is responsible for setting the IMS fields. */
1254 bool pkgCacheGenerator::SelectFile(std::string const &File,
1255                                    pkgIndexFile const &Index,
1256                                    std::string const &Architecture,
1257                                    std::string const &Component,
1258                                    unsigned long const Flags)
1259 {
1260    // Get some space for the structure
1261    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1262    if (unlikely(idxFile == 0))
1263       return false;
1264    CurrentFile = Cache.PkgFileP + idxFile;
1265
1266    // Fill it in
1267    map_stringitem_t const idxFileName = WriteStringInMap(File);
1268    if (unlikely(idxFileName == 0))
1269       return false;
1270    CurrentFile->FileName = idxFileName;
1271    CurrentFile->NextFile = Cache.HeaderP->FileList;
1272    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1273    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1274    if (unlikely(idxIndexType == 0))
1275       return false;
1276    CurrentFile->IndexType = idxIndexType;
1277    if (Architecture.empty())
1278       CurrentFile->Architecture = 0;
1279    else
1280    {
1281       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1282       if (unlikely(arch == 0))
1283          return false;
1284       CurrentFile->Architecture = arch;
1285    }
1286    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1287    if (unlikely(component == 0))
1288       return false;
1289    CurrentFile->Component = component;
1290    CurrentFile->Flags = Flags;
1291    if (CurrentRlsFile != NULL)
1292       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1293    else
1294       CurrentFile->Release = 0;
1295    PkgFileName = File;
1296    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1297    Cache.HeaderP->PackageFileCount++;
1298
1299    if (Progress != 0)
1300       Progress->SubProgress(Index.Size());
1301    return true;
1302 }
1303                                                                         /*}}}*/
1304 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1305 // ---------------------------------------------------------------------
1306 /* This is used to create handles to strings. Given the same text it
1307    always returns the same number */
1308 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1309                                                  unsigned int Size)
1310 {
1311    auto strings = &strMixed;
1312    switch(type) {
1313       case MIXED: strings = &strMixed; break;
1314       case PKGNAME: strings = &strPkgNames; break;
1315       case VERSIONNUMBER: strings = &strVersions; break;
1316       case SECTION: strings = &strSections; break;
1317       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1318    }
1319
1320    auto const item = strings->find({S, Size, nullptr, 0});
1321    if (item != strings->end())
1322       return item->item;
1323
1324    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1325    strings->insert({nullptr, Size, this, idxString});
1326    return idxString;
1327 }
1328                                                                         /*}}}*/
1329 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1330 // ---------------------------------------------------------------------
1331 /* This just verifies that each file in the list of index files exists,
1332    has matching attributes with the cache and the cache does not have
1333    any extra files. */
1334 class APT_HIDDEN ScopedErrorRevert {
1335 public:
1336    ScopedErrorRevert() { _error->PushToStack(); }
1337    ~ScopedErrorRevert() { _error->RevertToStack(); }
1338 };
1339 static bool CheckValidity(const string &CacheFile,
1340                           pkgSourceList &List,
1341                           FileIterator const Start,
1342                           FileIterator const End,
1343                           MMap **OutMap = 0,
1344                           pkgCache **OutCache = 0)
1345 {
1346    ScopedErrorRevert ser;
1347    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1348    // No file, certainly invalid
1349    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1350    {
1351       if (Debug == true)
1352          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1353       return false;
1354    }
1355
1356    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1357    {
1358       if (Debug == true)
1359          std::clog << "sources.list is newer than the cache" << std::endl;
1360       return false;
1361    }
1362
1363    // Map it
1364    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1365    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1366    if (unlikely(Map->validData()) == false)
1367       return false;
1368    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1369    pkgCache &Cache = *CacheP.get();
1370    if (_error->PendingError() || Map->Size() == 0)
1371    {
1372       if (Debug == true)
1373          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1374       return false;
1375    }
1376
1377    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1378    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1379    std::vector<pkgIndexFile *> Files;
1380    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1381    {
1382       if (Debug == true)
1383          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1384       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1385       if (RlsFile.end() == true)
1386       {
1387          if (Debug == true)
1388             std::clog << "FindInCache returned end-Pointer" << std::endl;
1389          return false;
1390       }
1391
1392       RlsVisited[RlsFile->ID] = true;
1393       if (Debug == true)
1394          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1395
1396       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1397       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1398             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1399    }
1400    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1401       if (RlsVisited[I] == false)
1402       {
1403          if (Debug == true)
1404             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1405          return false;
1406       }
1407
1408    std::copy(Start, End, std::back_inserter(Files));
1409
1410    /* Now we check every index file, see if it is in the cache,
1411       verify the IMS data and check that it is on the disk too.. */
1412    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1413    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1414    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1415    {
1416       if (Debug == true)
1417          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1418       if ((*PkgFile)->Exists() == false)
1419       {
1420          if (Debug == true)
1421             std::clog << "file doesn't exist" << std::endl;
1422          continue;
1423       }
1424
1425       // FindInCache is also expected to do an IMS check.
1426       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1427       if (File.end() == true)
1428       {
1429          if (Debug == true)
1430             std::clog << "FindInCache returned end-Pointer" << std::endl;
1431          return false;
1432       }
1433
1434       Visited[File->ID] = true;
1435       if (Debug == true)
1436          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1437    }
1438
1439    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1440       if (Visited[I] == false)
1441       {
1442          if (Debug == true)
1443             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1444          return false;
1445       }
1446
1447    if (_error->PendingError() == true)
1448    {
1449       if (Debug == true)
1450       {
1451          std::clog << "Validity failed because of pending errors:" << std::endl;
1452          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1453       }
1454       return false;
1455    }
1456
1457    if (OutMap != 0)
1458       *OutMap = Map.release();
1459    if (OutCache != 0)
1460       *OutCache = CacheP.release();
1461    return true;
1462 }
1463                                                                         /*}}}*/
1464 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1465 // ---------------------------------------------------------------------
1466 /* Size is kind of an abstract notion that is only used for the progress
1467    meter */
1468 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1469 {
1470    map_filesize_t TotalSize = 0;
1471    if (List !=  NULL)
1472    {
1473       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1474       {
1475          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1476          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1477             if ((*j)->HasPackages() == true)
1478                TotalSize += (*j)->Size();
1479       }
1480    }
1481
1482    for (; Start < End; ++Start)
1483    {
1484       if ((*Start)->HasPackages() == false)
1485          continue;
1486       TotalSize += (*Start)->Size();
1487    }
1488    return TotalSize;
1489 }
1490                                                                         /*}}}*/
1491 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1492 static bool BuildCache(pkgCacheGenerator &Gen,
1493                        OpProgress * const Progress,
1494                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1495                        pkgSourceList const * const List,
1496                        FileIterator const Start, FileIterator const End)
1497 {
1498    bool mergeFailure = false;
1499
1500    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1501       if (I->HasPackages() == false || mergeFailure)
1502          return;
1503
1504       if (I->Exists() == false)
1505          return;
1506
1507       if (I->FindInCache(Gen.GetCache()).end() == false)
1508       {
1509          _error->Warning("Duplicate sources.list entry %s",
1510                I->Describe().c_str());
1511          return;
1512       }
1513
1514       map_filesize_t const Size = I->Size();
1515       if (Progress != NULL)
1516          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1517       CurrentSize += Size;
1518
1519       if (I->Merge(Gen,Progress) == false)
1520          mergeFailure = true;
1521    };
1522
1523    if (List !=  NULL)
1524    {
1525       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1526       {
1527          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1528          {
1529             _error->Warning("Duplicate sources.list entry %s",
1530                   (*i)->Describe().c_str());
1531             continue;
1532          }
1533
1534          if ((*i)->Merge(Gen, Progress) == false)
1535             return false;
1536
1537          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1538          if (Indexes != NULL)
1539             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1540          if (mergeFailure)
1541             return false;
1542       }
1543    }
1544
1545    if (Start != End)
1546    {
1547       Gen.SelectReleaseFile("", "");
1548       std::for_each(Start, End, indexFileMerge);
1549       if (mergeFailure)
1550          return false;
1551    }
1552    return true;
1553 }
1554                                                                         /*}}}*/
1555 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1556 // ---------------------------------------------------------------------
1557 /* This makes sure that the status cache (the cache that has all
1558    index files from the sources list and all local ones) is ready
1559    to be mmaped. If OutMap is not zero then a MMap object representing
1560    the cache will be stored there. This is pretty much mandetory if you
1561    are using AllowMem. AllowMem lets the function be run as non-root
1562    where it builds the cache 'fast' into a memory buffer. */
1563 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1564 {
1565    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1566    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1567    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1568    Flags |= MMap::Moveable;
1569    if (_config->FindB("APT::Cache-Fallback", false) == true)
1570       Flags |= MMap::Fallback;
1571    if (CacheF != NULL)
1572       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1573    else
1574       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1575 }
1576 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1577       std::string const &FileName)
1578 {
1579    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1580    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1581       return false;
1582
1583    fchmod(SCacheF.Fd(),0644);
1584
1585    // Write out the main data
1586    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1587       return _error->Error(_("IO Error saving source cache"));
1588
1589    // Write out the proper header
1590    Gen->GetCache().HeaderP->Dirty = false;
1591    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1592    if (SCacheF.Seek(0) == false ||
1593          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1594       return _error->Error(_("IO Error saving source cache"));
1595    Gen->GetCache().HeaderP->Dirty = true;
1596    return true;
1597 }
1598 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1599       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1600 {
1601    Map.reset(CreateDynamicMMap(NULL, 0));
1602    if (unlikely(Map->validData()) == false)
1603       return false;
1604    FileFd CacheF(FileName, FileFd::ReadOnly);
1605    if (CacheF.IsOpen() == false || CacheF.Failed())
1606       return false;
1607    _error->PushToStack();
1608    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1609    bool const newError = _error->PendingError();
1610    _error->MergeWithStack();
1611    if (alloc == 0 && newError)
1612       return false;
1613    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1614       return false;
1615    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1616    return Gen->Start();
1617 }
1618 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1619                         MMap **OutMap, bool AllowMem)
1620    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1621 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1622                         MMap **OutMap,bool)
1623 {
1624    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1625 }
1626 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1627                         MMap **OutMap,pkgCache **OutCache, bool)
1628 {
1629    // FIXME: deprecate the ignored AllowMem parameter
1630    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1631
1632    std::vector<pkgIndexFile *> Files;
1633    if (_system->AddStatusFiles(Files) == false)
1634       return false;
1635
1636    // Decide if we can write to the files..
1637    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1638    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1639
1640    // ensure the cache directory exists
1641    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1642    {
1643       string dir = _config->FindDir("Dir::Cache");
1644       size_t const len = dir.size();
1645       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1646          dir = dir.substr(0, len - 5);
1647       if (CacheFile.empty() == false)
1648          CreateDirectory(dir, flNotFile(CacheFile));
1649       if (SrcCacheFile.empty() == false)
1650          CreateDirectory(dir, flNotFile(SrcCacheFile));
1651    }
1652
1653    if (Progress != NULL)
1654       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1655
1656    bool pkgcache_fine = false;
1657    bool srcpkgcache_fine = false;
1658    bool volatile_fine = List.GetVolatileFiles().empty();
1659
1660    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1661                      volatile_fine ? OutCache : NULL) == true)
1662    {
1663       if (Debug == true)
1664          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1665       pkgcache_fine = true;
1666       srcpkgcache_fine = true;
1667    }
1668    if (pkgcache_fine == false)
1669    {
1670       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1671       {
1672          if (Debug == true)
1673             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1674          srcpkgcache_fine = true;
1675       }
1676    }
1677
1678    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1679    {
1680       if (Progress != NULL)
1681          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1682       return true;
1683    }
1684
1685    bool Writeable = false;
1686    if (srcpkgcache_fine == false || pkgcache_fine == false)
1687    {
1688       if (CacheFile.empty() == false)
1689          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1690       else if (SrcCacheFile.empty() == false)
1691          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1692
1693       if (Debug == true)
1694          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1695    }
1696
1697    // At this point we know we need to construct something, so get storage ready
1698    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1699    if (unlikely(Map->validData()) == false)
1700       return false;
1701    if (Debug == true)
1702       std::clog << "Open memory Map (not filebased)" << std::endl;
1703
1704    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1705    map_filesize_t CurrentSize = 0;
1706    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1707    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1708    if (srcpkgcache_fine == true && pkgcache_fine == false)
1709    {
1710       if (Debug == true)
1711          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1712       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1713          return false;
1714       srcpkgcache_fine = true;
1715       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1716    }
1717    else if (srcpkgcache_fine == false)
1718    {
1719       if (Debug == true)
1720          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1721       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1722       if (Gen->Start() == false)
1723          return false;
1724
1725       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1726       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1727                Files.end(),Files.end()) == false)
1728          return false;
1729
1730       if (Writeable == true && SrcCacheFile.empty() == false)
1731          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1732             return false;
1733    }
1734
1735    if (pkgcache_fine == false)
1736    {
1737       if (Debug == true)
1738          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1739       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1740                Files.begin(), Files.end()) == false)
1741          return false;
1742
1743       if (Writeable == true && CacheFile.empty() == false)
1744          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1745             return false;
1746    }
1747
1748    if (Debug == true)
1749       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1750
1751    if (volatile_fine == false)
1752    {
1753       if (Gen == nullptr)
1754       {
1755          if (Debug == true)
1756             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1757          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1758             return false;
1759       }
1760
1761       Files = List.GetVolatileFiles();
1762       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1763                Files.begin(), Files.end()) == false)
1764          return false;
1765    }
1766
1767    if (OutMap != nullptr)
1768       *OutMap = Map.release();
1769
1770    if (Debug == true)
1771       std::clog << "Everything is ready for shipping" << std::endl;
1772    return true;
1773 }
1774                                                                         /*}}}*/
1775 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1776 class APT_HIDDEN ScopedErrorMerge {
1777 public:
1778    ScopedErrorMerge() { _error->PushToStack(); }
1779    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1780 };
1781 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1782    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1783 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1784 {
1785    std::vector<pkgIndexFile *> Files;
1786    if (_system->AddStatusFiles(Files) == false)
1787       return false;
1788
1789    ScopedErrorMerge sem;
1790    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1791    if (unlikely(Map->validData()) == false)
1792       return false;
1793    map_filesize_t CurrentSize = 0;
1794    map_filesize_t TotalSize = 0;
1795    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1796
1797    // Build the status cache
1798    if (Progress != NULL)
1799       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1800    pkgCacheGenerator Gen(Map.get(),Progress);
1801    if (Gen.Start() == false || _error->PendingError() == true)
1802       return false;
1803    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1804                   Files.begin(), Files.end()) == false)
1805       return false;
1806
1807    if (_error->PendingError() == true)
1808       return false;
1809    *OutMap = Map.release();
1810
1811    return true;
1812 }
1813                                                                         /*}}}*/
1814 // IsDuplicateDescription                                               /*{{{*/
1815 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1816                             MD5SumValue const &CurMd5, std::string const &CurLang)
1817 {
1818    // Descriptions in the same link-list have all the same md5
1819    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1820       return false;
1821    for (; Desc.end() == false; ++Desc)
1822       if (Desc.LanguageCode() == CurLang)
1823          return true;
1824    return false;
1825 }
1826                                                                         /*}}}*/
1827
1828 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1829 pkgCacheListParser::~pkgCacheListParser() {}