apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 181       // Ignore views outside of the cache.
 182       if (ViewP->data() < static_cast<const char*>(oldMap)
 183          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 184          continue;
 185       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 186       *ViewP = StringView(data , ViewP->size());
 187    }
 188 }                                                                       /*}}}*/
 189 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 190 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 191                                         const unsigned long &Len) {
 192    size_t oldSize = Map.Size();
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String, Len);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data(), oldSize);
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 201 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 202    size_t oldSize = Map.Size();
 203    void const * const oldMap = Map.Data();
 204    map_stringitem_t const index = Map.WriteString(String);
 205    if (index != 0)
 206       ReMap(oldMap, Map.Data(), oldSize);
 207    return index;
 208 }
 209                                                                         /*}}}*/
 210 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 211    size_t oldSize = Map.Size();
 212    void const * const oldMap = Map.Data();
 213    map_pointer_t const index = Map.Allocate(size);
 214    if (index != 0)
 215       ReMap(oldMap, Map.Data(), oldSize);
 216    return index;
 217 }
 218                                                                         /*}}}*/
 219 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 220 // ---------------------------------------------------------------------
 221 /* This provides the generation of the entries in the cache. Each loop
 222    goes through a single package record from the underlying parse engine. */
 223 bool pkgCacheGenerator::MergeList(ListParser &List,
 224                                   pkgCache::VerIterator *OutVer)
 225 {
 226    List.Owner = this;
 227
 228    unsigned int Counter = 0;
 229    while (List.Step() == true)
 230    {
 231       string const PackageName = List.Package();
 232       if (PackageName.empty() == true)
 233          return false;
 234
 235       Counter++;
 236       if (Counter % 100 == 0 && Progress != 0)
 237          Progress->Progress(List.Offset());
 238
 239       APT::StringView Arch = List.Architecture();
 240       Dynamic<APT::StringView> DynArch(Arch);
 241       APT::StringView Version = List.Version();
 242       Dynamic<APT::StringView> DynVersion(Version);
 243       if (Version.empty() == true && Arch.empty() == true)
 244       {
 245          // package descriptions
 246          if (MergeListGroup(List, PackageName) == false)
 247             return false;
 248          continue;
 249       }
 250
 251       // Get a pointer to the package structure
 252       pkgCache::PkgIterator Pkg;
 253       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 254       if (NewPackage(Pkg, PackageName, Arch) == false)
 255          // TRANSLATOR: The first placeholder is a package name,
 256          // the other two should be copied verbatim as they include debug info
 257          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 258                               PackageName.c_str(), "NewPackage", 1);
 259
 260
 261       if (Version.empty() == true)
 262       {
 263          if (MergeListPackage(List, Pkg) == false)
 264             return false;
 265       }
 266       else
 267       {
 268          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 269             return false;
 270       }
 271
 272       if (OutVer != 0)
 273          return true;
 274    }
 275
 276    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 277       return _error->Error(_("Wow, you exceeded the number of package "
 278                              "names this APT is capable of."));
 279    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 280       return _error->Error(_("Wow, you exceeded the number of versions "
 281                              "this APT is capable of."));
 282    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 283       return _error->Error(_("Wow, you exceeded the number of descriptions "
 284                              "this APT is capable of."));
 285    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 286       return _error->Error(_("Wow, you exceeded the number of dependencies "
 287                              "this APT is capable of."));
 288
 289    return true;
 290 }
 291 // CacheGenerator::MergeListGroup                                       /*{{{*/
 292 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 293 {
 294    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 295    // a group has no data on it's own, only packages have it but these
 296    // stanzas like this come from Translation- files to add descriptions,
 297    // but without a version we don't need a description for it…
 298    if (Grp.end() == true)
 299       return true;
 300    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 301
 302    pkgCache::PkgIterator Pkg;
 303    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 304    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 305       if (MergeListPackage(List, Pkg) == false)
 306          return false;
 307
 308    return true;
 309 }
 310                                                                         /*}}}*/
 311 // CacheGenerator::MergeListPackage                                     /*{{{*/
 312 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 313 {
 314    // we first process the package, then the descriptions
 315    // (for deb this package processing is in fact a no-op)
 316    pkgCache::VerIterator Ver(Cache);
 317    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 318    if (List.UsePackage(Pkg, Ver) == false)
 319       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 320                            Pkg.Name(), "UsePackage", 1);
 321
 322    // Find the right version to write the description
 323    MD5SumValue CurMd5 = List.Description_md5();
 324    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 325    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 326    {
 327       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 328
 329       // a version can only have one md5 describing it
 330       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 331          continue;
 332
 333       map_stringitem_t md5idx = VerDesc->md5sum;
 334       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 335       {
 336          // don't add a new description if we have one for the given
 337          // md5 && language
 338          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 339             continue;
 340
 341          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 342       }
 343
 344       // we can stop here as all "same" versions will share the description
 345       break;
 346    }
 347
 348    return true;
 349 }
 350                                                                         /*}}}*/
 351 // CacheGenerator::MergeListVersion                                     /*{{{*/
 352 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 353                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 354 {
 355    pkgCache::VerIterator Ver = Pkg.VersionList();
 356    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 357    map_pointer_t *LastVer = &Pkg->VersionList;
 358    void const * oldMap = Map.Data();
 359
 360    unsigned short const Hash = List.VersionHash();
 361    if (Ver.end() == false)
 362    {
 363       /* We know the list is sorted so we use that fact in the search.
 364          Insertion of new versions is done with correct sorting */
 365       int Res = 1;
 366       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 367       {
 368          char const * const VerStr = Ver.VerStr();
 369          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 370                VerStr, VerStr + strlen(VerStr));
 371          // Version is higher as current version - insert here
 372          if (Res > 0)
 373             break;
 374          // Versionstrings are equal - is hash also equal?
 375          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 376             break;
 377          // proceed with the next till we have either the right
 378          // or we found another version (which will be lower)
 379       }
 380
 381       /* We already have a version for this item, record that we saw it */
 382       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 383       {
 384          if (List.UsePackage(Pkg,Ver) == false)
 385             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 386                                  Pkg.Name(), "UsePackage", 2);
 387
 388          if (NewFileVer(Ver,List) == false)
 389             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 390                                  Pkg.Name(), "NewFileVer", 1);
 391
 392          // Read only a single record and return
 393          if (OutVer != 0)
 394          {
 395             *OutVer = Ver;
 396             return true;
 397          }
 398
 399          return true;
 400       }
 401    }
 402
 403    // Add a new version
 404    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 405    if (unlikely(verindex == 0))
 406       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 407                            Pkg.Name(), "NewVersion", 1);
 408
 409    if (oldMap != Map.Data())
 410          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 411    *LastVer = verindex;
 412
 413    if (unlikely(List.NewVersion(Ver) == false))
 414       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 415                            Pkg.Name(), "NewVersion", 2);
 416
 417    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "UsePackage", 3);
 420
 421    if (unlikely(NewFileVer(Ver,List) == false))
 422       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 423                            Pkg.Name(), "NewFileVer", 2);
 424
 425    pkgCache::GrpIterator Grp = Pkg.Group();
 426    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 427
 428    /* If it is the first version of this package we need to add implicit
 429       Multi-Arch dependencies to all other package versions in the group now -
 430       otherwise we just add them for this new version */
 431    if (Pkg.VersionList()->NextVer == 0)
 432    {
 433       pkgCache::PkgIterator P = Grp.PackageList();
 434       Dynamic<pkgCache::PkgIterator> DynP(P);
 435       for (; P.end() != true; P = Grp.NextPkg(P))
 436       {
 437          if (P->ID == Pkg->ID)
 438             continue;
 439          pkgCache::VerIterator V = P.VersionList();
 440          Dynamic<pkgCache::VerIterator> DynV(V);
 441          for (; V.end() != true; ++V)
 442             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 443                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 444                                     Pkg.Name(), "AddImplicitDepends", 1);
 445       }
 446    }
 447    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 448       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 449                            Pkg.Name(), "AddImplicitDepends", 2);
 450
 451    // Read only a single record and return
 452    if (OutVer != 0)
 453    {
 454       *OutVer = Ver;
 455       return true;
 456    }
 457
 458    /* Record the Description(s) based on their master md5sum */
 459    MD5SumValue CurMd5 = List.Description_md5();
 460
 461    /* Before we add a new description we first search in the group for
 462       a version with a description of the same MD5 - if so we reuse this
 463       description group instead of creating our own for this version */
 464    for (pkgCache::PkgIterator P = Grp.PackageList();
 465         P.end() == false; P = Grp.NextPkg(P))
 466    {
 467       for (pkgCache::VerIterator V = P.VersionList();
 468            V.end() == false; ++V)
 469       {
 470          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 471             continue;
 472          Ver->DescriptionList = V->DescriptionList;
 473       }
 474    }
 475
 476    // We haven't found reusable descriptions, so add the first description(s)
 477    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 478    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 479    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 480       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 481          return false;
 482    return true;
 483 }
 484                                                                         /*}}}*/
 485 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 486 {
 487    pkgCache::DescIterator Desc;
 488    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 489
 490    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 491    if (unlikely(descindex == 0))
 492       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 493             Ver.ParentPkg().Name(), "NewDescription", 1);
 494
 495    md5idx = Desc->md5sum;
 496    Desc->ParentPkg = Ver.ParentPkg().Index();
 497
 498    // we add at the end, so that the start is constant as we need
 499    // that to be able to efficiently share these lists
 500    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 501    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 502    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 503    *LastNextDesc = descindex;
 504
 505    if (NewFileDesc(Desc,List) == false)
 506       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 507             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 508
 509    return true;
 510 }
 511                                                                         /*}}}*/
 512                                                                         /*}}}*/
 513 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 514 // ---------------------------------------------------------------------
 515 /* This creates a new group structure and adds it to the hash table */
 516 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 517 {
 518    Dynamic<StringView> DName(Name);
 519    Grp = Cache.FindGrp(Name);
 520    if (Grp.end() == false)
 521       return true;
 522
 523    // Get a structure
 524    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 525    if (unlikely(Group == 0))
 526       return false;
 527
 528    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 529    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 530    if (unlikely(idxName == 0))
 531       return false;
 532    Grp->Name = idxName;
 533
 534    // Insert it into the hash table
 535    unsigned long const Hash = Cache.Hash(Name);
 536    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 537
 538    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 539       insertAt = &(Cache.GrpP + *insertAt)->Next;
 540    Grp->Next = *insertAt;
 541    *insertAt = Group;
 542
 543    Grp->ID = Cache.HeaderP->GroupCount++;
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 548 // ---------------------------------------------------------------------
 549 /* This creates a new package structure and adds it to the hash table */
 550 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 551                                         StringView Arch) {
 552    pkgCache::GrpIterator Grp;
 553    Dynamic<StringView> DName(Name);
 554    Dynamic<StringView> DArch(Arch);
 555    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 556    if (unlikely(NewGroup(Grp, Name) == false))
 557       return false;
 558
 559    Pkg = Grp.FindPkg(Arch);
 560       if (Pkg.end() == false)
 561          return true;
 562
 563    // Get a structure
 564    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 565    if (unlikely(Package == 0))
 566       return false;
 567    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 568
 569    // Set the name, arch and the ID
 570    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 571    Pkg->Group = Grp.Index();
 572    // all is mapped to the native architecture
 573    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 574    if (unlikely(idxArch == 0))
 575       return false;
 576    Pkg->Arch = idxArch;
 577    Pkg->ID = Cache.HeaderP->PackageCount++;
 578
 579    // Insert the package into our package list
 580    if (Grp->FirstPackage == 0) // the group is new
 581    {
 582       Grp->FirstPackage = Package;
 583       // Insert it into the hash table
 584       map_id_t const Hash = Cache.Hash(Name);
 585       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 586       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 587          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 588       Pkg->NextPackage = *insertAt;
 589       *insertAt = Package;
 590    }
 591    else // Group the Packages together
 592    {
 593       // if sibling is provided by another package, this one is too
 594       {
 595          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 596          if (M.end() == false) {
 597             pkgCache::PrvIterator Prv;
 598             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 599             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 600             {
 601                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 602                   continue;
 603                pkgCache::VerIterator Ver = Prv.OwnerVer();
 604                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 605                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 606                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 607                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 608                {
 609                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 610                      continue;
 611                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 612                      return false;
 613                }
 614             }
 615          }
 616       }
 617       // let M-A:foreign package siblings provide this package
 618       {
 619          pkgCache::PkgIterator P;
 620          pkgCache::VerIterator Ver;
 621          Dynamic<pkgCache::PkgIterator> DynP(P);
 622          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 623
 624          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 625          {
 626             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 627                continue;
 628             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 629                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 630                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 631                      return false;
 632          }
 633       }
 634       // and negative dependencies, don't forget negative dependencies
 635       {
 636          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 637          if (M.end() == false) {
 638             pkgCache::DepIterator Dep;
 639             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 640             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 641             {
 642                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 643                   continue;
 644                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 645                      Dep->Type != pkgCache::Dep::Replaces)
 646                   continue;
 647                pkgCache::VerIterator Ver = Dep.ParentVer();
 648                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 649                map_pointer_t * unused = NULL;
 650                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 651                   return false;
 652             }
 653          }
 654       }
 655
 656       // this package is the new last package
 657       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 658       Pkg->NextPackage = LastPkg->NextPackage;
 659       LastPkg->NextPackage = Package;
 660    }
 661    Grp->LastPackage = Package;
 662
 663    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 664    if (Arch == "any")
 665    {
 666       size_t const found = Name.rfind(':');
 667       StringView ArchA = Name.substr(found + 1);
 668       if (ArchA != "any")
 669       {
 670          // ArchA is used inside the loop which might remap (NameA is not used)
 671          Dynamic<StringView> DynArchA(ArchA);
 672          StringView NameA = Name.substr(0, found);
 673          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 674          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 675          if (PkgA.end())
 676          {
 677             Dynamic<StringView> DynNameA(NameA);
 678             if (NewPackage(PkgA, NameA, ArchA) == false)
 679                return false;
 680          }
 681          if (unlikely(PkgA.end()))
 682             return _error->Fatal("NewPackage was successful for %s:%s,"
 683                   "but the package doesn't exist anyhow!",
 684                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 685          else
 686          {
 687             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 688             for (; Prv.end() == false; ++Prv)
 689             {
 690                if (Prv.IsMultiArchImplicit())
 691                   continue;
 692                pkgCache::VerIterator V = Prv.OwnerVer();
 693                if (ArchA != V.ParentPkg().Arch())
 694                   continue;
 695                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 696                   return false;
 697             }
 698             pkgCache::VerIterator V = PkgA.VersionList();
 699             Dynamic<pkgCache::VerIterator> DynV(V);
 700             for (; V.end() == false; ++V)
 701             {
 702                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 703                   return false;
 704             }
 705          }
 706       }
 707    }
 708    return true;
 709 }
 710                                                                         /*}}}*/
 711 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 712 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 713                                            pkgCache::PkgIterator &P,
 714                                            pkgCache::VerIterator &V)
 715 {
 716    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 717    Dynamic<APT::StringView> DynArch(Arch);
 718    map_pointer_t *OldDepLast = NULL;
 719    /* MultiArch handling introduces a lot of implicit Dependencies:
 720       - MultiArch: same → Co-Installable if they have the same version
 721       - All others conflict with all other group members */
 722    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 723    pkgCache::PkgIterator D = G.PackageList();
 724    Dynamic<pkgCache::PkgIterator> DynD(D);
 725    map_stringitem_t const VerStrIdx = V->VerStr;
 726    for (; D.end() != true; D = G.NextPkg(D))
 727    {
 728       if (Arch == D.Arch() || D->VersionList == 0)
 729          continue;
 730       /* We allow only one installed arch at the time
 731          per group, therefore each group member conflicts
 732          with all other group members */
 733       if (coInstall == true)
 734       {
 735          // Replaces: ${self}:other ( << ${binary:Version})
 736          NewDepends(D, V, VerStrIdx,
 737                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 738                     OldDepLast);
 739          // Breaks: ${self}:other (!= ${binary:Version})
 740          NewDepends(D, V, VerStrIdx,
 741                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 742                     OldDepLast);
 743       } else {
 744          // Conflicts: ${self}:other
 745          NewDepends(D, V, 0,
 746                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 747                     OldDepLast);
 748       }
 749    }
 750    return true;
 751 }
 752 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 753                                            pkgCache::PkgIterator &D)
 754 {
 755    /* MultiArch handling introduces a lot of implicit Dependencies:
 756       - MultiArch: same → Co-Installable if they have the same version
 757       - All others conflict with all other group members */
 758    map_pointer_t *OldDepLast = NULL;
 759    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 760    if (coInstall == true)
 761    {
 762       map_stringitem_t const VerStrIdx = V->VerStr;
 763       // Replaces: ${self}:other ( << ${binary:Version})
 764       NewDepends(D, V, VerStrIdx,
 765                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 766                  OldDepLast);
 767       // Breaks: ${self}:other (!= ${binary:Version})
 768       NewDepends(D, V, VerStrIdx,
 769                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 770                  OldDepLast);
 771    } else {
 772       // Conflicts: ${self}:other
 773       NewDepends(D, V, 0,
 774                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 775                  OldDepLast);
 776    }
 777    return true;
 778 }
 779
 780                                                                         /*}}}*/
 781 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 782 // ---------------------------------------------------------------------
 783 /* */
 784 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 785                                    ListParser &List)
 786 {
 787    if (CurrentFile == 0)
 788       return true;
 789
 790    // Get a structure
 791    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 792    if (VerFile == 0)
 793       return false;
 794
 795    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 796    VF->File = CurrentFile - Cache.PkgFileP;
 797
 798    // Link it to the end of the list
 799    map_pointer_t *Last = &Ver->FileList;
 800    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 801       Last = &V->NextFile;
 802    VF->NextFile = *Last;
 803    *Last = VF.Index();
 804
 805    VF->Offset = List.Offset();
 806    VF->Size = List.Size();
 807    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 808       Cache.HeaderP->MaxVerFileSize = VF->Size;
 809    Cache.HeaderP->VerFileCount++;
 810
 811    return true;
 812 }
 813                                                                         /*}}}*/
 814 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 815 // ---------------------------------------------------------------------
 816 /* This puts a version structure in the linked list */
 817 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 818                                             APT::StringView const &VerStr,
 819                                             map_pointer_t const ParentPkg,
 820                                             unsigned short const Hash,
 821                                             map_pointer_t const Next)
 822 {
 823    // Get a structure
 824    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 825    if (Version == 0)
 826       return 0;
 827
 828    // Fill it in
 829    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 830    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 831    Ver->NextVer = Next;
 832    Ver->ParentPkg = ParentPkg;
 833    Ver->Hash = Hash;
 834    Ver->ID = Cache.HeaderP->VersionCount++;
 835
 836    // try to find the version string in the group for reuse
 837    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 838    pkgCache::GrpIterator Grp = Pkg.Group();
 839    if (Pkg.end() == false && Grp.end() == false)
 840    {
 841       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 842       {
 843          if (Pkg == P)
 844             continue;
 845          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 846          {
 847             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 848             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 849             {
 850                Ver->VerStr = V->VerStr;
 851                return Version;
 852             }
 853             else if (cmp < 0)
 854                break;
 855          }
 856       }
 857    }
 858    // haven't found the version string, so create
 859    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 860    if (unlikely(idxVerStr == 0))
 861       return 0;
 862    Ver->VerStr = idxVerStr;
 863    return Version;
 864 }
 865                                                                         /*}}}*/
 866 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 867 // ---------------------------------------------------------------------
 868 /* */
 869 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 870                                    ListParser &List)
 871 {
 872    if (CurrentFile == 0)
 873       return true;
 874
 875    // Get a structure
 876    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 877    if (DescFile == 0)
 878       return false;
 879
 880    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 881    DF->File = CurrentFile - Cache.PkgFileP;
 882
 883    // Link it to the end of the list
 884    map_pointer_t *Last = &Desc->FileList;
 885    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 886       Last = &D->NextFile;
 887
 888    DF->NextFile = *Last;
 889    *Last = DF.Index();
 890
 891    DF->Offset = List.Offset();
 892    DF->Size = List.Size();
 893    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 894       Cache.HeaderP->MaxDescFileSize = DF->Size;
 895    Cache.HeaderP->DescFileCount++;
 896
 897    return true;
 898 }
 899                                                                         /*}}}*/
 900 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 901 // ---------------------------------------------------------------------
 902 /* This puts a description structure in the linked list */
 903 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 904                                             const string &Lang,
 905                                             const MD5SumValue &md5sum,
 906                                             map_stringitem_t const idxmd5str)
 907 {
 908    // Get a structure
 909    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 910    if (Description == 0)
 911       return 0;
 912
 913    // Fill it in
 914    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 915    Desc->ID = Cache.HeaderP->DescriptionCount++;
 916    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 917    if (unlikely(idxlanguage_code == 0))
 918       return 0;
 919    Desc->language_code = idxlanguage_code;
 920
 921    if (idxmd5str != 0)
 922       Desc->md5sum = idxmd5str;
 923    else
 924    {
 925       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 926       if (unlikely(idxmd5sum == 0))
 927          return 0;
 928       Desc->md5sum = idxmd5sum;
 929    }
 930
 931    return Description;
 932 }
 933                                                                         /*}}}*/
 934 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 935 // ---------------------------------------------------------------------
 936 /* This creates a dependency element in the tree. It is linked to the
 937    version and to the package that it is pointing to. */
 938 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 939                                    pkgCache::VerIterator &Ver,
 940                                    map_pointer_t const Version,
 941                                    uint8_t const Op,
 942                                    uint8_t const Type,
 943                                    map_pointer_t* &OldDepLast)
 944 {
 945    void const * const oldMap = Map.Data();
 946    // Get a structure
 947    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 948    if (unlikely(Dependency == 0))
 949       return false;
 950
 951    bool isDuplicate = false;
 952    map_pointer_t DependencyData = 0;
 953    map_pointer_t PreviousData = 0;
 954    if (Pkg->RevDepends != 0)
 955    {
 956       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 957       DependencyData = L->DependencyData;
 958       do {
 959          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 960          if (Version > D->Version)
 961             break;
 962          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 963          {
 964             isDuplicate = true;
 965             break;
 966          }
 967          PreviousData = DependencyData;
 968          DependencyData = D->NextData;
 969       } while (DependencyData != 0);
 970    }
 971
 972    if (isDuplicate == false)
 973    {
 974       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 975       if (unlikely(DependencyData == 0))
 976         return false;
 977    }
 978
 979    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 980    Link->ParentVer = Ver.Index();
 981    Link->DependencyData = DependencyData;
 982    Link->ID = Cache.HeaderP->DependsCount++;
 983
 984    pkgCache::DepIterator Dep(Cache, Link);
 985    if (isDuplicate == false)
 986    {
 987       Dep->Type = Type;
 988       Dep->CompareOp = Op;
 989       Dep->Version = Version;
 990       Dep->Package = Pkg.Index();
 991       ++Cache.HeaderP->DependsDataCount;
 992       if (PreviousData != 0)
 993       {
 994          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 995          Dep->NextData = D->NextData;
 996          D->NextData = DependencyData;
 997       }
 998       else if (Pkg->RevDepends != 0)
 999       {
1000          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1001          Dep->NextData = D->DependencyData;
1002       }
1003    }
1004
1005    if (isDuplicate == true || PreviousData != 0)
1006    {
1007       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1008       Link->NextRevDepends = L->NextRevDepends;
1009       L->NextRevDepends = Dependency;
1010    }
1011    else
1012    {
1013       Link->NextRevDepends = Pkg->RevDepends;
1014       Pkg->RevDepends = Dependency;
1015    }
1016
1017
1018    // Do we know where to link the Dependency to?
1019    if (OldDepLast == NULL)
1020    {
1021       OldDepLast = &Ver->DependsList;
1022       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1023          OldDepLast = &D->NextDepends;
1024    } else if (oldMap != Map.Data())
1025       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1026
1027    Dep->NextDepends = *OldDepLast;
1028    *OldDepLast = Dependency;
1029    OldDepLast = &Dep->NextDepends;
1030    return true;
1031 }
1032                                                                         /*}}}*/
1033 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* This creates a Group and the Package to link this dependency to if
1036    needed and handles also the caching of the old endpoint */
1037 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1038                                                StringView PackageName,
1039                                                StringView Arch,
1040                                                StringView Version,
1041                                                uint8_t const Op,
1042                                                uint8_t const Type)
1043 {
1044    pkgCache::GrpIterator Grp;
1045    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1046    Dynamic<StringView> DynPackageName(PackageName);
1047    Dynamic<StringView> DynArch(Arch);
1048    Dynamic<StringView> DynVersion(Version);
1049    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1050       return false;
1051
1052    map_stringitem_t idxVersion = 0;
1053    if (Version.empty() == false)
1054    {
1055       int const CmpOp = Op & 0x0F;
1056       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1057       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1058          idxVersion = Ver->VerStr;
1059
1060       if (idxVersion == 0)
1061       {
1062          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1063          if (unlikely(idxVersion == 0))
1064             return false;
1065       }
1066    }
1067
1068    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1069          Type == pkgCache::Dep::Conflicts ||
1070          Type == pkgCache::Dep::Replaces);
1071
1072    pkgCache::PkgIterator Pkg;
1073    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1074    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1075    {
1076       // Locate the target package
1077       Pkg = Grp.FindPkg(Arch);
1078       if (Pkg.end() == true) {
1079          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1080             return false;
1081       }
1082
1083       /* Caching the old end point speeds up generation substantially */
1084       if (OldDepVer != Ver) {
1085          OldDepLast = NULL;
1086          OldDepVer = Ver;
1087       }
1088
1089       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1090    }
1091    else
1092    {
1093       /* Caching the old end point speeds up generation substantially */
1094       if (OldDepVer != Ver) {
1095          OldDepLast = NULL;
1096          OldDepVer = Ver;
1097       }
1098
1099       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1100       {
1101          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1102                return false;
1103       }
1104    }
1105    return true;
1106 }
1107                                                                         /*}}}*/
1108 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1109 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1110                                                 StringView PkgName,
1111                                                 StringView PkgArch,
1112                                                 StringView Version,
1113                                                 uint8_t const Flags)
1114 {
1115    pkgCache const &Cache = Owner->Cache;
1116    Dynamic<StringView> DynPkgName(PkgName);
1117    Dynamic<StringView> DynArch(PkgArch);
1118    Dynamic<StringView> DynVersion(Version);
1119
1120    // We do not add self referencing provides
1121    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1122         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1123          (Version.empty() || Version == Ver.VerStr()))
1124       return true;
1125
1126    // Locate the target package
1127    pkgCache::PkgIterator Pkg;
1128    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1129    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1130       return false;
1131
1132    map_stringitem_t idxProvideVersion = 0;
1133    if (Version.empty() == false) {
1134       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1135       if (unlikely(idxProvideVersion == 0))
1136          return false;
1137    }
1138    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1139 }
1140 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1141                                     pkgCache::PkgIterator &Pkg,
1142                                     map_pointer_t const ProvideVersion,
1143                                     uint8_t const Flags)
1144 {
1145    // Get a structure
1146    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1147    if (unlikely(Provides == 0))
1148       return false;
1149    ++Cache.HeaderP->ProvidesCount;
1150
1151    // Fill it in
1152    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1153    Prv->Version = Ver.Index();
1154    Prv->ProvideVersion = ProvideVersion;
1155    Prv->Flags = Flags;
1156    Prv->NextPkgProv = Ver->ProvidesList;
1157    Ver->ProvidesList = Prv.Index();
1158
1159    // Link it to the package
1160    Prv->ParentPkg = Pkg.Index();
1161    Prv->NextProvides = Pkg->ProvidesList;
1162    Pkg->ProvidesList = Prv.Index();
1163    return true;
1164 }
1165                                                                         /*}}}*/
1166 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1167 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1168                                 StringView Version, uint8_t const Flags) {
1169    pkgCache &Cache = Owner->Cache;
1170    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1171    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1172    Dynamic<StringView> DynPackage(Package);
1173    Dynamic<StringView> DynVersion(Version);
1174
1175    if (Grp.end() == true)
1176       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1177    else
1178    {
1179       map_stringitem_t idxProvideVersion = 0;
1180       if (Version.empty() == false) {
1181          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1182          if (unlikely(idxProvideVersion == 0))
1183             return false;
1184       }
1185
1186       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1187       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1188       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1189       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1190       pkgCache::PkgIterator Pkg;
1191       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1192       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1193       {
1194          if (isImplicit && OwnerPkg == Pkg)
1195             continue;
1196          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1197             continue;
1198          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1199             return false;
1200       }
1201    }
1202    return true;
1203 }
1204                                                                         /*}}}*/
1205 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1206       pkgCache::VerIterator const &Ver)
1207 {
1208    return Hash == Ver->Hash;
1209 }
1210                                                                         /*}}}*/
1211 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1212 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1213                                    unsigned long Flags)
1214 {
1215    if (File.empty() && Site.empty())
1216    {
1217       CurrentRlsFile = NULL;
1218       return true;
1219    }
1220
1221    // Get some space for the structure
1222    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1223    if (unlikely(idxFile == 0))
1224       return false;
1225    CurrentRlsFile = Cache.RlsFileP + idxFile;
1226
1227    // Fill it in
1228    map_stringitem_t const idxFileName = WriteStringInMap(File);
1229    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1230    if (unlikely(idxFileName == 0 || idxSite == 0))
1231       return false;
1232    CurrentRlsFile->FileName = idxFileName;
1233    CurrentRlsFile->Site = idxSite;
1234    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1235    CurrentRlsFile->Flags = Flags;
1236    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1237    RlsFileName = File;
1238    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1239    Cache.HeaderP->ReleaseFileCount++;
1240
1241    return true;
1242 }
1243                                                                         /*}}}*/
1244 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1245 // ---------------------------------------------------------------------
1246 /* This is used to select which file is to be associated with all newly
1247    added versions. The caller is responsible for setting the IMS fields. */
1248 bool pkgCacheGenerator::SelectFile(std::string const &File,
1249                                    pkgIndexFile const &Index,
1250                                    std::string const &Architecture,
1251                                    std::string const &Component,
1252                                    unsigned long const Flags)
1253 {
1254    // Get some space for the structure
1255    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1256    if (unlikely(idxFile == 0))
1257       return false;
1258    CurrentFile = Cache.PkgFileP + idxFile;
1259
1260    // Fill it in
1261    map_stringitem_t const idxFileName = WriteStringInMap(File);
1262    if (unlikely(idxFileName == 0))
1263       return false;
1264    CurrentFile->FileName = idxFileName;
1265    CurrentFile->NextFile = Cache.HeaderP->FileList;
1266    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1267    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1268    if (unlikely(idxIndexType == 0))
1269       return false;
1270    CurrentFile->IndexType = idxIndexType;
1271    if (Architecture.empty())
1272       CurrentFile->Architecture = 0;
1273    else
1274    {
1275       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1276       if (unlikely(arch == 0))
1277          return false;
1278       CurrentFile->Architecture = arch;
1279    }
1280    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1281    if (unlikely(component == 0))
1282       return false;
1283    CurrentFile->Component = component;
1284    CurrentFile->Flags = Flags;
1285    if (CurrentRlsFile != NULL)
1286       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1287    else
1288       CurrentFile->Release = 0;
1289    PkgFileName = File;
1290    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1291    Cache.HeaderP->PackageFileCount++;
1292
1293    if (Progress != 0)
1294       Progress->SubProgress(Index.Size());
1295    return true;
1296 }
1297                                                                         /*}}}*/
1298 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1299 // ---------------------------------------------------------------------
1300 /* This is used to create handles to strings. Given the same text it
1301    always returns the same number */
1302 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1303                                                  unsigned int Size)
1304 {
1305    auto strings = &strMixed;
1306    switch(type) {
1307       case MIXED: strings = &strMixed; break;
1308       case PKGNAME: strings = &strPkgNames; break;
1309       case VERSIONNUMBER: strings = &strVersions; break;
1310       case SECTION: strings = &strSections; break;
1311       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1312    }
1313
1314    auto const item = strings->find({S, Size, nullptr, 0});
1315    if (item != strings->end())
1316       return item->item;
1317
1318    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1319    strings->insert({nullptr, Size, this, idxString});
1320    return idxString;
1321 }
1322                                                                         /*}}}*/
1323 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1324 // ---------------------------------------------------------------------
1325 /* This just verifies that each file in the list of index files exists,
1326    has matching attributes with the cache and the cache does not have
1327    any extra files. */
1328 class APT_HIDDEN ScopedErrorRevert {
1329 public:
1330    ScopedErrorRevert() { _error->PushToStack(); }
1331    ~ScopedErrorRevert() { _error->RevertToStack(); }
1332 };
1333 static bool CheckValidity(const string &CacheFile,
1334                           pkgSourceList &List,
1335                           FileIterator const Start,
1336                           FileIterator const End,
1337                           MMap **OutMap = 0,
1338                           pkgCache **OutCache = 0)
1339 {
1340    ScopedErrorRevert ser;
1341    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1342    // No file, certainly invalid
1343    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1344    {
1345       if (Debug == true)
1346          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1347       return false;
1348    }
1349
1350    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1351    {
1352       if (Debug == true)
1353          std::clog << "sources.list is newer than the cache" << std::endl;
1354       return false;
1355    }
1356
1357    // Map it
1358    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1359    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1360    if (unlikely(Map->validData()) == false)
1361       return false;
1362    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1363    pkgCache &Cache = *CacheP.get();
1364    if (_error->PendingError() || Map->Size() == 0)
1365    {
1366       if (Debug == true)
1367          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1368       return false;
1369    }
1370
1371    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1372    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1373    std::vector<pkgIndexFile *> Files;
1374    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1375    {
1376       if (Debug == true)
1377          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1378       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1379       if (RlsFile.end() == true)
1380       {
1381          if (Debug == true)
1382             std::clog << "FindInCache returned end-Pointer" << std::endl;
1383          return false;
1384       }
1385
1386       RlsVisited[RlsFile->ID] = true;
1387       if (Debug == true)
1388          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1389
1390       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1391       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1392             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1393    }
1394    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1395       if (RlsVisited[I] == false)
1396       {
1397          if (Debug == true)
1398             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1399          return false;
1400       }
1401
1402    std::copy(Start, End, std::back_inserter(Files));
1403
1404    /* Now we check every index file, see if it is in the cache,
1405       verify the IMS data and check that it is on the disk too.. */
1406    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1407    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1408    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1409    {
1410       if (Debug == true)
1411          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1412       if ((*PkgFile)->Exists() == false)
1413       {
1414          if (Debug == true)
1415             std::clog << "file doesn't exist" << std::endl;
1416          continue;
1417       }
1418
1419       // FindInCache is also expected to do an IMS check.
1420       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1421       if (File.end() == true)
1422       {
1423          if (Debug == true)
1424             std::clog << "FindInCache returned end-Pointer" << std::endl;
1425          return false;
1426       }
1427
1428       Visited[File->ID] = true;
1429       if (Debug == true)
1430          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1431    }
1432
1433    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1434       if (Visited[I] == false)
1435       {
1436          if (Debug == true)
1437             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1438          return false;
1439       }
1440
1441    if (_error->PendingError() == true)
1442    {
1443       if (Debug == true)
1444       {
1445          std::clog << "Validity failed because of pending errors:" << std::endl;
1446          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1447       }
1448       return false;
1449    }
1450
1451    if (OutMap != 0)
1452       *OutMap = Map.release();
1453    if (OutCache != 0)
1454       *OutCache = CacheP.release();
1455    return true;
1456 }
1457                                                                         /*}}}*/
1458 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1459 // ---------------------------------------------------------------------
1460 /* Size is kind of an abstract notion that is only used for the progress
1461    meter */
1462 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1463 {
1464    map_filesize_t TotalSize = 0;
1465    if (List !=  NULL)
1466    {
1467       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1468       {
1469          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1470          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1471             if ((*j)->HasPackages() == true)
1472                TotalSize += (*j)->Size();
1473       }
1474    }
1475
1476    for (; Start < End; ++Start)
1477    {
1478       if ((*Start)->HasPackages() == false)
1479          continue;
1480       TotalSize += (*Start)->Size();
1481    }
1482    return TotalSize;
1483 }
1484                                                                         /*}}}*/
1485 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1486 static bool BuildCache(pkgCacheGenerator &Gen,
1487                        OpProgress * const Progress,
1488                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1489                        pkgSourceList const * const List,
1490                        FileIterator const Start, FileIterator const End)
1491 {
1492    bool mergeFailure = false;
1493
1494    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1495       if (I->HasPackages() == false || mergeFailure)
1496          return;
1497
1498       if (I->Exists() == false)
1499          return;
1500
1501       if (I->FindInCache(Gen.GetCache()).end() == false)
1502       {
1503          _error->Warning("Duplicate sources.list entry %s",
1504                I->Describe().c_str());
1505          return;
1506       }
1507
1508       map_filesize_t const Size = I->Size();
1509       if (Progress != NULL)
1510          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1511       CurrentSize += Size;
1512
1513       if (I->Merge(Gen,Progress) == false)
1514          mergeFailure = true;
1515    };
1516
1517    if (List !=  NULL)
1518    {
1519       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1520       {
1521          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1522          {
1523             _error->Warning("Duplicate sources.list entry %s",
1524                   (*i)->Describe().c_str());
1525             continue;
1526          }
1527
1528          if ((*i)->Merge(Gen, Progress) == false)
1529             return false;
1530
1531          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1532          if (Indexes != NULL)
1533             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1534          if (mergeFailure)
1535             return false;
1536       }
1537    }
1538
1539    if (Start != End)
1540    {
1541       Gen.SelectReleaseFile("", "");
1542       std::for_each(Start, End, indexFileMerge);
1543       if (mergeFailure)
1544          return false;
1545    }
1546    return true;
1547 }
1548                                                                         /*}}}*/
1549 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1550 // ---------------------------------------------------------------------
1551 /* This makes sure that the status cache (the cache that has all
1552    index files from the sources list and all local ones) is ready
1553    to be mmaped. If OutMap is not zero then a MMap object representing
1554    the cache will be stored there. This is pretty much mandetory if you
1555    are using AllowMem. AllowMem lets the function be run as non-root
1556    where it builds the cache 'fast' into a memory buffer. */
1557 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1558 {
1559    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1560    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1561    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1562    Flags |= MMap::Moveable;
1563    if (_config->FindB("APT::Cache-Fallback", false) == true)
1564       Flags |= MMap::Fallback;
1565    if (CacheF != NULL)
1566       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1567    else
1568       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1569 }
1570 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1571       std::string const &FileName)
1572 {
1573    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1574    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1575       return false;
1576
1577    fchmod(SCacheF.Fd(),0644);
1578
1579    // Write out the main data
1580    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1581       return _error->Error(_("IO Error saving source cache"));
1582
1583    // Write out the proper header
1584    Gen->GetCache().HeaderP->Dirty = false;
1585    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1586    if (SCacheF.Seek(0) == false ||
1587          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1588       return _error->Error(_("IO Error saving source cache"));
1589    Gen->GetCache().HeaderP->Dirty = true;
1590    return true;
1591 }
1592 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1593       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1594 {
1595    Map.reset(CreateDynamicMMap(NULL, 0));
1596    if (unlikely(Map->validData()) == false)
1597       return false;
1598    FileFd CacheF(FileName, FileFd::ReadOnly);
1599    if (CacheF.IsOpen() == false || CacheF.Failed())
1600       return false;
1601    _error->PushToStack();
1602    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1603    bool const newError = _error->PendingError();
1604    _error->MergeWithStack();
1605    if (alloc == 0 && newError)
1606       return false;
1607    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1608       return false;
1609    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1610    return true;
1611 }
1612 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1613                         MMap **OutMap, bool AllowMem)
1614    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1615 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1616                         MMap **OutMap,bool)
1617 {
1618    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1619 }
1620 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1621                         MMap **OutMap,pkgCache **OutCache, bool)
1622 {
1623    // FIXME: deprecate the ignored AllowMem parameter
1624    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1625
1626    std::vector<pkgIndexFile *> Files;
1627    if (_system->AddStatusFiles(Files) == false)
1628       return false;
1629
1630    // Decide if we can write to the files..
1631    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1632    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1633
1634    // ensure the cache directory exists
1635    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1636    {
1637       string dir = _config->FindDir("Dir::Cache");
1638       size_t const len = dir.size();
1639       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1640          dir = dir.substr(0, len - 5);
1641       if (CacheFile.empty() == false)
1642          CreateDirectory(dir, flNotFile(CacheFile));
1643       if (SrcCacheFile.empty() == false)
1644          CreateDirectory(dir, flNotFile(SrcCacheFile));
1645    }
1646
1647    if (Progress != NULL)
1648       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1649
1650    bool pkgcache_fine = false;
1651    bool srcpkgcache_fine = false;
1652    bool volatile_fine = List.GetVolatileFiles().empty();
1653
1654    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1655                      volatile_fine ? OutCache : NULL) == true)
1656    {
1657       if (Debug == true)
1658          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1659       pkgcache_fine = true;
1660       srcpkgcache_fine = true;
1661    }
1662    if (pkgcache_fine == false)
1663    {
1664       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1665       {
1666          if (Debug == true)
1667             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1668          srcpkgcache_fine = true;
1669       }
1670    }
1671
1672    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1673    {
1674       if (Progress != NULL)
1675          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1676       return true;
1677    }
1678
1679    bool Writeable = false;
1680    if (srcpkgcache_fine == false || pkgcache_fine == false)
1681    {
1682       if (CacheFile.empty() == false)
1683          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1684       else if (SrcCacheFile.empty() == false)
1685          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1686
1687       if (Debug == true)
1688          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1689    }
1690
1691    // At this point we know we need to construct something, so get storage ready
1692    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1693    if (unlikely(Map->validData()) == false)
1694       return false;
1695    if (Debug == true)
1696       std::clog << "Open memory Map (not filebased)" << std::endl;
1697
1698    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1699    map_filesize_t CurrentSize = 0;
1700    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1701    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1702    if (srcpkgcache_fine == true && pkgcache_fine == false)
1703    {
1704       if (Debug == true)
1705          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1706       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1707          return false;
1708       srcpkgcache_fine = true;
1709       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1710    }
1711    else if (srcpkgcache_fine == false)
1712    {
1713       if (Debug == true)
1714          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1715       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1716
1717       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1718       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1719                Files.end(),Files.end()) == false)
1720          return false;
1721
1722       if (Writeable == true && SrcCacheFile.empty() == false)
1723          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1724             return false;
1725    }
1726
1727    if (pkgcache_fine == false)
1728    {
1729       if (Debug == true)
1730          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1731       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1732                Files.begin(), Files.end()) == false)
1733          return false;
1734
1735       if (Writeable == true && CacheFile.empty() == false)
1736          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1737             return false;
1738    }
1739
1740    if (Debug == true)
1741       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1742
1743    if (volatile_fine == false)
1744    {
1745       if (Gen == nullptr)
1746       {
1747          if (Debug == true)
1748             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1749          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1750             return false;
1751       }
1752
1753       Files = List.GetVolatileFiles();
1754       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1755                Files.begin(), Files.end()) == false)
1756          return false;
1757    }
1758
1759    if (OutMap != nullptr)
1760       *OutMap = Map.release();
1761
1762    if (Debug == true)
1763       std::clog << "Everything is ready for shipping" << std::endl;
1764    return true;
1765 }
1766                                                                         /*}}}*/
1767 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1768 class APT_HIDDEN ScopedErrorMerge {
1769 public:
1770    ScopedErrorMerge() { _error->PushToStack(); }
1771    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1772 };
1773 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1774    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1775 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1776 {
1777    std::vector<pkgIndexFile *> Files;
1778    if (_system->AddStatusFiles(Files) == false)
1779       return false;
1780
1781    ScopedErrorMerge sem;
1782    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1783    if (unlikely(Map->validData()) == false)
1784       return false;
1785    map_filesize_t CurrentSize = 0;
1786    map_filesize_t TotalSize = 0;
1787    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1788
1789    // Build the status cache
1790    if (Progress != NULL)
1791       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1792    pkgCacheGenerator Gen(Map.get(),Progress);
1793    if (_error->PendingError() == true)
1794       return false;
1795    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1796                   Files.begin(), Files.end()) == false)
1797       return false;
1798
1799    if (_error->PendingError() == true)
1800       return false;
1801    *OutMap = Map.release();
1802
1803    return true;
1804 }
1805                                                                         /*}}}*/
1806 // IsDuplicateDescription                                               /*{{{*/
1807 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1808                             MD5SumValue const &CurMd5, std::string const &CurLang)
1809 {
1810    // Descriptions in the same link-list have all the same md5
1811    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1812       return false;
1813    for (; Desc.end() == false; ++Desc)
1814       if (Desc.LanguageCode() == CurLang)
1815          return true;
1816    return false;
1817 }
1818                                                                         /*}}}*/
1819
1820 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1821 pkgCacheListParser::~pkgCacheListParser() {}