apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
  49                             APT::StringView CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return false;
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (_error->PendingError() == true || Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 180         if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 184       if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 199       if (std::get<1>(seen.insert(ViewP)) == false)
 200          continue;
 201       // Ignore views outside of the cache.
 202       if (ViewP->data() < static_cast<const char*>(oldMap)
 203          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 204          continue;
 205       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 206       *ViewP = StringView(data , ViewP->size());
 207    }
 208 }                                                                       /*}}}*/
 209 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 210 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 211                                         const unsigned long &Len) {
 212    size_t oldSize = Map.Size();
 213    void const * const oldMap = Map.Data();
 214    map_stringitem_t const index = Map.WriteString(String, Len);
 215    if (index != 0)
 216       ReMap(oldMap, Map.Data(), oldSize);
 217    return index;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 221 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 222    size_t oldSize = Map.Size();
 223    void const * const oldMap = Map.Data();
 224    map_stringitem_t const index = Map.WriteString(String);
 225    if (index != 0)
 226       ReMap(oldMap, Map.Data(), oldSize);
 227    return index;
 228 }
 229                                                                         /*}}}*/
 230 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 231    size_t oldSize = Map.Size();
 232    void const * const oldMap = Map.Data();
 233    map_pointer_t const index = Map.Allocate(size);
 234    if (index != 0)
 235       ReMap(oldMap, Map.Data(), oldSize);
 236    return index;
 237 }
 238                                                                         /*}}}*/
 239 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This provides the generation of the entries in the cache. Each loop
 242    goes through a single package record from the underlying parse engine. */
 243 bool pkgCacheGenerator::MergeList(ListParser &List,
 244                                   pkgCache::VerIterator *OutVer)
 245 {
 246    List.Owner = this;
 247
 248    unsigned int Counter = 0;
 249    while (List.Step() == true)
 250    {
 251       string const PackageName = List.Package();
 252       if (PackageName.empty() == true)
 253          return false;
 254
 255       Counter++;
 256       if (Counter % 100 == 0 && Progress != 0)
 257          Progress->Progress(List.Offset());
 258
 259       APT::StringView Arch = List.Architecture();
 260       Dynamic<APT::StringView> DynArch(Arch);
 261       APT::StringView Version = List.Version();
 262       Dynamic<APT::StringView> DynVersion(Version);
 263       if (Version.empty() == true && Arch.empty() == true)
 264       {
 265          // package descriptions
 266          if (MergeListGroup(List, PackageName) == false)
 267             return false;
 268          continue;
 269       }
 270
 271       // Get a pointer to the package structure
 272       pkgCache::PkgIterator Pkg;
 273       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 274       if (NewPackage(Pkg, PackageName, Arch) == false)
 275          // TRANSLATOR: The first placeholder is a package name,
 276          // the other two should be copied verbatim as they include debug info
 277          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 278                               PackageName.c_str(), "NewPackage", 1);
 279
 280
 281       if (Version.empty() == true)
 282       {
 283          if (MergeListPackage(List, Pkg) == false)
 284             return false;
 285       }
 286       else
 287       {
 288          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 289             return false;
 290       }
 291
 292       if (OutVer != 0)
 293          return true;
 294    }
 295
 296    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 297       return _error->Error(_("Wow, you exceeded the number of package "
 298                              "names this APT is capable of."));
 299    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 300       return _error->Error(_("Wow, you exceeded the number of versions "
 301                              "this APT is capable of."));
 302    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 303       return _error->Error(_("Wow, you exceeded the number of descriptions "
 304                              "this APT is capable of."));
 305    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 306       return _error->Error(_("Wow, you exceeded the number of dependencies "
 307                              "this APT is capable of."));
 308
 309    return true;
 310 }
 311 // CacheGenerator::MergeListGroup                                       /*{{{*/
 312 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 313 {
 314    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 315    // a group has no data on it's own, only packages have it but these
 316    // stanzas like this come from Translation- files to add descriptions,
 317    // but without a version we don't need a description for it…
 318    if (Grp.end() == true)
 319       return true;
 320    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 321
 322    pkgCache::PkgIterator Pkg;
 323    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 324    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 325       if (MergeListPackage(List, Pkg) == false)
 326          return false;
 327
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // CacheGenerator::MergeListPackage                                     /*{{{*/
 332 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 333 {
 334    // we first process the package, then the descriptions
 335    // (for deb this package processing is in fact a no-op)
 336    pkgCache::VerIterator Ver(Cache);
 337    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 338    if (List.UsePackage(Pkg, Ver) == false)
 339       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 340                            Pkg.Name(), "UsePackage", 1);
 341
 342    // Find the right version to write the description
 343    StringView CurMd5 = List.Description_md5();
 344    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 345    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 346    {
 347       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 348
 349       // a version can only have one md5 describing it
 350       if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
 351          continue;
 352
 353       map_stringitem_t md5idx = VerDesc->md5sum;
 354       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 355       {
 356          // don't add a new description if we have one for the given
 357          // md5 && language
 358          if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
 359             continue;
 360
 361          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 362       }
 363
 364       // we can stop here as all "same" versions will share the description
 365       break;
 366    }
 367
 368    return true;
 369 }
 370                                                                         /*}}}*/
 371 // CacheGenerator::MergeListVersion                                     /*{{{*/
 372 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 373                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 374 {
 375    pkgCache::VerIterator Ver = Pkg.VersionList();
 376    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 377    map_pointer_t *LastVer = &Pkg->VersionList;
 378    void const * oldMap = Map.Data();
 379
 380    unsigned short const Hash = List.VersionHash();
 381    if (Ver.end() == false)
 382    {
 383       /* We know the list is sorted so we use that fact in the search.
 384          Insertion of new versions is done with correct sorting */
 385       int Res = 1;
 386       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 387       {
 388          char const * const VerStr = Ver.VerStr();
 389          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 390                VerStr, VerStr + strlen(VerStr));
 391          // Version is higher as current version - insert here
 392          if (Res > 0)
 393             break;
 394          // Versionstrings are equal - is hash also equal?
 395          if (Res == 0)
 396          {
 397             if (List.SameVersion(Hash, Ver) == true)
 398                break;
 399             // sort (volatile) sources above not-sources like the status file
 400             if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 401             {
 402                auto VF = Ver.FileList();
 403                for (; VF.end() == false; ++VF)
 404                   if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
 405                      break;
 406                if (VF.end() == true)
 407                   break;
 408             }
 409          }
 410          // proceed with the next till we have either the right
 411          // or we found another version (which will be lower)
 412       }
 413
 414       /* We already have a version for this item, record that we saw it */
 415       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 416       {
 417          if (List.UsePackage(Pkg,Ver) == false)
 418             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                                  Pkg.Name(), "UsePackage", 2);
 420
 421          if (NewFileVer(Ver,List) == false)
 422             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 423                                  Pkg.Name(), "NewFileVer", 1);
 424
 425          // Read only a single record and return
 426          if (OutVer != 0)
 427          {
 428             *OutVer = Ver;
 429             return true;
 430          }
 431
 432          return true;
 433       }
 434    }
 435
 436    // Add a new version
 437    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 438    if (unlikely(verindex == 0))
 439       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 440                            Pkg.Name(), "NewVersion", 1);
 441
 442    if (oldMap != Map.Data())
 443          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 444    *LastVer = verindex;
 445
 446    if (unlikely(List.NewVersion(Ver) == false))
 447       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 448                            Pkg.Name(), "NewVersion", 2);
 449
 450    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 451       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 452                            Pkg.Name(), "UsePackage", 3);
 453
 454    if (unlikely(NewFileVer(Ver,List) == false))
 455       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 456                            Pkg.Name(), "NewFileVer", 2);
 457
 458    pkgCache::GrpIterator Grp = Pkg.Group();
 459    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 460
 461    /* If it is the first version of this package we need to add implicit
 462       Multi-Arch dependencies to all other package versions in the group now -
 463       otherwise we just add them for this new version */
 464    if (Pkg.VersionList()->NextVer == 0)
 465    {
 466       pkgCache::PkgIterator P = Grp.PackageList();
 467       Dynamic<pkgCache::PkgIterator> DynP(P);
 468       for (; P.end() != true; P = Grp.NextPkg(P))
 469       {
 470          if (P->ID == Pkg->ID)
 471             continue;
 472          pkgCache::VerIterator V = P.VersionList();
 473          Dynamic<pkgCache::VerIterator> DynV(V);
 474          for (; V.end() != true; ++V)
 475             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 476                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 477                                     Pkg.Name(), "AddImplicitDepends", 1);
 478       }
 479    }
 480    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482                            Pkg.Name(), "AddImplicitDepends", 2);
 483
 484    // Read only a single record and return
 485    if (OutVer != 0)
 486    {
 487       *OutVer = Ver;
 488       return true;
 489    }
 490
 491    /* Record the Description(s) based on their master md5sum */
 492    StringView CurMd5 = List.Description_md5();
 493
 494    /* Before we add a new description we first search in the group for
 495       a version with a description of the same MD5 - if so we reuse this
 496       description group instead of creating our own for this version */
 497    for (pkgCache::PkgIterator P = Grp.PackageList();
 498         P.end() == false; P = Grp.NextPkg(P))
 499    {
 500       for (pkgCache::VerIterator V = P.VersionList();
 501            V.end() == false; ++V)
 502       {
 503          if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
 504             continue;
 505          Ver->DescriptionList = V->DescriptionList;
 506       }
 507    }
 508
 509    // We haven't found reusable descriptions, so add the first description(s)
 510    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 511    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 512    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 513       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 514          return false;
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 519 {
 520    pkgCache::DescIterator Desc;
 521    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 522
 523    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 524    if (unlikely(descindex == 0))
 525       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 526             Ver.ParentPkg().Name(), "NewDescription", 1);
 527
 528    md5idx = Desc->md5sum;
 529    Desc->ParentPkg = Ver.ParentPkg().Index();
 530
 531    // we add at the end, so that the start is constant as we need
 532    // that to be able to efficiently share these lists
 533    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 534    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 535    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 536    *LastNextDesc = descindex;
 537
 538    if (NewFileDesc(Desc,List) == false)
 539       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 540             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 541
 542    return true;
 543 }
 544                                                                         /*}}}*/
 545                                                                         /*}}}*/
 546 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* This creates a new group structure and adds it to the hash table */
 549 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 550 {
 551    Dynamic<StringView> DName(Name);
 552    Grp = Cache.FindGrp(Name);
 553    if (Grp.end() == false)
 554       return true;
 555
 556    // Get a structure
 557    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 558    if (unlikely(Group == 0))
 559       return false;
 560
 561    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 562    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 563    if (unlikely(idxName == 0))
 564       return false;
 565    Grp->Name = idxName;
 566
 567    // Insert it into the hash table
 568    unsigned long const Hash = Cache.Hash(Name);
 569    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 570
 571    while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 572       insertAt = &(Cache.GrpP + *insertAt)->Next;
 573    Grp->Next = *insertAt;
 574    *insertAt = Group;
 575
 576    Grp->ID = Cache.HeaderP->GroupCount++;
 577    return true;
 578 }
 579                                                                         /*}}}*/
 580 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* This creates a new package structure and adds it to the hash table */
 583 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 584                                         StringView Arch) {
 585    pkgCache::GrpIterator Grp;
 586    Dynamic<StringView> DName(Name);
 587    Dynamic<StringView> DArch(Arch);
 588    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 589    if (unlikely(NewGroup(Grp, Name) == false))
 590       return false;
 591
 592    Pkg = Grp.FindPkg(Arch);
 593       if (Pkg.end() == false)
 594          return true;
 595
 596    // Get a structure
 597    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 598    if (unlikely(Package == 0))
 599       return false;
 600    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 601
 602    // Set the name, arch and the ID
 603    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 604    Pkg->Group = Grp.Index();
 605    // all is mapped to the native architecture
 606    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 607    if (unlikely(idxArch == 0))
 608       return false;
 609    Pkg->Arch = idxArch;
 610    Pkg->ID = Cache.HeaderP->PackageCount++;
 611
 612    // Insert the package into our package list
 613    if (Grp->FirstPackage == 0) // the group is new
 614    {
 615       Grp->FirstPackage = Package;
 616       // Insert it into the hash table
 617       map_id_t const Hash = Cache.Hash(Name);
 618       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 619       while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
 620          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 621       Pkg->NextPackage = *insertAt;
 622       *insertAt = Package;
 623    }
 624    else // Group the Packages together
 625    {
 626       // if sibling is provided by another package, this one is too
 627       {
 628          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 629          if (M.end() == false) {
 630             pkgCache::PrvIterator Prv;
 631             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 632             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 633             {
 634                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 635                   continue;
 636                pkgCache::VerIterator Ver = Prv.OwnerVer();
 637                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 638                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 639                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 640                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 641                {
 642                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 643                      continue;
 644                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 645                      return false;
 646                }
 647             }
 648          }
 649       }
 650       // let M-A:foreign package siblings provide this package
 651       {
 652          pkgCache::PkgIterator P;
 653          pkgCache::VerIterator Ver;
 654          Dynamic<pkgCache::PkgIterator> DynP(P);
 655          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 656
 657          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 658          {
 659             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 660                continue;
 661             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 662                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 663                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 664                      return false;
 665          }
 666       }
 667       // and negative dependencies, don't forget negative dependencies
 668       {
 669          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 670          if (M.end() == false) {
 671             pkgCache::DepIterator Dep;
 672             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 673             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 674             {
 675                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 676                   continue;
 677                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 678                      Dep->Type != pkgCache::Dep::Replaces)
 679                   continue;
 680                pkgCache::VerIterator Ver = Dep.ParentVer();
 681                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 682                map_pointer_t * unused = NULL;
 683                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 684                   return false;
 685             }
 686          }
 687       }
 688
 689       // this package is the new last package
 690       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 691       Pkg->NextPackage = LastPkg->NextPackage;
 692       LastPkg->NextPackage = Package;
 693    }
 694    Grp->LastPackage = Package;
 695
 696    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 697    if (Arch == "any")
 698    {
 699       size_t const found = Name.rfind(':');
 700       StringView ArchA = Name.substr(found + 1);
 701       if (ArchA != "any")
 702       {
 703          // ArchA is used inside the loop which might remap (NameA is not used)
 704          Dynamic<StringView> DynArchA(ArchA);
 705          StringView NameA = Name.substr(0, found);
 706          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 707          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 708          if (PkgA.end())
 709          {
 710             Dynamic<StringView> DynNameA(NameA);
 711             if (NewPackage(PkgA, NameA, ArchA) == false)
 712                return false;
 713          }
 714          if (unlikely(PkgA.end()))
 715             return _error->Fatal("NewPackage was successful for %s:%s,"
 716                   "but the package doesn't exist anyhow!",
 717                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 718          else
 719          {
 720             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 721             for (; Prv.end() == false; ++Prv)
 722             {
 723                if (Prv.IsMultiArchImplicit())
 724                   continue;
 725                pkgCache::VerIterator V = Prv.OwnerVer();
 726                if (ArchA != V.ParentPkg().Arch())
 727                   continue;
 728                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 729                   return false;
 730             }
 731             pkgCache::VerIterator V = PkgA.VersionList();
 732             Dynamic<pkgCache::VerIterator> DynV(V);
 733             for (; V.end() == false; ++V)
 734             {
 735                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 736                   return false;
 737             }
 738          }
 739       }
 740    }
 741    return true;
 742 }
 743                                                                         /*}}}*/
 744 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 745 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 746                                            pkgCache::PkgIterator &P,
 747                                            pkgCache::VerIterator &V)
 748 {
 749    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 750    Dynamic<APT::StringView> DynArch(Arch);
 751    map_pointer_t *OldDepLast = NULL;
 752    /* MultiArch handling introduces a lot of implicit Dependencies:
 753       - MultiArch: same → Co-Installable if they have the same version
 754       - All others conflict with all other group members */
 755    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 756    pkgCache::PkgIterator D = G.PackageList();
 757    Dynamic<pkgCache::PkgIterator> DynD(D);
 758    map_stringitem_t const VerStrIdx = V->VerStr;
 759    for (; D.end() != true; D = G.NextPkg(D))
 760    {
 761       if (Arch == D.Arch() || D->VersionList == 0)
 762          continue;
 763       /* We allow only one installed arch at the time
 764          per group, therefore each group member conflicts
 765          with all other group members */
 766       if (coInstall == true)
 767       {
 768          // Replaces: ${self}:other ( << ${binary:Version})
 769          NewDepends(D, V, VerStrIdx,
 770                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 771                     OldDepLast);
 772          // Breaks: ${self}:other (!= ${binary:Version})
 773          NewDepends(D, V, VerStrIdx,
 774                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 775                     OldDepLast);
 776       } else {
 777          // Conflicts: ${self}:other
 778          NewDepends(D, V, 0,
 779                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 780                     OldDepLast);
 781       }
 782    }
 783    return true;
 784 }
 785 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 786                                            pkgCache::PkgIterator &D)
 787 {
 788    /* MultiArch handling introduces a lot of implicit Dependencies:
 789       - MultiArch: same → Co-Installable if they have the same version
 790       - All others conflict with all other group members */
 791    map_pointer_t *OldDepLast = NULL;
 792    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 793    if (coInstall == true)
 794    {
 795       map_stringitem_t const VerStrIdx = V->VerStr;
 796       // Replaces: ${self}:other ( << ${binary:Version})
 797       NewDepends(D, V, VerStrIdx,
 798                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 799                  OldDepLast);
 800       // Breaks: ${self}:other (!= ${binary:Version})
 801       NewDepends(D, V, VerStrIdx,
 802                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 803                  OldDepLast);
 804    } else {
 805       // Conflicts: ${self}:other
 806       NewDepends(D, V, 0,
 807                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 808                  OldDepLast);
 809    }
 810    return true;
 811 }
 812
 813                                                                         /*}}}*/
 814 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 815 // ---------------------------------------------------------------------
 816 /* */
 817 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 818                                    ListParser &List)
 819 {
 820    if (CurrentFile == 0)
 821       return true;
 822
 823    // Get a structure
 824    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 825    if (VerFile == 0)
 826       return false;
 827
 828    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 829    VF->File = CurrentFile - Cache.PkgFileP;
 830
 831    // Link it to the end of the list
 832    map_pointer_t *Last = &Ver->FileList;
 833    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 834       Last = &V->NextFile;
 835    VF->NextFile = *Last;
 836    *Last = VF.Index();
 837
 838    VF->Offset = List.Offset();
 839    VF->Size = List.Size();
 840    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 841       Cache.HeaderP->MaxVerFileSize = VF->Size;
 842    Cache.HeaderP->VerFileCount++;
 843
 844    return true;
 845 }
 846                                                                         /*}}}*/
 847 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This puts a version structure in the linked list */
 850 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 851                                             APT::StringView const &VerStr,
 852                                             map_pointer_t const ParentPkg,
 853                                             unsigned short const Hash,
 854                                             map_pointer_t const Next)
 855 {
 856    // Get a structure
 857    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 858    if (Version == 0)
 859       return 0;
 860
 861    // Fill it in
 862    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 863    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 864    Ver->NextVer = Next;
 865    Ver->ParentPkg = ParentPkg;
 866    Ver->Hash = Hash;
 867    Ver->ID = Cache.HeaderP->VersionCount++;
 868
 869    // try to find the version string in the group for reuse
 870    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 871    pkgCache::GrpIterator Grp = Pkg.Group();
 872    if (Pkg.end() == false && Grp.end() == false)
 873    {
 874       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 875       {
 876          if (Pkg == P)
 877             continue;
 878          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 879          {
 880             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 881             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 882             {
 883                Ver->VerStr = V->VerStr;
 884                return Version;
 885             }
 886             else if (cmp < 0)
 887                break;
 888          }
 889       }
 890    }
 891    // haven't found the version string, so create
 892    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 893    if (unlikely(idxVerStr == 0))
 894       return 0;
 895    Ver->VerStr = idxVerStr;
 896    return Version;
 897 }
 898                                                                         /*}}}*/
 899 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 900 // ---------------------------------------------------------------------
 901 /* */
 902 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 903                                    ListParser &List)
 904 {
 905    if (CurrentFile == 0)
 906       return true;
 907
 908    // Get a structure
 909    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 910    if (DescFile == 0)
 911       return false;
 912
 913    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 914    DF->File = CurrentFile - Cache.PkgFileP;
 915
 916    // Link it to the end of the list
 917    map_pointer_t *Last = &Desc->FileList;
 918    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 919       Last = &D->NextFile;
 920
 921    DF->NextFile = *Last;
 922    *Last = DF.Index();
 923
 924    DF->Offset = List.Offset();
 925    DF->Size = List.Size();
 926    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 927       Cache.HeaderP->MaxDescFileSize = DF->Size;
 928    Cache.HeaderP->DescFileCount++;
 929
 930    return true;
 931 }
 932                                                                         /*}}}*/
 933 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 934 // ---------------------------------------------------------------------
 935 /* This puts a description structure in the linked list */
 936 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 937                                             const string &Lang,
 938                                             APT::StringView md5sum,
 939                                             map_stringitem_t const idxmd5str)
 940 {
 941    // Get a structure
 942    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 943    if (Description == 0)
 944       return 0;
 945
 946    // Fill it in
 947    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 948    Desc->ID = Cache.HeaderP->DescriptionCount++;
 949    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 950    if (unlikely(idxlanguage_code == 0))
 951       return 0;
 952    Desc->language_code = idxlanguage_code;
 953
 954    if (idxmd5str != 0)
 955       Desc->md5sum = idxmd5str;
 956    else
 957    {
 958       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
 959       if (unlikely(idxmd5sum == 0))
 960          return 0;
 961       Desc->md5sum = idxmd5sum;
 962    }
 963
 964    return Description;
 965 }
 966                                                                         /*}}}*/
 967 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 968 // ---------------------------------------------------------------------
 969 /* This creates a dependency element in the tree. It is linked to the
 970    version and to the package that it is pointing to. */
 971 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 972                                    pkgCache::VerIterator &Ver,
 973                                    map_pointer_t const Version,
 974                                    uint8_t const Op,
 975                                    uint8_t const Type,
 976                                    map_pointer_t* &OldDepLast)
 977 {
 978    void const * const oldMap = Map.Data();
 979    // Get a structure
 980    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 981    if (unlikely(Dependency == 0))
 982       return false;
 983
 984    bool isDuplicate = false;
 985    map_pointer_t DependencyData = 0;
 986    map_pointer_t PreviousData = 0;
 987    if (Pkg->RevDepends != 0)
 988    {
 989       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 990       DependencyData = L->DependencyData;
 991       do {
 992          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 993          if (Version > D->Version)
 994             break;
 995          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 996          {
 997             isDuplicate = true;
 998             break;
 999          }
1000          PreviousData = DependencyData;
1001          DependencyData = D->NextData;
1002       } while (DependencyData != 0);
1003    }
1004
1005    if (isDuplicate == false)
1006    {
1007       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1008       if (unlikely(DependencyData == 0))
1009         return false;
1010    }
1011
1012    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1013    Link->ParentVer = Ver.Index();
1014    Link->DependencyData = DependencyData;
1015    Link->ID = Cache.HeaderP->DependsCount++;
1016
1017    pkgCache::DepIterator Dep(Cache, Link);
1018    if (isDuplicate == false)
1019    {
1020       Dep->Type = Type;
1021       Dep->CompareOp = Op;
1022       Dep->Version = Version;
1023       Dep->Package = Pkg.Index();
1024       ++Cache.HeaderP->DependsDataCount;
1025       if (PreviousData != 0)
1026       {
1027          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1028          Dep->NextData = D->NextData;
1029          D->NextData = DependencyData;
1030       }
1031       else if (Pkg->RevDepends != 0)
1032       {
1033          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1034          Dep->NextData = D->DependencyData;
1035       }
1036    }
1037
1038    if (isDuplicate == true || PreviousData != 0)
1039    {
1040       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1041       Link->NextRevDepends = L->NextRevDepends;
1042       L->NextRevDepends = Dependency;
1043    }
1044    else
1045    {
1046       Link->NextRevDepends = Pkg->RevDepends;
1047       Pkg->RevDepends = Dependency;
1048    }
1049
1050
1051    // Do we know where to link the Dependency to?
1052    if (OldDepLast == NULL)
1053    {
1054       OldDepLast = &Ver->DependsList;
1055       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1056          OldDepLast = &D->NextDepends;
1057    } else if (oldMap != Map.Data())
1058       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1059
1060    Dep->NextDepends = *OldDepLast;
1061    *OldDepLast = Dependency;
1062    OldDepLast = &Dep->NextDepends;
1063    return true;
1064 }
1065                                                                         /*}}}*/
1066 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1067 // ---------------------------------------------------------------------
1068 /* This creates a Group and the Package to link this dependency to if
1069    needed and handles also the caching of the old endpoint */
1070 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1071                                                StringView PackageName,
1072                                                StringView Arch,
1073                                                StringView Version,
1074                                                uint8_t const Op,
1075                                                uint8_t const Type)
1076 {
1077    pkgCache::GrpIterator Grp;
1078    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1079    Dynamic<StringView> DynPackageName(PackageName);
1080    Dynamic<StringView> DynArch(Arch);
1081    Dynamic<StringView> DynVersion(Version);
1082    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1083       return false;
1084
1085    map_stringitem_t idxVersion = 0;
1086    if (Version.empty() == false)
1087    {
1088       int const CmpOp = Op & 0x0F;
1089       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1090       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1091          idxVersion = Ver->VerStr;
1092
1093       if (idxVersion == 0)
1094       {
1095          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1096          if (unlikely(idxVersion == 0))
1097             return false;
1098       }
1099    }
1100
1101    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1102          Type == pkgCache::Dep::Conflicts ||
1103          Type == pkgCache::Dep::Replaces);
1104
1105    pkgCache::PkgIterator Pkg;
1106    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1107    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1108    {
1109       // Locate the target package
1110       Pkg = Grp.FindPkg(Arch);
1111       if (Pkg.end() == true) {
1112          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1113             return false;
1114       }
1115
1116       /* Caching the old end point speeds up generation substantially */
1117       if (OldDepVer != Ver) {
1118          OldDepLast = NULL;
1119          OldDepVer = Ver;
1120       }
1121
1122       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1123    }
1124    else
1125    {
1126       /* Caching the old end point speeds up generation substantially */
1127       if (OldDepVer != Ver) {
1128          OldDepLast = NULL;
1129          OldDepVer = Ver;
1130       }
1131
1132       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1133       {
1134          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1135                return false;
1136       }
1137    }
1138    return true;
1139 }
1140                                                                         /*}}}*/
1141 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1142 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1143                                                 StringView PkgName,
1144                                                 StringView PkgArch,
1145                                                 StringView Version,
1146                                                 uint8_t const Flags)
1147 {
1148    pkgCache const &Cache = Owner->Cache;
1149    Dynamic<StringView> DynPkgName(PkgName);
1150    Dynamic<StringView> DynArch(PkgArch);
1151    Dynamic<StringView> DynVersion(Version);
1152
1153    // We do not add self referencing provides
1154    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1155         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1156          (Version.empty() || Version == Ver.VerStr()))
1157       return true;
1158
1159    // Locate the target package
1160    pkgCache::PkgIterator Pkg;
1161    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1162    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1163       return false;
1164
1165    map_stringitem_t idxProvideVersion = 0;
1166    if (Version.empty() == false) {
1167       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1168       if (unlikely(idxProvideVersion == 0))
1169          return false;
1170    }
1171    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1172 }
1173 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1174                                     pkgCache::PkgIterator &Pkg,
1175                                     map_pointer_t const ProvideVersion,
1176                                     uint8_t const Flags)
1177 {
1178    // Get a structure
1179    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1180    if (unlikely(Provides == 0))
1181       return false;
1182    ++Cache.HeaderP->ProvidesCount;
1183
1184    // Fill it in
1185    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1186    Prv->Version = Ver.Index();
1187    Prv->ProvideVersion = ProvideVersion;
1188    Prv->Flags = Flags;
1189    Prv->NextPkgProv = Ver->ProvidesList;
1190    Ver->ProvidesList = Prv.Index();
1191
1192    // Link it to the package
1193    Prv->ParentPkg = Pkg.Index();
1194    Prv->NextProvides = Pkg->ProvidesList;
1195    Pkg->ProvidesList = Prv.Index();
1196    return true;
1197 }
1198                                                                         /*}}}*/
1199 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1200 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1201                                 StringView Version, uint8_t const Flags) {
1202    pkgCache &Cache = Owner->Cache;
1203    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1204    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1205    Dynamic<StringView> DynPackage(Package);
1206    Dynamic<StringView> DynVersion(Version);
1207
1208    if (Grp.end() == true)
1209       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1210    else
1211    {
1212       map_stringitem_t idxProvideVersion = 0;
1213       if (Version.empty() == false) {
1214          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1215          if (unlikely(idxProvideVersion == 0))
1216             return false;
1217       }
1218
1219       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1220       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1221       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1222       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1223       pkgCache::PkgIterator Pkg;
1224       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1225       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1226       {
1227          if (isImplicit && OwnerPkg == Pkg)
1228             continue;
1229          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1230             continue;
1231          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1232             return false;
1233       }
1234    }
1235    return true;
1236 }
1237                                                                         /*}}}*/
1238 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1239       pkgCache::VerIterator const &Ver)
1240 {
1241    return Hash == Ver->Hash;
1242 }
1243                                                                         /*}}}*/
1244 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1245 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1246                                    unsigned long Flags)
1247 {
1248    if (File.empty() && Site.empty())
1249    {
1250       CurrentRlsFile = NULL;
1251       return true;
1252    }
1253
1254    // Get some space for the structure
1255    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1256    if (unlikely(idxFile == 0))
1257       return false;
1258    CurrentRlsFile = Cache.RlsFileP + idxFile;
1259
1260    // Fill it in
1261    map_stringitem_t const idxFileName = WriteStringInMap(File);
1262    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1263    if (unlikely(idxFileName == 0 || idxSite == 0))
1264       return false;
1265    CurrentRlsFile->FileName = idxFileName;
1266    CurrentRlsFile->Site = idxSite;
1267    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1268    CurrentRlsFile->Flags = Flags;
1269    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1270    RlsFileName = File;
1271    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1272    Cache.HeaderP->ReleaseFileCount++;
1273
1274    return true;
1275 }
1276                                                                         /*}}}*/
1277 // ListParser::NewTag - Create a Tag element                            /*{{{*/
1278 // ---------------------------------------------------------------------
1279 /* */
1280 bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
1281                                            const char *NameStart,
1282                                            unsigned int NameSize)
1283 {
1284    return Owner->NewTag(Ver, NameStart, NameSize);
1285 }
1286 bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
1287                                            const char *NameStart,
1288                                            unsigned int NameSize)
1289 {
1290    // Get a structure
1291    unsigned long Tagg = AllocateInMap(sizeof(pkgCache::Tag));
1292    if (Tagg == 0)
1293       return false;
1294    Cache.HeaderP->TagCount++;
1295
1296    // Fill it in
1297    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
1298    Tg->Name = WriteStringInMap(NameStart,NameSize);
1299    if (Tg->Name == 0)
1300       return false;
1301    Tg->NextTag = Ver->TagList;
1302    Ver->TagList = Tg.Index();
1303
1304    return true;
1305 }
1306                                                                         /*}}}*/
1307 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1308 // ---------------------------------------------------------------------
1309 /* This is used to select which file is to be associated with all newly
1310    added versions. The caller is responsible for setting the IMS fields. */
1311 bool pkgCacheGenerator::SelectFile(std::string const &File,
1312                                    pkgIndexFile const &Index,
1313                                    std::string const &Architecture,
1314                                    std::string const &Component,
1315                                    unsigned long const Flags)
1316 {
1317    // Get some space for the structure
1318    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1319    if (unlikely(idxFile == 0))
1320       return false;
1321    CurrentFile = Cache.PkgFileP + idxFile;
1322
1323    // Fill it in
1324    map_stringitem_t const idxFileName = WriteStringInMap(File);
1325    if (unlikely(idxFileName == 0))
1326       return false;
1327    CurrentFile->FileName = idxFileName;
1328    CurrentFile->NextFile = Cache.HeaderP->FileList;
1329    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1330    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1331    if (unlikely(idxIndexType == 0))
1332       return false;
1333    CurrentFile->IndexType = idxIndexType;
1334    if (Architecture.empty())
1335       CurrentFile->Architecture = 0;
1336    else
1337    {
1338       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1339       if (unlikely(arch == 0))
1340          return false;
1341       CurrentFile->Architecture = arch;
1342    }
1343    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1344    if (unlikely(component == 0))
1345       return false;
1346    CurrentFile->Component = component;
1347    CurrentFile->Flags = Flags;
1348    if (CurrentRlsFile != NULL)
1349       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1350    else
1351       CurrentFile->Release = 0;
1352    PkgFileName = File;
1353    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1354    Cache.HeaderP->PackageFileCount++;
1355
1356    if (Progress != 0)
1357       Progress->SubProgress(Index.Size());
1358    return true;
1359 }
1360                                                                         /*}}}*/
1361 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1362 // ---------------------------------------------------------------------
1363 /* This is used to create handles to strings. Given the same text it
1364    always returns the same number */
1365 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1366                                                  unsigned int Size)
1367 {
1368    auto strings = &strMixed;
1369    switch(type) {
1370       case MIXED: strings = &strMixed; break;
1371       case PKGNAME: strings = &strPkgNames; break;
1372       case VERSIONNUMBER: strings = &strVersions; break;
1373       case SECTION: strings = &strSections; break;
1374       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1375    }
1376
1377    auto const item = strings->find({S, Size, nullptr, 0});
1378    if (item != strings->end())
1379       return item->item;
1380
1381    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1382    strings->insert({nullptr, Size, this, idxString});
1383    return idxString;
1384 }
1385                                                                         /*}}}*/
1386 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1387 // ---------------------------------------------------------------------
1388 /* This just verifies that each file in the list of index files exists,
1389    has matching attributes with the cache and the cache does not have
1390    any extra files. */
1391 class APT_HIDDEN ScopedErrorRevert {
1392 public:
1393    ScopedErrorRevert() { _error->PushToStack(); }
1394    ~ScopedErrorRevert() { _error->RevertToStack(); }
1395 };
1396 static bool CheckValidity(const string &CacheFile,
1397                           pkgSourceList &List,
1398                           FileIterator const Start,
1399                           FileIterator const End,
1400                           MMap **OutMap = 0,
1401                           pkgCache **OutCache = 0)
1402 {
1403    ScopedErrorRevert ser;
1404    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1405    // No file, certainly invalid
1406    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1407    {
1408       if (Debug == true)
1409          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1410       return false;
1411    }
1412
1413    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1414    {
1415       if (Debug == true)
1416          std::clog << "sources.list is newer than the cache" << std::endl;
1417       return false;
1418    }
1419
1420    // Map it
1421    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1422    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1423    if (unlikely(Map->validData()) == false)
1424       return false;
1425    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1426    pkgCache &Cache = *CacheP.get();
1427    if (_error->PendingError() || Map->Size() == 0)
1428    {
1429       if (Debug == true)
1430          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1431       return false;
1432    }
1433
1434    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1435    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1436    std::vector<pkgIndexFile *> Files;
1437    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1438    {
1439       if (Debug == true)
1440          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1441       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1442       if (RlsFile.end() == true)
1443       {
1444          if (Debug == true)
1445             std::clog << "FindInCache returned end-Pointer" << std::endl;
1446          return false;
1447       }
1448
1449       RlsVisited[RlsFile->ID] = true;
1450       if (Debug == true)
1451          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1452
1453       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1454       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1455             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1456    }
1457    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1458       if (RlsVisited[I] == false)
1459       {
1460          if (Debug == true)
1461             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1462          return false;
1463       }
1464
1465    std::copy(Start, End, std::back_inserter(Files));
1466
1467    /* Now we check every index file, see if it is in the cache,
1468       verify the IMS data and check that it is on the disk too.. */
1469    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1470    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1471    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1472    {
1473       if (Debug == true)
1474          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1475       if ((*PkgFile)->Exists() == false)
1476       {
1477          if (Debug == true)
1478             std::clog << "file doesn't exist" << std::endl;
1479          continue;
1480       }
1481
1482       // FindInCache is also expected to do an IMS check.
1483       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1484       if (File.end() == true)
1485       {
1486          if (Debug == true)
1487             std::clog << "FindInCache returned end-Pointer" << std::endl;
1488          return false;
1489       }
1490
1491       Visited[File->ID] = true;
1492       if (Debug == true)
1493          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1494    }
1495
1496    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1497       if (Visited[I] == false)
1498       {
1499          if (Debug == true)
1500             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1501          return false;
1502       }
1503
1504    if (_error->PendingError() == true)
1505    {
1506       if (Debug == true)
1507       {
1508          std::clog << "Validity failed because of pending errors:" << std::endl;
1509          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1510       }
1511       return false;
1512    }
1513
1514    if (OutMap != 0)
1515       *OutMap = Map.release();
1516    if (OutCache != 0)
1517       *OutCache = CacheP.release();
1518    return true;
1519 }
1520                                                                         /*}}}*/
1521 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1522 // ---------------------------------------------------------------------
1523 /* Size is kind of an abstract notion that is only used for the progress
1524    meter */
1525 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1526 {
1527    map_filesize_t TotalSize = 0;
1528    if (List !=  NULL)
1529    {
1530       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1531       {
1532          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1533          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1534             if ((*j)->HasPackages() == true)
1535                TotalSize += (*j)->Size();
1536       }
1537    }
1538
1539    for (; Start < End; ++Start)
1540    {
1541       if ((*Start)->HasPackages() == false)
1542          continue;
1543       TotalSize += (*Start)->Size();
1544    }
1545    return TotalSize;
1546 }
1547                                                                         /*}}}*/
1548 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1549 static bool BuildCache(pkgCacheGenerator &Gen,
1550                        OpProgress * const Progress,
1551                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1552                        pkgSourceList const * const List,
1553                        FileIterator const Start, FileIterator const End)
1554 {
1555    bool mergeFailure = false;
1556
1557    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1558       if (I->HasPackages() == false || mergeFailure)
1559          return;
1560
1561       if (I->Exists() == false)
1562          return;
1563
1564       if (I->FindInCache(Gen.GetCache()).end() == false)
1565       {
1566          _error->Warning("Duplicate sources.list entry %s",
1567                I->Describe().c_str());
1568          return;
1569       }
1570
1571       map_filesize_t const Size = I->Size();
1572       if (Progress != NULL)
1573          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1574       CurrentSize += Size;
1575
1576       if (I->Merge(Gen,Progress) == false)
1577          mergeFailure = true;
1578    };
1579
1580    if (List !=  NULL)
1581    {
1582       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1583       {
1584          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1585          {
1586             _error->Warning("Duplicate sources.list entry %s",
1587                   (*i)->Describe().c_str());
1588             continue;
1589          }
1590
1591          if ((*i)->Merge(Gen, Progress) == false)
1592             return false;
1593
1594          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1595          if (Indexes != NULL)
1596             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1597          if (mergeFailure)
1598             return false;
1599       }
1600    }
1601
1602    if (Start != End)
1603    {
1604       Gen.SelectReleaseFile("", "");
1605       std::for_each(Start, End, indexFileMerge);
1606       if (mergeFailure)
1607          return false;
1608    }
1609    return true;
1610 }
1611                                                                         /*}}}*/
1612 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1613 // ---------------------------------------------------------------------
1614 /* This makes sure that the status cache (the cache that has all
1615    index files from the sources list and all local ones) is ready
1616    to be mmaped. If OutMap is not zero then a MMap object representing
1617    the cache will be stored there. This is pretty much mandetory if you
1618    are using AllowMem. AllowMem lets the function be run as non-root
1619    where it builds the cache 'fast' into a memory buffer. */
1620 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1621 {
1622    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1623    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1624    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1625    Flags |= MMap::Moveable;
1626    if (_config->FindB("APT::Cache-Fallback", false) == true)
1627       Flags |= MMap::Fallback;
1628    if (CacheF != NULL)
1629       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1630    else
1631       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1632 }
1633 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1634       std::string const &FileName)
1635 {
1636    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1637    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1638       return false;
1639
1640    fchmod(SCacheF.Fd(),0644);
1641
1642    // Write out the main data
1643    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1644       return _error->Error(_("IO Error saving source cache"));
1645
1646    // Write out the proper header
1647    Gen->GetCache().HeaderP->Dirty = false;
1648    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1649    if (SCacheF.Seek(0) == false ||
1650          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1651       return _error->Error(_("IO Error saving source cache"));
1652    Gen->GetCache().HeaderP->Dirty = true;
1653    return true;
1654 }
1655 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1656       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1657 {
1658    Map.reset(CreateDynamicMMap(NULL, 0));
1659    if (unlikely(Map->validData()) == false)
1660       return false;
1661    FileFd CacheF(FileName, FileFd::ReadOnly);
1662    if (CacheF.IsOpen() == false || CacheF.Failed())
1663       return false;
1664    _error->PushToStack();
1665    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1666    bool const newError = _error->PendingError();
1667    _error->MergeWithStack();
1668    if (alloc == 0 && newError)
1669       return false;
1670    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1671       return false;
1672    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1673    return Gen->Start();
1674 }
1675 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1676                         MMap **OutMap, bool AllowMem)
1677    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1678 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1679                         MMap **OutMap,bool)
1680 {
1681    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1682 }
1683 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1684                         MMap **OutMap,pkgCache **OutCache, bool)
1685 {
1686    // FIXME: deprecate the ignored AllowMem parameter
1687    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1688
1689    std::vector<pkgIndexFile *> Files;
1690    if (_system->AddStatusFiles(Files) == false)
1691       return false;
1692
1693    // Decide if we can write to the files..
1694    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1695    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1696
1697    // ensure the cache directory exists
1698    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1699    {
1700       string dir = _config->FindDir("Dir::Cache");
1701       size_t const len = dir.size();
1702       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1703          dir = dir.substr(0, len - 5);
1704       if (CacheFile.empty() == false)
1705          CreateDirectory(dir, flNotFile(CacheFile));
1706       if (SrcCacheFile.empty() == false)
1707          CreateDirectory(dir, flNotFile(SrcCacheFile));
1708    }
1709
1710    if (Progress != NULL)
1711       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1712
1713    bool pkgcache_fine = false;
1714    bool srcpkgcache_fine = false;
1715    bool volatile_fine = List.GetVolatileFiles().empty();
1716
1717    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1718                      volatile_fine ? OutCache : NULL) == true)
1719    {
1720       if (Debug == true)
1721          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1722       pkgcache_fine = true;
1723       srcpkgcache_fine = true;
1724    }
1725    if (pkgcache_fine == false)
1726    {
1727       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1728       {
1729          if (Debug == true)
1730             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1731          srcpkgcache_fine = true;
1732       }
1733    }
1734
1735    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1736    {
1737       if (Progress != NULL)
1738          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1739       return true;
1740    }
1741
1742    bool Writeable = false;
1743    if (srcpkgcache_fine == false || pkgcache_fine == false)
1744    {
1745       if (CacheFile.empty() == false)
1746          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1747       else if (SrcCacheFile.empty() == false)
1748          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1749
1750       if (Debug == true)
1751          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1752    }
1753
1754    // At this point we know we need to construct something, so get storage ready
1755    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1756    if (unlikely(Map->validData()) == false)
1757       return false;
1758    if (Debug == true)
1759       std::clog << "Open memory Map (not filebased)" << std::endl;
1760
1761    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1762    map_filesize_t CurrentSize = 0;
1763    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1764    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1765    if (srcpkgcache_fine == true && pkgcache_fine == false)
1766    {
1767       if (Debug == true)
1768          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1769       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1770          return false;
1771       srcpkgcache_fine = true;
1772       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1773    }
1774    else if (srcpkgcache_fine == false)
1775    {
1776       if (Debug == true)
1777          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1778       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1779       if (Gen->Start() == false)
1780          return false;
1781
1782       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1783       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1784                Files.end(),Files.end()) == false)
1785          return false;
1786
1787       if (Writeable == true && SrcCacheFile.empty() == false)
1788          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1789             return false;
1790    }
1791
1792    if (pkgcache_fine == false)
1793    {
1794       if (Debug == true)
1795          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1796       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1797                Files.begin(), Files.end()) == false)
1798          return false;
1799
1800       if (Writeable == true && CacheFile.empty() == false)
1801          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1802             return false;
1803    }
1804
1805    if (Debug == true)
1806       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1807
1808    if (volatile_fine == false)
1809    {
1810       if (Gen == nullptr)
1811       {
1812          if (Debug == true)
1813             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1814          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1815             return false;
1816       }
1817
1818       Files = List.GetVolatileFiles();
1819       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1820                Files.begin(), Files.end()) == false)
1821          return false;
1822    }
1823
1824    if (OutMap != nullptr)
1825       *OutMap = Map.release();
1826
1827    if (Debug == true)
1828       std::clog << "Everything is ready for shipping" << std::endl;
1829    return true;
1830 }
1831                                                                         /*}}}*/
1832 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1833 class APT_HIDDEN ScopedErrorMerge {
1834 public:
1835    ScopedErrorMerge() { _error->PushToStack(); }
1836    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1837 };
1838 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1839    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1840 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1841 {
1842    std::vector<pkgIndexFile *> Files;
1843    if (_system->AddStatusFiles(Files) == false)
1844       return false;
1845
1846    ScopedErrorMerge sem;
1847    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1848    if (unlikely(Map->validData()) == false)
1849       return false;
1850    map_filesize_t CurrentSize = 0;
1851    map_filesize_t TotalSize = 0;
1852    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1853
1854    // Build the status cache
1855    if (Progress != NULL)
1856       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1857    pkgCacheGenerator Gen(Map.get(),Progress);
1858    if (Gen.Start() == false || _error->PendingError() == true)
1859       return false;
1860    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1861                   Files.begin(), Files.end()) == false)
1862       return false;
1863
1864    if (_error->PendingError() == true)
1865       return false;
1866    *OutMap = Map.release();
1867
1868    return true;
1869 }
1870                                                                         /*}}}*/
1871 // IsDuplicateDescription                                               /*{{{*/
1872 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
1873                             APT::StringView CurMd5, std::string const &CurLang)
1874 {
1875    // Descriptions in the same link-list have all the same md5
1876    if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1877       return false;
1878    for (; Desc.end() == false; ++Desc)
1879       if (Desc.LanguageCode() == CurLang)
1880          return true;
1881    return false;
1882 }
1883                                                                         /*}}}*/
1884
1885 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1886 pkgCacheListParser::~pkgCacheListParser() {}