apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
  49                             APT::StringView CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return _error->ReturnError();
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::TagIterator*>::const_iterator i = Dynamic<pkgCache::TagIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::TagIterator>::toReMap.end(); ++i)
 180       if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 184         if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 199         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 200       if (std::get<1>(seen.insert(*i)) == true)
 201          (*i)->ReMap(oldMap, newMap);
 202    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 203       if (std::get<1>(seen.insert(ViewP)) == false)
 204          continue;
 205       // Ignore views outside of the cache.
 206       if (ViewP->data() < static_cast<const char*>(oldMap)
 207          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 208          continue;
 209       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 210       *ViewP = StringView(data , ViewP->size());
 211    }
 212 }                                                                       /*}}}*/
 213 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 214 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 215                                         const unsigned long &Len) {
 216    size_t oldSize = Map.Size();
 217    void const * const oldMap = Map.Data();
 218    map_stringitem_t const index = Map.WriteString(String, Len);
 219    if (index != 0)
 220       ReMap(oldMap, Map.Data(), oldSize);
 221    return index;
 222 }
 223                                                                         /*}}}*/
 224 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 225 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 226    size_t oldSize = Map.Size();
 227    void const * const oldMap = Map.Data();
 228    map_stringitem_t const index = Map.WriteString(String);
 229    if (index != 0)
 230       ReMap(oldMap, Map.Data(), oldSize);
 231    return index;
 232 }
 233                                                                         /*}}}*/
 234 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 235    size_t oldSize = Map.Size();
 236    void const * const oldMap = Map.Data();
 237    map_pointer_t const index = Map.Allocate(size);
 238    if (index != 0)
 239       ReMap(oldMap, Map.Data(), oldSize);
 240    return index;
 241 }
 242                                                                         /*}}}*/
 243 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 244 // ---------------------------------------------------------------------
 245 /* This provides the generation of the entries in the cache. Each loop
 246    goes through a single package record from the underlying parse engine. */
 247 bool pkgCacheGenerator::MergeList(ListParser &List,
 248                                   pkgCache::VerIterator *OutVer)
 249 {
 250    List.Owner = this;
 251
 252    unsigned int Counter = 0;
 253    while (List.Step() == true)
 254    {
 255       string const PackageName = List.Package();
 256       if (PackageName.empty() == true)
 257           continue;
 258
 259       Counter++;
 260       if (Counter % 100 == 0 && Progress != 0)
 261          Progress->Progress(List.Offset());
 262
 263       APT::StringView Arch = List.Architecture();
 264       Dynamic<APT::StringView> DynArch(Arch);
 265       APT::StringView Version = List.Version();
 266       Dynamic<APT::StringView> DynVersion(Version);
 267       if (Version.empty() == true && Arch.empty() == true)
 268       {
 269          // package descriptions
 270          if (MergeListGroup(List, PackageName) == false)
 271             continue;
 272          continue;
 273       }
 274
 275       // Get a pointer to the package structure
 276       pkgCache::PkgIterator Pkg;
 277       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 278       if (NewPackage(Pkg, PackageName, Arch) == false) {
 279          // TRANSLATOR: The first placeholder is a package name,
 280          // the other two should be copied verbatim as they include debug info
 281          _error->Error(_("Error occurred while processing %s (%s%d)"),
 282                               PackageName.c_str(), "NewPackage", 1);
 283          continue;
 284       }
 285
 286
 287       if (Version.empty() == true)
 288       {
 289          if (MergeListPackage(List, Pkg) == false)
 290             continue;
 291       }
 292       else
 293       {
 294          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 295             return false;
 296       }
 297
 298       if (OutVer != 0)
 299          return true;
 300    }
 301
 302    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 303       return _error->Error(_("Wow, you exceeded the number of package "
 304                              "names this APT is capable of."));
 305    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 306       return _error->Error(_("Wow, you exceeded the number of versions "
 307                              "this APT is capable of."));
 308    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 309       return _error->Error(_("Wow, you exceeded the number of descriptions "
 310                              "this APT is capable of."));
 311    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 312       return _error->Error(_("Wow, you exceeded the number of dependencies "
 313                              "this APT is capable of."));
 314
 315    return true;
 316 }
 317 // CacheGenerator::MergeListGroup                                       /*{{{*/
 318 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 319 {
 320    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 321    // a group has no data on it's own, only packages have it but these
 322    // stanzas like this come from Translation- files to add descriptions,
 323    // but without a version we don't need a description for it…
 324    if (Grp.end() == true)
 325       return true;
 326    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 327
 328    pkgCache::PkgIterator Pkg;
 329    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 330    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 331       if (MergeListPackage(List, Pkg) == false)
 332          return false;
 333
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // CacheGenerator::MergeListPackage                                     /*{{{*/
 338 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 339 {
 340    // we first process the package, then the descriptions
 341    // (for deb this package processing is in fact a no-op)
 342    pkgCache::VerIterator Ver(Cache);
 343    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 344    if (List.UsePackage(Pkg, Ver) == false)
 345       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 346                            Pkg.Name(), "UsePackage", 1);
 347
 348    // Find the right version to write the description
 349    StringView CurMd5 = List.Description_md5();
 350    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 351    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 352    {
 353       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 354
 355       // a version can only have one md5 describing it
 356       if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
 357          continue;
 358
 359       map_stringitem_t md5idx = VerDesc->md5sum;
 360       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 361       {
 362          // don't add a new description if we have one for the given
 363          // md5 && language
 364          if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
 365             continue;
 366
 367          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 368       }
 369
 370       // we can stop here as all "same" versions will share the description
 371       break;
 372    }
 373
 374    return true;
 375 }
 376                                                                         /*}}}*/
 377 // CacheGenerator::MergeListVersion                                     /*{{{*/
 378 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 379                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 380 {
 381    pkgCache::VerIterator Ver = Pkg.VersionList();
 382    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 383    map_pointer_t *LastVer = &Pkg->VersionList;
 384    void const * oldMap = Map.Data();
 385
 386    unsigned short const Hash = List.VersionHash();
 387    if (Ver.end() == false)
 388    {
 389       /* We know the list is sorted so we use that fact in the search.
 390          Insertion of new versions is done with correct sorting */
 391       int Res = 1;
 392       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 393       {
 394          char const * const VerStr = Ver.VerStr();
 395          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 396                VerStr, VerStr + strlen(VerStr));
 397          // Version is higher as current version - insert here
 398          if (Res > 0)
 399             break;
 400          // Versionstrings are equal - is hash also equal?
 401          if (Res == 0)
 402          {
 403             if (List.SameVersion(Hash, Ver) == true)
 404                break;
 405             // sort (volatile) sources above not-sources like the status file
 406             if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 407             {
 408                auto VF = Ver.FileList();
 409                for (; VF.end() == false; ++VF)
 410                   if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
 411                      break;
 412                if (VF.end() == true)
 413                   break;
 414             }
 415          }
 416          // proceed with the next till we have either the right
 417          // or we found another version (which will be lower)
 418       }
 419
 420       /* We already have a version for this item, record that we saw it */
 421       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 422       {
 423          if (List.UsePackage(Pkg,Ver) == false)
 424             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                                  Pkg.Name(), "UsePackage", 2);
 426
 427          if (NewFileVer(Ver,List) == false)
 428             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 429                                  Pkg.Name(), "NewFileVer", 1);
 430
 431          // Read only a single record and return
 432          if (OutVer != 0)
 433          {
 434             *OutVer = Ver;
 435             return true;
 436          }
 437
 438          return true;
 439       }
 440    }
 441
 442    // Add a new version
 443    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 444    if (unlikely(verindex == 0))
 445       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 446                            Pkg.Name(), "NewVersion", 1);
 447
 448    if (oldMap != Map.Data())
 449          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 450    *LastVer = verindex;
 451
 452    if (unlikely(List.NewVersion(Ver) == false))
 453       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 454                            Pkg.Name(), "NewVersion", 2);
 455
 456    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 457       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 458                            Pkg.Name(), "UsePackage", 3);
 459
 460    if (unlikely(NewFileVer(Ver,List) == false))
 461       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 462                            Pkg.Name(), "NewFileVer", 2);
 463
 464    pkgCache::GrpIterator Grp = Pkg.Group();
 465    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 466
 467    /* If it is the first version of this package we need to add implicit
 468       Multi-Arch dependencies to all other package versions in the group now -
 469       otherwise we just add them for this new version */
 470    if (Pkg.VersionList()->NextVer == 0)
 471    {
 472       pkgCache::PkgIterator P = Grp.PackageList();
 473       Dynamic<pkgCache::PkgIterator> DynP(P);
 474       for (; P.end() != true; P = Grp.NextPkg(P))
 475       {
 476          if (P->ID == Pkg->ID)
 477             continue;
 478          pkgCache::VerIterator V = P.VersionList();
 479          Dynamic<pkgCache::VerIterator> DynV(V);
 480          for (; V.end() != true; ++V)
 481             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 482                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 483                                     Pkg.Name(), "AddImplicitDepends", 1);
 484       }
 485    }
 486    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 487       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 488                            Pkg.Name(), "AddImplicitDepends", 2);
 489
 490    // Read only a single record and return
 491    if (OutVer != 0)
 492    {
 493       *OutVer = Ver;
 494       return true;
 495    }
 496
 497    /* Record the Description(s) based on their master md5sum */
 498    StringView CurMd5 = List.Description_md5();
 499
 500    /* Before we add a new description we first search in the group for
 501       a version with a description of the same MD5 - if so we reuse this
 502       description group instead of creating our own for this version */
 503    for (pkgCache::PkgIterator P = Grp.PackageList();
 504         P.end() == false; P = Grp.NextPkg(P))
 505    {
 506       for (pkgCache::VerIterator V = P.VersionList();
 507            V.end() == false; ++V)
 508       {
 509          if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
 510             continue;
 511          Ver->DescriptionList = V->DescriptionList;
 512       }
 513    }
 514
 515    // We haven't found reusable descriptions, so add the first description(s)
 516    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 517    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 518    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 519       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 520          return false;
 521    return true;
 522 }
 523                                                                         /*}}}*/
 524 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 525 {
 526    pkgCache::DescIterator Desc;
 527    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 528
 529    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 530    if (unlikely(descindex == 0))
 531       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 532             Ver.ParentPkg().Name(), "NewDescription", 1);
 533
 534    md5idx = Desc->md5sum;
 535    Desc->ParentPkg = Ver.ParentPkg().Index();
 536
 537    // we add at the end, so that the start is constant as we need
 538    // that to be able to efficiently share these lists
 539    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 540    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 541    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 542    *LastNextDesc = descindex;
 543
 544    if (NewFileDesc(Desc,List) == false)
 545       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 546             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 547
 548    return true;
 549 }
 550                                                                         /*}}}*/
 551                                                                         /*}}}*/
 552 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* This creates a new group structure and adds it to the hash table */
 555 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 556 {
 557    Dynamic<StringView> DName(Name);
 558    Grp = Cache.FindGrp(Name);
 559    if (Grp.end() == false)
 560       return true;
 561
 562    // Get a structure
 563    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 564    if (unlikely(Group == 0))
 565       return false;
 566
 567    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 568    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 569    if (unlikely(idxName == 0))
 570       return false;
 571    Grp->Name = idxName;
 572
 573    // Insert it into the hash table
 574    unsigned long const Hash = Cache.Hash(Name);
 575    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 576
 577    while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 578       insertAt = &(Cache.GrpP + *insertAt)->Next;
 579    Grp->Next = *insertAt;
 580    *insertAt = Group;
 581
 582    Grp->ID = Cache.HeaderP->GroupCount++;
 583    return true;
 584 }
 585                                                                         /*}}}*/
 586 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 587 // ---------------------------------------------------------------------
 588 /* This creates a new package structure and adds it to the hash table */
 589 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 590                                         StringView Arch) {
 591    pkgCache::GrpIterator Grp;
 592    Dynamic<StringView> DName(Name);
 593    Dynamic<StringView> DArch(Arch);
 594    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 595    if (unlikely(NewGroup(Grp, Name) == false))
 596       return false;
 597
 598    Pkg = Grp.FindPkg(Arch);
 599       if (Pkg.end() == false)
 600          return true;
 601
 602    // Get a structure
 603    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 604    if (unlikely(Package == 0))
 605       return false;
 606    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 607
 608    // Set the name, arch and the ID
 609    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 610    Pkg->Group = Grp.Index();
 611    // all is mapped to the native architecture
 612    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 613    if (unlikely(idxArch == 0))
 614       return false;
 615    Pkg->Arch = idxArch;
 616    Pkg->ID = Cache.HeaderP->PackageCount++;
 617
 618    // Insert the package into our package list
 619    if (Grp->FirstPackage == 0) // the group is new
 620    {
 621       Grp->FirstPackage = Package;
 622       // Insert it into the hash table
 623       map_id_t const Hash = Cache.Hash(Name);
 624       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 625       while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
 626          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 627       Pkg->NextPackage = *insertAt;
 628       *insertAt = Package;
 629    }
 630    else // Group the Packages together
 631    {
 632       // if sibling is provided by another package, this one is too
 633       {
 634          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 635          if (M.end() == false) {
 636             pkgCache::PrvIterator Prv;
 637             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 638             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 639             {
 640                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 641                   continue;
 642                pkgCache::VerIterator Ver = Prv.OwnerVer();
 643                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 644                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 645                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 646                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 647                {
 648                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 649                      continue;
 650                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 651                      return false;
 652                }
 653             }
 654          }
 655       }
 656       // let M-A:foreign package siblings provide this package
 657       {
 658          pkgCache::PkgIterator P;
 659          pkgCache::VerIterator Ver;
 660          Dynamic<pkgCache::PkgIterator> DynP(P);
 661          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 662
 663          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 664          {
 665             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 666                continue;
 667             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 668                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 669                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 670                      return false;
 671          }
 672       }
 673       // and negative dependencies, don't forget negative dependencies
 674       {
 675          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 676          if (M.end() == false) {
 677             pkgCache::DepIterator Dep;
 678             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 679             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 680             {
 681                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 682                   continue;
 683                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 684                      Dep->Type != pkgCache::Dep::Replaces)
 685                   continue;
 686                pkgCache::VerIterator Ver = Dep.ParentVer();
 687                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 688                map_pointer_t * unused = NULL;
 689                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 690                   return false;
 691             }
 692          }
 693       }
 694
 695       // this package is the new last package
 696       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 697       Pkg->NextPackage = LastPkg->NextPackage;
 698       LastPkg->NextPackage = Package;
 699    }
 700    Grp->LastPackage = Package;
 701
 702    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 703    if (Arch == "any")
 704    {
 705       size_t const found = Name.rfind(':');
 706       StringView ArchA = Name.substr(found + 1);
 707       if (ArchA != "any")
 708       {
 709          // ArchA is used inside the loop which might remap (NameA is not used)
 710          Dynamic<StringView> DynArchA(ArchA);
 711          StringView NameA = Name.substr(0, found);
 712          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 713          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 714          if (PkgA.end())
 715          {
 716             Dynamic<StringView> DynNameA(NameA);
 717             if (NewPackage(PkgA, NameA, ArchA) == false)
 718                return false;
 719          }
 720          if (unlikely(PkgA.end()))
 721             return _error->Fatal("NewPackage was successful for %s:%s,"
 722                   "but the package doesn't exist anyhow!",
 723                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 724          else
 725          {
 726             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 727             for (; Prv.end() == false; ++Prv)
 728             {
 729                if (Prv.IsMultiArchImplicit())
 730                   continue;
 731                pkgCache::VerIterator V = Prv.OwnerVer();
 732                if (ArchA != V.ParentPkg().Arch())
 733                   continue;
 734                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 735                   return false;
 736             }
 737             pkgCache::VerIterator V = PkgA.VersionList();
 738             Dynamic<pkgCache::VerIterator> DynV(V);
 739             for (; V.end() == false; ++V)
 740             {
 741                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 742                   return false;
 743             }
 744          }
 745       }
 746    }
 747    return true;
 748 }
 749                                                                         /*}}}*/
 750 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 751 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 752                                            pkgCache::PkgIterator &P,
 753                                            pkgCache::VerIterator &V)
 754 {
 755    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 756    Dynamic<APT::StringView> DynArch(Arch);
 757    map_pointer_t *OldDepLast = NULL;
 758    /* MultiArch handling introduces a lot of implicit Dependencies:
 759       - MultiArch: same → Co-Installable if they have the same version
 760       - All others conflict with all other group members */
 761    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 762    pkgCache::PkgIterator D = G.PackageList();
 763    Dynamic<pkgCache::PkgIterator> DynD(D);
 764    map_stringitem_t const VerStrIdx = V->VerStr;
 765    for (; D.end() != true; D = G.NextPkg(D))
 766    {
 767       if (Arch == D.Arch() || D->VersionList == 0)
 768          continue;
 769       /* We allow only one installed arch at the time
 770          per group, therefore each group member conflicts
 771          with all other group members */
 772       if (coInstall == true)
 773       {
 774          // Replaces: ${self}:other ( << ${binary:Version})
 775          NewDepends(D, V, VerStrIdx,
 776                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 777                     OldDepLast);
 778          // Breaks: ${self}:other (!= ${binary:Version})
 779          NewDepends(D, V, VerStrIdx,
 780                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 781                     OldDepLast);
 782       } else {
 783          // Conflicts: ${self}:other
 784          NewDepends(D, V, 0,
 785                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 786                     OldDepLast);
 787       }
 788    }
 789    return true;
 790 }
 791 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 792                                            pkgCache::PkgIterator &D)
 793 {
 794    /* MultiArch handling introduces a lot of implicit Dependencies:
 795       - MultiArch: same → Co-Installable if they have the same version
 796       - All others conflict with all other group members */
 797    map_pointer_t *OldDepLast = NULL;
 798    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 799    if (coInstall == true)
 800    {
 801       map_stringitem_t const VerStrIdx = V->VerStr;
 802       // Replaces: ${self}:other ( << ${binary:Version})
 803       NewDepends(D, V, VerStrIdx,
 804                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 805                  OldDepLast);
 806       // Breaks: ${self}:other (!= ${binary:Version})
 807       NewDepends(D, V, VerStrIdx,
 808                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 809                  OldDepLast);
 810    } else {
 811       // Conflicts: ${self}:other
 812       NewDepends(D, V, 0,
 813                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 814                  OldDepLast);
 815    }
 816    return true;
 817 }
 818
 819                                                                         /*}}}*/
 820 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 821 // ---------------------------------------------------------------------
 822 /* */
 823 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 824                                    ListParser &List)
 825 {
 826    if (CurrentFile == 0)
 827       return true;
 828
 829    // Get a structure
 830    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 831    if (VerFile == 0)
 832       return false;
 833
 834    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 835    VF->File = CurrentFile - Cache.PkgFileP;
 836
 837    // Link it to the end of the list
 838    map_pointer_t *Last = &Ver->FileList;
 839    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 840       Last = &V->NextFile;
 841    VF->NextFile = *Last;
 842    *Last = VF.Index();
 843
 844    VF->Offset = List.Offset();
 845    VF->Size = List.Size();
 846    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 847       Cache.HeaderP->MaxVerFileSize = VF->Size;
 848    Cache.HeaderP->VerFileCount++;
 849
 850    return true;
 851 }
 852                                                                         /*}}}*/
 853 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 854 // ---------------------------------------------------------------------
 855 /* This puts a version structure in the linked list */
 856 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 857                                             APT::StringView const &VerStr,
 858                                             map_pointer_t const ParentPkg,
 859                                             unsigned short const Hash,
 860                                             map_pointer_t const Next)
 861 {
 862    // Get a structure
 863    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 864    if (Version == 0)
 865       return 0;
 866
 867    // Fill it in
 868    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 869    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 870    Ver->NextVer = Next;
 871    Ver->ParentPkg = ParentPkg;
 872    Ver->Hash = Hash;
 873    Ver->ID = Cache.HeaderP->VersionCount++;
 874
 875    // try to find the version string in the group for reuse
 876    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 877    pkgCache::GrpIterator Grp = Pkg.Group();
 878    if (Pkg.end() == false && Grp.end() == false)
 879    {
 880       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 881       {
 882          if (Pkg == P)
 883             continue;
 884          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 885          {
 886             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 887             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 888             {
 889                Ver->VerStr = V->VerStr;
 890                return Version;
 891             }
 892             else if (cmp < 0)
 893                break;
 894          }
 895       }
 896    }
 897    // haven't found the version string, so create
 898    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 899    if (unlikely(idxVerStr == 0))
 900       return 0;
 901    Ver->VerStr = idxVerStr;
 902    return Version;
 903 }
 904                                                                         /*}}}*/
 905 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* */
 908 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 909                                    ListParser &List)
 910 {
 911    if (CurrentFile == 0)
 912       return true;
 913
 914    // Get a structure
 915    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 916    if (DescFile == 0)
 917       return false;
 918
 919    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 920    DF->File = CurrentFile - Cache.PkgFileP;
 921
 922    // Link it to the end of the list
 923    map_pointer_t *Last = &Desc->FileList;
 924    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 925       Last = &D->NextFile;
 926
 927    DF->NextFile = *Last;
 928    *Last = DF.Index();
 929
 930    DF->Offset = List.Offset();
 931    DF->Size = List.Size();
 932    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 933       Cache.HeaderP->MaxDescFileSize = DF->Size;
 934    Cache.HeaderP->DescFileCount++;
 935
 936    return true;
 937 }
 938                                                                         /*}}}*/
 939 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* This puts a description structure in the linked list */
 942 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 943                                             const string &Lang,
 944                                             APT::StringView md5sum,
 945                                             map_stringitem_t const idxmd5str)
 946 {
 947    // Get a structure
 948    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 949    if (Description == 0)
 950       return 0;
 951
 952    // Fill it in
 953    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 954    Desc->ID = Cache.HeaderP->DescriptionCount++;
 955    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 956    if (unlikely(idxlanguage_code == 0))
 957       return 0;
 958    Desc->language_code = idxlanguage_code;
 959
 960    if (idxmd5str != 0)
 961       Desc->md5sum = idxmd5str;
 962    else
 963    {
 964       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
 965       if (unlikely(idxmd5sum == 0))
 966          return 0;
 967       Desc->md5sum = idxmd5sum;
 968    }
 969
 970    return Description;
 971 }
 972                                                                         /*}}}*/
 973 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 974 // ---------------------------------------------------------------------
 975 /* This creates a dependency element in the tree. It is linked to the
 976    version and to the package that it is pointing to. */
 977 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 978                                    pkgCache::VerIterator &Ver,
 979                                    map_pointer_t const Version,
 980                                    uint8_t const Op,
 981                                    uint8_t const Type,
 982                                    map_pointer_t* &OldDepLast)
 983 {
 984    void const * const oldMap = Map.Data();
 985    // Get a structure
 986    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 987    if (unlikely(Dependency == 0))
 988       return false;
 989
 990    bool isDuplicate = false;
 991    map_pointer_t DependencyData = 0;
 992    map_pointer_t PreviousData = 0;
 993    if (Pkg->RevDepends != 0)
 994    {
 995       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 996       DependencyData = L->DependencyData;
 997       do {
 998          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 999          if (Version > D->Version)
1000             break;
1001          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
1002          {
1003             isDuplicate = true;
1004             break;
1005          }
1006          PreviousData = DependencyData;
1007          DependencyData = D->NextData;
1008       } while (DependencyData != 0);
1009    }
1010
1011    if (isDuplicate == false)
1012    {
1013       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1014       if (unlikely(DependencyData == 0))
1015         return false;
1016    }
1017
1018    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1019    Link->ParentVer = Ver.Index();
1020    Link->DependencyData = DependencyData;
1021    Link->ID = Cache.HeaderP->DependsCount++;
1022
1023    pkgCache::DepIterator Dep(Cache, Link);
1024    if (isDuplicate == false)
1025    {
1026       Dep->Type = Type;
1027       Dep->CompareOp = Op;
1028       Dep->Version = Version;
1029       Dep->Package = Pkg.Index();
1030       ++Cache.HeaderP->DependsDataCount;
1031       if (PreviousData != 0)
1032       {
1033          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1034          Dep->NextData = D->NextData;
1035          D->NextData = DependencyData;
1036       }
1037       else if (Pkg->RevDepends != 0)
1038       {
1039          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1040          Dep->NextData = D->DependencyData;
1041       }
1042    }
1043
1044    if (isDuplicate == true || PreviousData != 0)
1045    {
1046       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1047       Link->NextRevDepends = L->NextRevDepends;
1048       L->NextRevDepends = Dependency;
1049    }
1050    else
1051    {
1052       Link->NextRevDepends = Pkg->RevDepends;
1053       Pkg->RevDepends = Dependency;
1054    }
1055
1056
1057    // Do we know where to link the Dependency to?
1058    if (OldDepLast == NULL)
1059    {
1060       OldDepLast = &Ver->DependsList;
1061       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1062          OldDepLast = &D->NextDepends;
1063    } else if (oldMap != Map.Data())
1064       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1065
1066    Dep->NextDepends = *OldDepLast;
1067    *OldDepLast = Dependency;
1068    OldDepLast = &Dep->NextDepends;
1069    return true;
1070 }
1071                                                                         /*}}}*/
1072 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1073 // ---------------------------------------------------------------------
1074 /* This creates a Group and the Package to link this dependency to if
1075    needed and handles also the caching of the old endpoint */
1076 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1077                                                StringView PackageName,
1078                                                StringView Arch,
1079                                                StringView Version,
1080                                                uint8_t const Op,
1081                                                uint8_t const Type)
1082 {
1083    pkgCache::GrpIterator Grp;
1084    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1085    Dynamic<StringView> DynPackageName(PackageName);
1086    Dynamic<StringView> DynArch(Arch);
1087    Dynamic<StringView> DynVersion(Version);
1088    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1089       return false;
1090
1091    map_stringitem_t idxVersion = 0;
1092    if (Version.empty() == false)
1093    {
1094       int const CmpOp = Op & 0x0F;
1095       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1096       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1097          idxVersion = Ver->VerStr;
1098
1099       if (idxVersion == 0)
1100       {
1101          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1102          if (unlikely(idxVersion == 0))
1103             return false;
1104       }
1105    }
1106
1107    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1108          Type == pkgCache::Dep::Conflicts ||
1109          Type == pkgCache::Dep::Replaces);
1110
1111    pkgCache::PkgIterator Pkg;
1112    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1113    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1114    {
1115       // Locate the target package
1116       Pkg = Grp.FindPkg(Arch);
1117       if (Pkg.end() == true) {
1118          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1119             return false;
1120       }
1121
1122       /* Caching the old end point speeds up generation substantially */
1123       if (OldDepVer != Ver) {
1124          OldDepLast = NULL;
1125          OldDepVer = Ver;
1126       }
1127
1128       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1129    }
1130    else
1131    {
1132       /* Caching the old end point speeds up generation substantially */
1133       if (OldDepVer != Ver) {
1134          OldDepLast = NULL;
1135          OldDepVer = Ver;
1136       }
1137
1138       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1139       {
1140          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1141                return false;
1142       }
1143    }
1144    return true;
1145 }
1146                                                                         /*}}}*/
1147 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1148 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1149                                                 StringView PkgName,
1150                                                 StringView PkgArch,
1151                                                 StringView Version,
1152                                                 uint8_t const Flags)
1153 {
1154    pkgCache const &Cache = Owner->Cache;
1155    Dynamic<StringView> DynPkgName(PkgName);
1156    Dynamic<StringView> DynArch(PkgArch);
1157    Dynamic<StringView> DynVersion(Version);
1158
1159    // We do not add self referencing provides
1160    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1161         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1162          (Version.empty() || Version == Ver.VerStr()))
1163       return true;
1164
1165    // Locate the target package
1166    pkgCache::PkgIterator Pkg;
1167    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1168    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1169       return false;
1170
1171    map_stringitem_t idxProvideVersion = 0;
1172    if (Version.empty() == false) {
1173       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1174       if (unlikely(idxProvideVersion == 0))
1175          return false;
1176    }
1177    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1178 }
1179 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1180                                     pkgCache::PkgIterator &Pkg,
1181                                     map_pointer_t const ProvideVersion,
1182                                     uint8_t const Flags)
1183 {
1184    // Get a structure
1185    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1186    if (unlikely(Provides == 0))
1187       return false;
1188    ++Cache.HeaderP->ProvidesCount;
1189
1190    // Fill it in
1191    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1192    Prv->Version = Ver.Index();
1193    Prv->ProvideVersion = ProvideVersion;
1194    Prv->Flags = Flags;
1195    Prv->NextPkgProv = Ver->ProvidesList;
1196    Ver->ProvidesList = Prv.Index();
1197
1198    // Link it to the package
1199    Prv->ParentPkg = Pkg.Index();
1200    Prv->NextProvides = Pkg->ProvidesList;
1201    Pkg->ProvidesList = Prv.Index();
1202    return true;
1203 }
1204                                                                         /*}}}*/
1205 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1206 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1207                                 StringView Version, uint8_t const Flags) {
1208    pkgCache &Cache = Owner->Cache;
1209    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1210    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1211    Dynamic<StringView> DynPackage(Package);
1212    Dynamic<StringView> DynVersion(Version);
1213
1214    if (Grp.end() == true)
1215       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1216    else
1217    {
1218       map_stringitem_t idxProvideVersion = 0;
1219       if (Version.empty() == false) {
1220          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1221          if (unlikely(idxProvideVersion == 0))
1222             return false;
1223       }
1224
1225       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1226       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1227       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1228       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1229       pkgCache::PkgIterator Pkg;
1230       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1231       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1232       {
1233          if (isImplicit && OwnerPkg == Pkg)
1234             continue;
1235          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1236             continue;
1237          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1238             return false;
1239       }
1240    }
1241    return true;
1242 }
1243                                                                         /*}}}*/
1244 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1245       pkgCache::VerIterator const &Ver)
1246 {
1247    return Hash == Ver->Hash;
1248 }
1249                                                                         /*}}}*/
1250 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1251 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1252                                    unsigned long Flags)
1253 {
1254    if (File.empty() && Site.empty())
1255    {
1256       CurrentRlsFile = NULL;
1257       return true;
1258    }
1259
1260    // Get some space for the structure
1261    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1262    if (unlikely(idxFile == 0))
1263       return false;
1264    CurrentRlsFile = Cache.RlsFileP + idxFile;
1265
1266    // Fill it in
1267    map_stringitem_t const idxFileName = WriteStringInMap(File);
1268    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1269    if (unlikely(idxFileName == 0 || idxSite == 0))
1270       return false;
1271    CurrentRlsFile->FileName = idxFileName;
1272    CurrentRlsFile->Site = idxSite;
1273    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1274    CurrentRlsFile->Flags = Flags;
1275    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1276    RlsFileName = File;
1277    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1278    Cache.HeaderP->ReleaseFileCount++;
1279
1280    return true;
1281 }
1282                                                                         /*}}}*/
1283 // ListParser::NewTag - Create a Tag element                            /*{{{*/
1284 // ---------------------------------------------------------------------
1285 /* */
1286 bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
1287                                            const char *NameStart,
1288                                            unsigned int NameSize)
1289 {
1290    return Owner->NewTag(Ver, NameStart, NameSize);
1291 }
1292 bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
1293                                            const char *NameStart,
1294                                            unsigned int NameSize)
1295 {
1296    // Get a structure
1297    map_pointer_t const idxTag = AllocateInMap(sizeof(pkgCache::Tag));
1298    if (unlikely(idxTag == 0))
1299       return false;
1300
1301    // Fill it in
1302    pkgCache::TagIterator Tg(Cache,Cache.TagP + idxTag);
1303    map_pointer_t const idxName = StoreString(TAG,NameStart,NameSize);
1304    if (idxName == 0)
1305       return false;
1306    Tg->Name = idxName;
1307
1308    Tg->NextTag = Ver->TagList;
1309    Ver->TagList = Tg.Index();
1310    Cache.HeaderP->TagCount++;
1311
1312    return true;
1313 }
1314                                                                         /*}}}*/
1315 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1316 // ---------------------------------------------------------------------
1317 /* This is used to select which file is to be associated with all newly
1318    added versions. The caller is responsible for setting the IMS fields. */
1319 bool pkgCacheGenerator::SelectFile(std::string const &File,
1320                                    pkgIndexFile const &Index,
1321                                    std::string const &Architecture,
1322                                    std::string const &Component,
1323                                    unsigned long const Flags)
1324 {
1325    // Get some space for the structure
1326    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1327    if (unlikely(idxFile == 0))
1328       return false;
1329    CurrentFile = Cache.PkgFileP + idxFile;
1330
1331    // Fill it in
1332    map_stringitem_t const idxFileName = WriteStringInMap(File);
1333    if (unlikely(idxFileName == 0))
1334       return false;
1335    CurrentFile->FileName = idxFileName;
1336    CurrentFile->NextFile = Cache.HeaderP->FileList;
1337    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1338    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1339    if (unlikely(idxIndexType == 0))
1340       return false;
1341    CurrentFile->IndexType = idxIndexType;
1342    if (Architecture.empty())
1343       CurrentFile->Architecture = 0;
1344    else
1345    {
1346       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1347       if (unlikely(arch == 0))
1348          return false;
1349       CurrentFile->Architecture = arch;
1350    }
1351    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1352    if (unlikely(component == 0))
1353       return false;
1354    CurrentFile->Component = component;
1355    CurrentFile->Flags = Flags;
1356    if (CurrentRlsFile != NULL)
1357       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1358    else
1359       CurrentFile->Release = 0;
1360    PkgFileName = File;
1361    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1362    Cache.HeaderP->PackageFileCount++;
1363
1364    if (Progress != 0)
1365       Progress->SubProgress(Index.Size());
1366    return true;
1367 }
1368                                                                         /*}}}*/
1369 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1370 // ---------------------------------------------------------------------
1371 /* This is used to create handles to strings. Given the same text it
1372    always returns the same number */
1373 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1374                                                  unsigned int Size)
1375 {
1376    auto strings = &strMixed;
1377    switch(type) {
1378       case MIXED: strings = &strMixed; break;
1379       case PKGNAME: strings = &strPkgNames; break;
1380       case VERSIONNUMBER: strings = &strVersions; break;
1381       case SECTION: strings = &strSections; break;
1382       case TAG: strings = &strTags; break;
1383       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1384    }
1385
1386    auto const item = strings->find({S, Size, nullptr, 0});
1387    if (item != strings->end())
1388       return item->item;
1389
1390    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1391    strings->insert({nullptr, Size, this, idxString});
1392    return idxString;
1393 }
1394                                                                         /*}}}*/
1395 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1396 // ---------------------------------------------------------------------
1397 /* This just verifies that each file in the list of index files exists,
1398    has matching attributes with the cache and the cache does not have
1399    any extra files. */
1400 class APT_HIDDEN ScopedErrorRevert {
1401 public:
1402    ScopedErrorRevert() { _error->PushToStack(); }
1403    ~ScopedErrorRevert() { _error->RevertToStack(); }
1404 };
1405 static bool CheckValidity(const string &CacheFile,
1406                           pkgSourceList &List,
1407                           FileIterator const Start,
1408                           FileIterator const End,
1409                           MMap **OutMap = 0,
1410                           pkgCache **OutCache = 0)
1411 {
1412    ScopedErrorRevert ser;
1413    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1414    // No file, certainly invalid
1415    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1416    {
1417       if (Debug == true)
1418          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1419       return false;
1420    }
1421
1422    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1423    {
1424       if (Debug == true)
1425          std::clog << "sources.list is newer than the cache" << std::endl;
1426       return false;
1427    }
1428
1429    // Map it
1430    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1431    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1432    if (unlikely(Map->validData()) == false)
1433       return false;
1434    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1435    pkgCache &Cache = *CacheP.get();
1436    if (_error->PendingError() || Map->Size() == 0)
1437    {
1438       if (Debug == true)
1439          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1440       return _error->ReturnError();
1441    }
1442
1443    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1444    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1445    std::vector<pkgIndexFile *> Files;
1446    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1447    {
1448       if (Debug == true)
1449          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1450       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1451       if (RlsFile.end() == true)
1452       {
1453          if (Debug == true)
1454             std::clog << "FindInCache returned end-Pointer" << std::endl;
1455          return false;
1456       }
1457
1458       RlsVisited[RlsFile->ID] = true;
1459       if (Debug == true)
1460          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1461
1462       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1463       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1464             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1465    }
1466    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1467       if (RlsVisited[I] == false)
1468       {
1469          if (Debug == true)
1470             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1471          return false;
1472       }
1473
1474    std::copy(Start, End, std::back_inserter(Files));
1475
1476    /* Now we check every index file, see if it is in the cache,
1477       verify the IMS data and check that it is on the disk too.. */
1478    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1479    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1480    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1481    {
1482       if (Debug == true)
1483          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1484       if ((*PkgFile)->Exists() == false)
1485       {
1486          if (Debug == true)
1487             std::clog << "file doesn't exist" << std::endl;
1488          continue;
1489       }
1490
1491       // FindInCache is also expected to do an IMS check.
1492       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1493       if (File.end() == true)
1494       {
1495          if (Debug == true)
1496             std::clog << "FindInCache returned end-Pointer" << std::endl;
1497          return false;
1498       }
1499
1500       Visited[File->ID] = true;
1501       if (Debug == true)
1502          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1503    }
1504
1505    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1506       if (Visited[I] == false)
1507       {
1508          if (Debug == true)
1509             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1510          return false;
1511       }
1512
1513    if (_error->PendingError() == true)
1514    {
1515       if (Debug == true)
1516       {
1517          std::clog << "Validity failed because of pending errors:" << std::endl;
1518          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1519       }
1520       return _error->ReturnError();
1521    }
1522
1523    if (OutMap != 0)
1524       *OutMap = Map.release();
1525    if (OutCache != 0)
1526       *OutCache = CacheP.release();
1527    return true;
1528 }
1529                                                                         /*}}}*/
1530 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1531 // ---------------------------------------------------------------------
1532 /* Size is kind of an abstract notion that is only used for the progress
1533    meter */
1534 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1535 {
1536    map_filesize_t TotalSize = 0;
1537    if (List !=  NULL)
1538    {
1539       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1540       {
1541          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1542          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1543             if ((*j)->HasPackages() == true)
1544                TotalSize += (*j)->Size();
1545       }
1546    }
1547
1548    for (; Start < End; ++Start)
1549    {
1550       if ((*Start)->HasPackages() == false)
1551          continue;
1552       TotalSize += (*Start)->Size();
1553    }
1554    return TotalSize;
1555 }
1556                                                                         /*}}}*/
1557 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1558 static void BuildCache(pkgCacheGenerator &Gen,
1559                        OpProgress * const Progress,
1560                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1561                        pkgSourceList const * const List,
1562                        FileIterator const Start, FileIterator const End)
1563 {
1564    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1565       if (I->HasPackages() == false)
1566          return;
1567
1568       if (I->Exists() == false)
1569          return;
1570
1571       if (I->FindInCache(Gen.GetCache()).end() == false)
1572       {
1573          _error->Warning("Duplicate sources.list entry %s",
1574                I->Describe().c_str());
1575          return;
1576       }
1577
1578       map_filesize_t const Size = I->Size();
1579       if (Progress != NULL)
1580          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1581       CurrentSize += Size;
1582
1583       if (I->Merge(Gen,Progress) == false) {
1584          _error->ReturnError();
1585          return;
1586       }
1587    };
1588
1589    if (List !=  NULL)
1590    {
1591       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1592       {
1593          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1594          {
1595             _error->Warning("Duplicate sources.list entry %s",
1596                   (*i)->Describe().c_str());
1597             continue;
1598          }
1599
1600          if ((*i)->Merge(Gen, Progress) == false) {
1601             _error->ReturnError();
1602             continue;
1603          }
1604
1605          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1606          if (Indexes != NULL)
1607             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1608       }
1609    }
1610
1611    if (Start != End)
1612    {
1613       Gen.SelectReleaseFile("", "");
1614       std::for_each(Start, End, indexFileMerge);
1615    }
1616 }
1617                                                                         /*}}}*/
1618 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1619 // ---------------------------------------------------------------------
1620 /* This makes sure that the status cache (the cache that has all
1621    index files from the sources list and all local ones) is ready
1622    to be mmaped. If OutMap is not zero then a MMap object representing
1623    the cache will be stored there. This is pretty much mandetory if you
1624    are using AllowMem. AllowMem lets the function be run as non-root
1625    where it builds the cache 'fast' into a memory buffer. */
1626 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1627 {
1628    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1629    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1630    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1631    Flags |= MMap::Moveable;
1632    if (_config->FindB("APT::Cache-Fallback", false) == true)
1633       Flags |= MMap::Fallback;
1634    if (CacheF != NULL)
1635       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1636    else
1637       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1638 }
1639 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1640       std::string const &FileName)
1641 {
1642    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1643    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1644       return false;
1645
1646    fchmod(SCacheF.Fd(),0644);
1647
1648    // Write out the main data
1649    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1650       return _error->Error(_("IO Error saving source cache"));
1651
1652    // Write out the proper header
1653    Gen->GetCache().HeaderP->Dirty = false;
1654    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1655    if (SCacheF.Seek(0) == false ||
1656          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1657       return _error->Error(_("IO Error saving source cache"));
1658    Gen->GetCache().HeaderP->Dirty = true;
1659    return true;
1660 }
1661 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1662       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1663 {
1664    Map.reset(CreateDynamicMMap(NULL, 0));
1665    if (unlikely(Map->validData()) == false)
1666       return false;
1667    FileFd CacheF(FileName, FileFd::ReadOnly);
1668    if (CacheF.IsOpen() == false || CacheF.Failed())
1669       return false;
1670    _error->PushToStack();
1671    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1672    bool const newError = _error->PendingError();
1673    _error->MergeWithStack();
1674    if (alloc == 0 && newError)
1675       return _error->ReturnError();
1676    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1677       return false;
1678    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1679    return Gen->Start();
1680 }
1681 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1682                         MMap **OutMap, bool AllowMem)
1683    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1684 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1685                         MMap **OutMap,bool)
1686 {
1687    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1688 }
1689 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1690                         MMap **OutMap,pkgCache **OutCache, bool)
1691 {
1692    // FIXME: deprecate the ignored AllowMem parameter
1693    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1694
1695    std::vector<pkgIndexFile *> Files;
1696    if (_system->AddStatusFiles(Files) == false)
1697       return false;
1698
1699    // Decide if we can write to the files..
1700    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1701    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1702
1703    // ensure the cache directory exists
1704    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1705    {
1706       string dir = _config->FindDir("Dir::Cache");
1707       size_t const len = dir.size();
1708       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1709          dir = dir.substr(0, len - 5);
1710       if (CacheFile.empty() == false)
1711          CreateDirectory(dir, flNotFile(CacheFile));
1712       if (SrcCacheFile.empty() == false)
1713          CreateDirectory(dir, flNotFile(SrcCacheFile));
1714    }
1715
1716    if (Progress != NULL)
1717       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1718
1719    bool pkgcache_fine = false;
1720    bool srcpkgcache_fine = false;
1721    bool volatile_fine = List.GetVolatileFiles().empty();
1722
1723    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1724                      volatile_fine ? OutCache : NULL) == true)
1725    {
1726       if (Debug == true)
1727          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1728       pkgcache_fine = true;
1729       srcpkgcache_fine = true;
1730    }
1731    if (pkgcache_fine == false)
1732    {
1733       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1734       {
1735          if (Debug == true)
1736             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1737          srcpkgcache_fine = true;
1738       }
1739    }
1740
1741    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1742    {
1743       if (Progress != NULL)
1744          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1745       return true;
1746    }
1747
1748    bool Writeable = false;
1749    if (srcpkgcache_fine == false || pkgcache_fine == false)
1750    {
1751       if (CacheFile.empty() == false)
1752          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1753       else if (SrcCacheFile.empty() == false)
1754          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1755
1756       if (Debug == true)
1757          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1758    }
1759
1760    // At this point we know we need to construct something, so get storage ready
1761    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1762    if (unlikely(Map->validData()) == false)
1763       return false;
1764    if (Debug == true)
1765       std::clog << "Open memory Map (not filebased)" << std::endl;
1766
1767    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1768    map_filesize_t CurrentSize = 0;
1769    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1770    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1771    if (srcpkgcache_fine == true && pkgcache_fine == false)
1772    {
1773       if (Debug == true)
1774          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1775       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1776          return false;
1777       srcpkgcache_fine = true;
1778       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1779    }
1780    else if (srcpkgcache_fine == false)
1781    {
1782       if (Debug == true)
1783          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1784       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1785       if (Gen->Start() == false)
1786          return false;
1787
1788       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1789       BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1790                Files.end(),Files.end());
1791
1792       if (Writeable == true && SrcCacheFile.empty() == false)
1793          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1794             return false;
1795    }
1796
1797    if (pkgcache_fine == false)
1798    {
1799       if (Debug == true)
1800          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1801       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1802                Files.begin(), Files.end());
1803
1804       if (Writeable == true && CacheFile.empty() == false)
1805          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1806             return false;
1807    }
1808
1809    if (Debug == true)
1810       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1811
1812    if (volatile_fine == false)
1813    {
1814       if (Gen == nullptr)
1815       {
1816          if (Debug == true)
1817             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1818          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1819             return false;
1820       }
1821
1822       Files = List.GetVolatileFiles();
1823       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1824                Files.begin(), Files.end());
1825    }
1826
1827    if (OutMap != nullptr)
1828       *OutMap = Map.release();
1829
1830    if (Debug == true)
1831       std::clog << "Everything is ready for shipping" << std::endl;
1832    return true;
1833 }
1834                                                                         /*}}}*/
1835 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1836 class APT_HIDDEN ScopedErrorMerge {
1837 public:
1838    ScopedErrorMerge() { _error->PushToStack(); }
1839    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1840 };
1841 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1842    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1843 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1844 {
1845    std::vector<pkgIndexFile *> Files;
1846    if (_system->AddStatusFiles(Files) == false)
1847       return false;
1848
1849    ScopedErrorMerge sem;
1850    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1851    if (unlikely(Map->validData()) == false)
1852       return false;
1853    map_filesize_t CurrentSize = 0;
1854    map_filesize_t TotalSize = 0;
1855    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1856
1857    // Build the status cache
1858    if (Progress != NULL)
1859       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1860    pkgCacheGenerator Gen(Map.get(),Progress);
1861    if (Gen.Start() == false)
1862       return false;
1863    if (_error->PendingError() == true)
1864       return _error->ReturnError();
1865    BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1866                   Files.begin(), Files.end());
1867    // We've passed the point of no return
1868    _error->ReturnError();
1869
1870    *OutMap = Map.release();
1871
1872    return true;
1873 }
1874                                                                         /*}}}*/
1875 // IsDuplicateDescription                                               /*{{{*/
1876 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
1877                             APT::StringView CurMd5, std::string const &CurLang)
1878 {
1879    // Descriptions in the same link-list have all the same md5
1880    if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1881       return false;
1882    for (; Desc.end() == false; ++Desc)
1883       if (Desc.LanguageCode() == CurLang)
1884          return true;
1885    return false;
1886 }
1887                                                                         /*}}}*/
1888
1889 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1890 pkgCacheListParser::~pkgCacheListParser() {}