apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
  49                             APT::StringView CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return _error->ReturnError();
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::TagIterator*>::const_iterator i = Dynamic<pkgCache::TagIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::TagIterator>::toReMap.end(); ++i)
 180       if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 184         if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 199         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 200       if (std::get<1>(seen.insert(*i)) == true)
 201          (*i)->ReMap(oldMap, newMap);
 202    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 203       if (std::get<1>(seen.insert(ViewP)) == false)
 204          continue;
 205       // Ignore views outside of the cache.
 206       if (ViewP->data() < static_cast<const char*>(oldMap)
 207          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 208          continue;
 209       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 210       *ViewP = StringView(data , ViewP->size());
 211    }
 212 }                                                                       /*}}}*/
 213 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 214 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 215                                         const unsigned long &Len) {
 216    size_t oldSize = Map.Size();
 217    void const * const oldMap = Map.Data();
 218    map_stringitem_t const index = Map.WriteString(String, Len);
 219    if (index != 0)
 220       ReMap(oldMap, Map.Data(), oldSize);
 221    return index;
 222 }
 223                                                                         /*}}}*/
 224 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 225 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 226    size_t oldSize = Map.Size();
 227    void const * const oldMap = Map.Data();
 228    map_stringitem_t const index = Map.WriteString(String);
 229    if (index != 0)
 230       ReMap(oldMap, Map.Data(), oldSize);
 231    return index;
 232 }
 233                                                                         /*}}}*/
 234 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 235    size_t oldSize = Map.Size();
 236    void const * const oldMap = Map.Data();
 237    map_pointer_t const index = Map.Allocate(size);
 238    if (index != 0)
 239       ReMap(oldMap, Map.Data(), oldSize);
 240    return index;
 241 }
 242                                                                         /*}}}*/
 243 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 244 // ---------------------------------------------------------------------
 245 /* This provides the generation of the entries in the cache. Each loop
 246    goes through a single package record from the underlying parse engine. */
 247 bool pkgCacheGenerator::MergeList(ListParser &List,
 248                                   pkgCache::VerIterator *OutVer)
 249 {
 250    List.Owner = this;
 251
 252    unsigned int Counter = 0;
 253    while (List.Step() == true)
 254    {
 255       string const PackageName = List.Package();
 256       if (PackageName.empty() == true)
 257           continue;
 258
 259       Counter++;
 260       if (Counter % 100 == 0 && Progress != 0)
 261          Progress->Progress(List.Offset());
 262
 263       APT::StringView Arch = List.Architecture();
 264       Dynamic<APT::StringView> DynArch(Arch);
 265       APT::StringView Version = List.Version();
 266       Dynamic<APT::StringView> DynVersion(Version);
 267       if (Version.empty() == true && Arch.empty() == true)
 268       {
 269          // package descriptions
 270          if (MergeListGroup(List, PackageName) == false)
 271             continue;
 272          continue;
 273       }
 274
 275       // Get a pointer to the package structure
 276       pkgCache::PkgIterator Pkg;
 277       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 278       if (NewPackage(Pkg, PackageName, Arch) == false) {
 279          // TRANSLATOR: The first placeholder is a package name,
 280          // the other two should be copied verbatim as they include debug info
 281          _error->Error(_("Error occurred while processing %s (%s%d)"),
 282                               PackageName.c_str(), "NewPackage", 1);
 283          continue;
 284       }
 285
 286
 287       if (Version.empty() == true)
 288       {
 289          if (MergeListPackage(List, Pkg) == false)
 290             continue;
 291       }
 292       else
 293       {
 294          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 295             return false;
 296       }
 297
 298       if (OutVer != 0)
 299          return true;
 300    }
 301
 302    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 303       return _error->Error(_("Wow, you exceeded the number of package "
 304                              "names this APT is capable of."));
 305    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 306       return _error->Error(_("Wow, you exceeded the number of versions "
 307                              "this APT is capable of."));
 308    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 309       return _error->Error(_("Wow, you exceeded the number of descriptions "
 310                              "this APT is capable of."));
 311    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 312       return _error->Error(_("Wow, you exceeded the number of dependencies "
 313                              "this APT is capable of."));
 314
 315    return true;
 316 }
 317 // CacheGenerator::MergeListGroup                                       /*{{{*/
 318 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 319 {
 320    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 321    // a group has no data on it's own, only packages have it but these
 322    // stanzas like this come from Translation- files to add descriptions,
 323    // but without a version we don't need a description for it…
 324    if (Grp.end() == true)
 325       return true;
 326    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 327
 328    pkgCache::PkgIterator Pkg;
 329    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 330    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 331       if (MergeListPackage(List, Pkg) == false)
 332          return false;
 333
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // CacheGenerator::MergeListPackage                                     /*{{{*/
 338 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 339 {
 340    // we first process the package, then the descriptions
 341    // (for deb this package processing is in fact a no-op)
 342    pkgCache::VerIterator Ver(Cache);
 343    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 344    if (List.UsePackage(Pkg, Ver) == false)
 345       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 346                            Pkg.Name(), "UsePackage", 1);
 347
 348    // Find the right version to write the description
 349    StringView CurMd5 = List.Description_md5();
 350    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 351    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 352    {
 353       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 354
 355       // a version can only have one md5 describing it
 356       if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
 357          continue;
 358
 359       map_stringitem_t md5idx = VerDesc->md5sum;
 360       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 361       {
 362          // don't add a new description if we have one for the given
 363          // md5 && language
 364          if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
 365             continue;
 366
 367          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 368       }
 369
 370       // we can stop here as all "same" versions will share the description
 371       break;
 372    }
 373
 374    return true;
 375 }
 376                                                                         /*}}}*/
 377 // CacheGenerator::MergeListVersion                                     /*{{{*/
 378 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 379                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 380 {
 381    pkgCache::VerIterator Ver = Pkg.VersionList();
 382    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 383    map_pointer_t *LastVer = &Pkg->VersionList;
 384    void const * oldMap = Map.Data();
 385
 386    unsigned short const Hash = List.VersionHash();
 387    if (Ver.end() == false)
 388    {
 389       /* We know the list is sorted so we use that fact in the search.
 390          Insertion of new versions is done with correct sorting */
 391       int Res = 1;
 392       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 393       {
 394          char const * const VerStr = Ver.VerStr();
 395          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 396                VerStr, VerStr + strlen(VerStr));
 397          // Version is higher as current version - insert here
 398          if (Res > 0)
 399             break;
 400          // Versionstrings are equal - is hash also equal?
 401          if (Res == 0)
 402          {
 403             if (List.SameVersion(Hash, Ver) == true)
 404                break;
 405             // sort (volatile) sources above not-sources like the status file
 406             if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 407             {
 408                auto VF = Ver.FileList();
 409                for (; VF.end() == false; ++VF)
 410                   if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
 411                      break;
 412                if (VF.end() == true)
 413                   break;
 414             }
 415          }
 416          // proceed with the next till we have either the right
 417          // or we found another version (which will be lower)
 418       }
 419
 420       /* We already have a version for this item, record that we saw it */
 421       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 422       {
 423          if (List.UsePackage(Pkg,Ver) == false)
 424             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                                  Pkg.Name(), "UsePackage", 2);
 426
 427          if (NewFileVer(Ver,List) == false)
 428             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 429                                  Pkg.Name(), "NewFileVer", 1);
 430
 431          // Read only a single record and return
 432          if (OutVer != 0)
 433          {
 434             *OutVer = Ver;
 435             return true;
 436          }
 437
 438          return true;
 439       }
 440    }
 441
 442    // Add a new version
 443    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 444    if (unlikely(verindex == 0))
 445       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 446                            Pkg.Name(), "NewVersion", 1);
 447
 448    if (oldMap != Map.Data())
 449          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 450    *LastVer = verindex;
 451
 452    if (unlikely(List.NewVersion(Ver) == false))
 453       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 454                            Pkg.Name(), "NewVersion", 2);
 455
 456    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 457       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 458                            Pkg.Name(), "UsePackage", 3);
 459
 460    if (unlikely(NewFileVer(Ver,List) == false))
 461       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 462                            Pkg.Name(), "NewFileVer", 2);
 463
 464    pkgCache::GrpIterator Grp = Pkg.Group();
 465    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 466
 467    /* If it is the first version of this package we need to add implicit
 468       Multi-Arch dependencies to all other package versions in the group now -
 469       otherwise we just add them for this new version */
 470    if (Pkg.VersionList()->NextVer == 0)
 471    {
 472       pkgCache::PkgIterator P = Grp.PackageList();
 473       Dynamic<pkgCache::PkgIterator> DynP(P);
 474       for (; P.end() != true; P = Grp.NextPkg(P))
 475       {
 476          if (P->ID == Pkg->ID)
 477             continue;
 478          pkgCache::VerIterator V = P.VersionList();
 479          Dynamic<pkgCache::VerIterator> DynV(V);
 480          for (; V.end() != true; ++V)
 481             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 482                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 483                                     Pkg.Name(), "AddImplicitDepends", 1);
 484       }
 485    }
 486    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 487       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 488                            Pkg.Name(), "AddImplicitDepends", 2);
 489
 490    // Read only a single record and return
 491    if (OutVer != 0)
 492    {
 493       *OutVer = Ver;
 494       return true;
 495    }
 496
 497    /* Record the Description(s) based on their master md5sum */
 498    StringView CurMd5 = List.Description_md5();
 499
 500    /* Before we add a new description we first search in the group for
 501       a version with a description of the same MD5 - if so we reuse this
 502       description group instead of creating our own for this version */
 503    for (pkgCache::PkgIterator P = Grp.PackageList();
 504         P.end() == false; P = Grp.NextPkg(P))
 505    {
 506       for (pkgCache::VerIterator V = P.VersionList();
 507            V.end() == false; ++V)
 508       {
 509          if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
 510             continue;
 511          Ver->DescriptionList = V->DescriptionList;
 512       }
 513    }
 514
 515    // We haven't found reusable descriptions, so add the first description(s)
 516    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 517    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 518    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 519       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 520          return false;
 521    return true;
 522 }
 523                                                                         /*}}}*/
 524 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 525 {
 526    pkgCache::DescIterator Desc;
 527    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 528
 529    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 530    if (unlikely(descindex == 0))
 531       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 532             Ver.ParentPkg().Name(), "NewDescription", 1);
 533
 534    md5idx = Desc->md5sum;
 535    Desc->ParentPkg = Ver.ParentPkg().Index();
 536
 537    // we add at the end, so that the start is constant as we need
 538    // that to be able to efficiently share these lists
 539    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 540    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 541    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 542    *LastNextDesc = descindex;
 543
 544    if (NewFileDesc(Desc,List) == false)
 545       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 546             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 547
 548    return true;
 549 }
 550                                                                         /*}}}*/
 551                                                                         /*}}}*/
 552 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* This creates a new group structure and adds it to the hash table */
 555 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 556 {
 557    Dynamic<StringView> DName(Name);
 558    Grp = Cache.FindGrp(Name);
 559    if (Grp.end() == false)
 560       return true;
 561
 562    // Get a structure
 563    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 564    if (unlikely(Group == 0))
 565       return false;
 566
 567    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 568    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 569    if (unlikely(idxName == 0))
 570       return false;
 571    Grp->Name = idxName;
 572
 573    // Insert it into the hash table
 574    unsigned long const Hash = Cache.Hash(Name);
 575    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 576
 577    while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 578       insertAt = &(Cache.GrpP + *insertAt)->Next;
 579    Grp->Next = *insertAt;
 580    *insertAt = Group;
 581
 582    Grp->ID = Cache.HeaderP->GroupCount++;
 583    return true;
 584 }
 585                                                                         /*}}}*/
 586 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 587 // ---------------------------------------------------------------------
 588 /* This creates a new package structure and adds it to the hash table */
 589 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 590                                         StringView Arch) {
 591    pkgCache::GrpIterator Grp;
 592    Dynamic<StringView> DName(Name);
 593    Dynamic<StringView> DArch(Arch);
 594    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 595    if (unlikely(NewGroup(Grp, Name) == false))
 596       return false;
 597
 598    Pkg = Grp.FindPkg(Arch);
 599       if (Pkg.end() == false)
 600          return true;
 601
 602    // Get a structure
 603    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 604    if (unlikely(Package == 0))
 605       return false;
 606    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 607
 608    // Set the name, arch and the ID
 609    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 610    Pkg->Group = Grp.Index();
 611    // all is mapped to the native architecture
 612    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 613    if (unlikely(idxArch == 0))
 614       return false;
 615    Pkg->Arch = idxArch;
 616    Pkg->ID = Cache.HeaderP->PackageCount++;
 617
 618    // Insert the package into our package list
 619    if (Grp->FirstPackage == 0) // the group is new
 620    {
 621       Grp->FirstPackage = Package;
 622       // Insert it into the hash table
 623       map_id_t const Hash = Cache.Hash(Name);
 624       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 625       while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
 626          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 627       Pkg->NextPackage = *insertAt;
 628       *insertAt = Package;
 629    }
 630    else // Group the Packages together
 631    {
 632       // if sibling is provided by another package, this one is too
 633       {
 634          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 635          if (M.end() == false) {
 636             pkgCache::PrvIterator Prv;
 637             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 638             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 639             {
 640                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 641                   continue;
 642                pkgCache::VerIterator Ver = Prv.OwnerVer();
 643                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 644                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 645                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 646                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 647                {
 648                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 649                      continue;
 650                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 651                      return false;
 652                }
 653             }
 654          }
 655       }
 656       // let M-A:foreign package siblings provide this package
 657       {
 658          pkgCache::PkgIterator P;
 659          pkgCache::VerIterator Ver;
 660          Dynamic<pkgCache::PkgIterator> DynP(P);
 661          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 662
 663          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 664          {
 665             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 666                continue;
 667             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 668                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 669                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 670                      return false;
 671          }
 672       }
 673       // and negative dependencies, don't forget negative dependencies
 674       {
 675          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 676          if (M.end() == false) {
 677             pkgCache::DepIterator Dep;
 678             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 679             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 680             {
 681                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 682                   continue;
 683                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 684                      Dep->Type != pkgCache::Dep::Replaces)
 685                   continue;
 686                pkgCache::VerIterator Ver = Dep.ParentVer();
 687                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 688                map_pointer_t * unused = NULL;
 689                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 690                   return false;
 691             }
 692          }
 693       }
 694
 695       // this package is the new last package
 696       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 697       Pkg->NextPackage = LastPkg->NextPackage;
 698       LastPkg->NextPackage = Package;
 699    }
 700    Grp->LastPackage = Package;
 701
 702    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 703    if (Arch == "any")
 704    {
 705       size_t const found = Name.rfind(':');
 706       StringView ArchA = Name.substr(found + 1);
 707       if (ArchA != "any")
 708       {
 709          // ArchA is used inside the loop which might remap (NameA is not used)
 710          Dynamic<StringView> DynArchA(ArchA);
 711          StringView NameA = Name.substr(0, found);
 712          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 713          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 714          if (PkgA.end())
 715          {
 716             Dynamic<StringView> DynNameA(NameA);
 717             if (NewPackage(PkgA, NameA, ArchA) == false)
 718                return false;
 719          }
 720          if (unlikely(PkgA.end()))
 721             return _error->Fatal("NewPackage was successful for %s:%s,"
 722                   "but the package doesn't exist anyhow!",
 723                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 724          else
 725          {
 726             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 727             for (; Prv.end() == false; ++Prv)
 728             {
 729                if (Prv.IsMultiArchImplicit())
 730                   continue;
 731                pkgCache::VerIterator V = Prv.OwnerVer();
 732                if (ArchA != V.ParentPkg().Arch())
 733                   continue;
 734                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 735                   return false;
 736             }
 737             pkgCache::VerIterator V = PkgA.VersionList();
 738             Dynamic<pkgCache::VerIterator> DynV(V);
 739             for (; V.end() == false; ++V)
 740             {
 741                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 742                   return false;
 743             }
 744          }
 745       }
 746    }
 747    return true;
 748 }
 749                                                                         /*}}}*/
 750 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 751 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 752                                            pkgCache::PkgIterator &P,
 753                                            pkgCache::VerIterator &V)
 754 {
 755    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 756    Dynamic<APT::StringView> DynArch(Arch);
 757    map_pointer_t *OldDepLast = NULL;
 758    /* MultiArch handling introduces a lot of implicit Dependencies:
 759       - MultiArch: same → Co-Installable if they have the same version
 760       - All others conflict with all other group members */
 761    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 762    pkgCache::PkgIterator D = G.PackageList();
 763    Dynamic<pkgCache::PkgIterator> DynD(D);
 764    map_stringitem_t const VerStrIdx = V->VerStr;
 765    for (; D.end() != true; D = G.NextPkg(D))
 766    {
 767       if (Arch == D.Arch() || D->VersionList == 0)
 768          continue;
 769       /* We allow only one installed arch at the time
 770          per group, therefore each group member conflicts
 771          with all other group members */
 772       if (coInstall == true)
 773       {
 774          // Replaces: ${self}:other ( << ${binary:Version})
 775          NewDepends(D, V, VerStrIdx,
 776                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 777                     OldDepLast);
 778          // Breaks: ${self}:other (!= ${binary:Version})
 779          NewDepends(D, V, VerStrIdx,
 780                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 781                     OldDepLast);
 782       } else {
 783          // Conflicts: ${self}:other
 784          NewDepends(D, V, 0,
 785                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 786                     OldDepLast);
 787       }
 788    }
 789    return true;
 790 }
 791 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 792                                            pkgCache::PkgIterator &D)
 793 {
 794    /* MultiArch handling introduces a lot of implicit Dependencies:
 795       - MultiArch: same → Co-Installable if they have the same version
 796       - All others conflict with all other group members */
 797    map_pointer_t *OldDepLast = NULL;
 798    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 799    if (coInstall == true)
 800    {
 801       map_stringitem_t const VerStrIdx = V->VerStr;
 802       // Replaces: ${self}:other ( << ${binary:Version})
 803       NewDepends(D, V, VerStrIdx,
 804                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 805                  OldDepLast);
 806       // Breaks: ${self}:other (!= ${binary:Version})
 807       NewDepends(D, V, VerStrIdx,
 808                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 809                  OldDepLast);
 810    } else {
 811       // Conflicts: ${self}:other
 812       NewDepends(D, V, 0,
 813                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 814                  OldDepLast);
 815    }
 816    return true;
 817 }
 818
 819                                                                         /*}}}*/
 820 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 821 // ---------------------------------------------------------------------
 822 /* */
 823 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 824                                    ListParser &List)
 825 {
 826    if (CurrentFile == 0)
 827       return true;
 828
 829    // Get a structure
 830    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 831    if (VerFile == 0)
 832       return false;
 833
 834    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 835    VF->File = CurrentFile - Cache.PkgFileP;
 836
 837    // Link it to the end of the list
 838    map_pointer_t *Last = &Ver->FileList;
 839    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 840       Last = &V->NextFile;
 841    VF->NextFile = *Last;
 842    *Last = VF.Index();
 843
 844    VF->Offset = List.Offset();
 845    VF->Size = List.Size();
 846    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 847       Cache.HeaderP->MaxVerFileSize = VF->Size;
 848    Cache.HeaderP->VerFileCount++;
 849
 850    return true;
 851 }
 852                                                                         /*}}}*/
 853 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 854 // ---------------------------------------------------------------------
 855 /* This puts a version structure in the linked list */
 856 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 857                                             APT::StringView const &VerStr,
 858                                             map_pointer_t const ParentPkg,
 859                                             unsigned short const Hash,
 860                                             map_pointer_t const Next)
 861 {
 862    // Get a structure
 863    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 864    if (Version == 0)
 865       return 0;
 866
 867    // Fill it in
 868    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 869    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 870    Ver->NextVer = Next;
 871    Ver->ParentPkg = ParentPkg;
 872    Ver->Hash = Hash;
 873    Ver->ID = Cache.HeaderP->VersionCount++;
 874
 875    // try to find the version string in the group for reuse
 876    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 877    pkgCache::GrpIterator Grp = Pkg.Group();
 878    if (Pkg.end() == false && Grp.end() == false)
 879    {
 880       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 881       {
 882          if (Pkg == P)
 883             continue;
 884          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 885          {
 886             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 887             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 888             {
 889                Ver->VerStr = V->VerStr;
 890                return Version;
 891             }
 892             else if (cmp < 0)
 893                break;
 894          }
 895       }
 896    }
 897    // haven't found the version string, so create
 898    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 899    if (unlikely(idxVerStr == 0))
 900       return 0;
 901    Ver->VerStr = idxVerStr;
 902    return Version;
 903 }
 904                                                                         /*}}}*/
 905 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* */
 908 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 909                                    ListParser &List)
 910 {
 911    if (CurrentFile == 0)
 912       return true;
 913
 914    // Get a structure
 915    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 916    if (DescFile == 0)
 917       return false;
 918
 919    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 920    DF->File = CurrentFile - Cache.PkgFileP;
 921
 922    // Link it to the end of the list
 923    map_pointer_t *Last = &Desc->FileList;
 924    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 925       Last = &D->NextFile;
 926
 927    DF->NextFile = *Last;
 928    *Last = DF.Index();
 929
 930    DF->Offset = List.Offset();
 931    DF->Size = List.Size();
 932    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 933       Cache.HeaderP->MaxDescFileSize = DF->Size;
 934    Cache.HeaderP->DescFileCount++;
 935
 936    return true;
 937 }
 938                                                                         /*}}}*/
 939 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* This puts a description structure in the linked list */
 942 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 943                                             const string &Lang,
 944                                             APT::StringView md5sum,
 945                                             map_stringitem_t const idxmd5str)
 946 {
 947    // Get a structure
 948    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 949    if (Description == 0)
 950       return 0;
 951
 952    // Fill it in
 953    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 954    Desc->ID = Cache.HeaderP->DescriptionCount++;
 955    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 956    if (unlikely(idxlanguage_code == 0))
 957       return 0;
 958    Desc->language_code = idxlanguage_code;
 959
 960    if (idxmd5str != 0)
 961       Desc->md5sum = idxmd5str;
 962    else
 963    {
 964       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
 965       if (unlikely(idxmd5sum == 0))
 966          return 0;
 967       Desc->md5sum = idxmd5sum;
 968    }
 969
 970    return Description;
 971 }
 972                                                                         /*}}}*/
 973 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 974 // ---------------------------------------------------------------------
 975 /* This creates a dependency element in the tree. It is linked to the
 976    version and to the package that it is pointing to. */
 977 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 978                                    pkgCache::VerIterator &Ver,
 979                                    map_pointer_t const Version,
 980                                    uint8_t const Op,
 981                                    uint8_t const Type,
 982                                    map_pointer_t* &OldDepLast)
 983 {
 984    void const * const oldMap = Map.Data();
 985    // Get a structure
 986    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 987    if (unlikely(Dependency == 0))
 988       return false;
 989
 990    bool isDuplicate = false;
 991    map_pointer_t DependencyData = 0;
 992    map_pointer_t PreviousData = 0;
 993    if (Pkg->RevDepends != 0)
 994    {
 995       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 996       DependencyData = L->DependencyData;
 997       do {
 998          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 999          if (Version > D->Version)
1000             break;
1001          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
1002          {
1003             isDuplicate = true;
1004             break;
1005          }
1006          PreviousData = DependencyData;
1007          DependencyData = D->NextData;
1008       } while (DependencyData != 0);
1009    }
1010
1011    if (isDuplicate == false)
1012    {
1013       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1014       if (unlikely(DependencyData == 0))
1015         return false;
1016    }
1017
1018    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1019    Link->ParentVer = Ver.Index();
1020    Link->DependencyData = DependencyData;
1021    Link->ID = Cache.HeaderP->DependsCount++;
1022
1023    pkgCache::DepIterator Dep(Cache, Link);
1024    if (isDuplicate == false)
1025    {
1026       Dep->Type = Type;
1027       Dep->CompareOp = Op;
1028       Dep->Version = Version;
1029       Dep->Package = Pkg.Index();
1030       ++Cache.HeaderP->DependsDataCount;
1031       if (PreviousData != 0)
1032       {
1033          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1034          Dep->NextData = D->NextData;
1035          D->NextData = DependencyData;
1036       }
1037       else if (Pkg->RevDepends != 0)
1038       {
1039          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1040          Dep->NextData = D->DependencyData;
1041       }
1042    }
1043
1044    if (isDuplicate == true || PreviousData != 0)
1045    {
1046       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1047       Link->NextRevDepends = L->NextRevDepends;
1048       L->NextRevDepends = Dependency;
1049    }
1050    else
1051    {
1052       Link->NextRevDepends = Pkg->RevDepends;
1053       Pkg->RevDepends = Dependency;
1054    }
1055
1056
1057    // Do we know where to link the Dependency to?
1058    if (OldDepLast == NULL)
1059    {
1060       OldDepLast = &Ver->DependsList;
1061       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1062          OldDepLast = &D->NextDepends;
1063    } else if (oldMap != Map.Data())
1064       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1065
1066    Dep->NextDepends = *OldDepLast;
1067    *OldDepLast = Dependency;
1068    OldDepLast = &Dep->NextDepends;
1069    return true;
1070 }
1071                                                                         /*}}}*/
1072 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1073 // ---------------------------------------------------------------------
1074 /* This creates a Group and the Package to link this dependency to if
1075    needed and handles also the caching of the old endpoint */
1076 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1077                                                StringView PackageName,
1078                                                StringView Arch,
1079                                                StringView Version,
1080                                                uint8_t const Op,
1081                                                uint8_t const Type)
1082 {
1083    pkgCache::GrpIterator Grp;
1084    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1085    Dynamic<StringView> DynPackageName(PackageName);
1086    Dynamic<StringView> DynArch(Arch);
1087    Dynamic<StringView> DynVersion(Version);
1088    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1089       return false;
1090
1091    map_stringitem_t idxVersion = 0;
1092    if (Version.empty() == false)
1093    {
1094       int const CmpOp = Op & 0x0F;
1095       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1096       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1097          idxVersion = Ver->VerStr;
1098
1099       if (idxVersion == 0)
1100       {
1101          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1102          if (unlikely(idxVersion == 0))
1103             return false;
1104       }
1105    }
1106
1107    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1108          Type == pkgCache::Dep::Conflicts ||
1109          Type == pkgCache::Dep::Replaces);
1110
1111    pkgCache::PkgIterator Pkg;
1112    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1113    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1114    {
1115       // Locate the target package
1116       Pkg = Grp.FindPkg(Arch);
1117       if (Pkg.end() == true) {
1118          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1119             return false;
1120       }
1121
1122       /* Caching the old end point speeds up generation substantially */
1123       if (OldDepVer != Ver) {
1124          OldDepLast = NULL;
1125          OldDepVer = Ver;
1126       }
1127
1128       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1129    }
1130    else
1131    {
1132       /* Caching the old end point speeds up generation substantially */
1133       if (OldDepVer != Ver) {
1134          OldDepLast = NULL;
1135          OldDepVer = Ver;
1136       }
1137
1138       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1139       {
1140          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1141                return false;
1142       }
1143    }
1144    return true;
1145 }
1146                                                                         /*}}}*/
1147 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1148 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1149                                                 StringView PkgName,
1150                                                 StringView PkgArch,
1151                                                 StringView Version,
1152                                                 uint8_t const Flags)
1153 {
1154    pkgCache const &Cache = Owner->Cache;
1155    Dynamic<StringView> DynPkgName(PkgName);
1156    Dynamic<StringView> DynArch(PkgArch);
1157    Dynamic<StringView> DynVersion(Version);
1158
1159    // We do not add self referencing provides
1160    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1161         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1162          (Version.empty() || Version == Ver.VerStr()))
1163       return true;
1164
1165    // Locate the target package
1166    pkgCache::PkgIterator Pkg;
1167    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1168    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1169       return false;
1170
1171    map_stringitem_t idxProvideVersion = 0;
1172    if (Version.empty() == false) {
1173       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1174       if (unlikely(idxProvideVersion == 0))
1175          return false;
1176    }
1177    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1178 }
1179 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1180                                     pkgCache::PkgIterator &Pkg,
1181                                     map_pointer_t const ProvideVersion,
1182                                     uint8_t const Flags)
1183 {
1184    // Get a structure
1185    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1186    if (unlikely(Provides == 0))
1187       return false;
1188    ++Cache.HeaderP->ProvidesCount;
1189
1190    // Fill it in
1191    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1192    Prv->Version = Ver.Index();
1193    Prv->ProvideVersion = ProvideVersion;
1194    Prv->Flags = Flags;
1195    Prv->NextPkgProv = Ver->ProvidesList;
1196    Ver->ProvidesList = Prv.Index();
1197
1198    // Link it to the package
1199    Prv->ParentPkg = Pkg.Index();
1200    Prv->NextProvides = Pkg->ProvidesList;
1201    Pkg->ProvidesList = Prv.Index();
1202    return true;
1203 }
1204                                                                         /*}}}*/
1205 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1206 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1207                                 StringView Version, uint8_t const Flags) {
1208    pkgCache &Cache = Owner->Cache;
1209    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1210    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1211    Dynamic<StringView> DynPackage(Package);
1212    Dynamic<StringView> DynVersion(Version);
1213
1214    if (Grp.end() == true)
1215       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1216    else
1217    {
1218       map_stringitem_t idxProvideVersion = 0;
1219       if (Version.empty() == false) {
1220          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1221          if (unlikely(idxProvideVersion == 0))
1222             return false;
1223       }
1224
1225       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1226       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1227       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1228       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1229       pkgCache::PkgIterator Pkg;
1230       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1231       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1232       {
1233          if (isImplicit && OwnerPkg == Pkg)
1234             continue;
1235          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1236             continue;
1237          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1238             return false;
1239       }
1240    }
1241    return true;
1242 }
1243                                                                         /*}}}*/
1244 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1245       pkgCache::VerIterator const &Ver)
1246 {
1247    return Hash == Ver->Hash;
1248 }
1249                                                                         /*}}}*/
1250 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1251 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1252                                    unsigned long Flags)
1253 {
1254    if (File.empty() && Site.empty())
1255    {
1256       CurrentRlsFile = NULL;
1257       return true;
1258    }
1259
1260    // Get some space for the structure
1261    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1262    if (unlikely(idxFile == 0))
1263       return false;
1264    CurrentRlsFile = Cache.RlsFileP + idxFile;
1265
1266    // Fill it in
1267    map_stringitem_t const idxFileName = WriteStringInMap(File);
1268    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1269    if (unlikely(idxFileName == 0 || idxSite == 0))
1270       return false;
1271    CurrentRlsFile->FileName = idxFileName;
1272    CurrentRlsFile->Site = idxSite;
1273    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1274    CurrentRlsFile->Flags = Flags;
1275    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1276    RlsFileName = File;
1277    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1278    Cache.HeaderP->ReleaseFileCount++;
1279
1280    return true;
1281 }
1282                                                                         /*}}}*/
1283 // ListParser::NewTag - Create a Tag element                            /*{{{*/
1284 // ---------------------------------------------------------------------
1285 /* */
1286 bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
1287                                            const char *NameStart,
1288                                            unsigned int NameSize)
1289 {
1290    return Owner->NewTag(Ver, NameStart, NameSize);
1291 }
1292 bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
1293                                            const char *NameStart,
1294                                            unsigned int NameSize)
1295 {
1296    // Get a structure
1297    map_pointer_t const idxTag = AllocateInMap(sizeof(pkgCache::Tag));
1298    if (unlikely(idxTag == 0))
1299       return false;
1300
1301    // Fill it in
1302    pkgCache::TagIterator Tg(Cache,Cache.TagP + idxTag);
1303    map_pointer_t const idxName = WriteStringInMap(NameStart,NameSize);
1304    if (idxName == 0)
1305       return false;
1306    Tg->Name = idxName;
1307
1308    Tg->NextTag = Ver->TagList;
1309    Ver->TagList = Tg.Index();
1310    Cache.HeaderP->TagCount++;
1311
1312    return true;
1313 }
1314                                                                         /*}}}*/
1315 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1316 // ---------------------------------------------------------------------
1317 /* This is used to select which file is to be associated with all newly
1318    added versions. The caller is responsible for setting the IMS fields. */
1319 bool pkgCacheGenerator::SelectFile(std::string const &File,
1320                                    pkgIndexFile const &Index,
1321                                    std::string const &Architecture,
1322                                    std::string const &Component,
1323                                    unsigned long const Flags)
1324 {
1325    // Get some space for the structure
1326    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1327    if (unlikely(idxFile == 0))
1328       return false;
1329    CurrentFile = Cache.PkgFileP + idxFile;
1330
1331    // Fill it in
1332    map_stringitem_t const idxFileName = WriteStringInMap(File);
1333    if (unlikely(idxFileName == 0))
1334       return false;
1335    CurrentFile->FileName = idxFileName;
1336    CurrentFile->NextFile = Cache.HeaderP->FileList;
1337    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1338    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1339    if (unlikely(idxIndexType == 0))
1340       return false;
1341    CurrentFile->IndexType = idxIndexType;
1342    if (Architecture.empty())
1343       CurrentFile->Architecture = 0;
1344    else
1345    {
1346       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1347       if (unlikely(arch == 0))
1348          return false;
1349       CurrentFile->Architecture = arch;
1350    }
1351    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1352    if (unlikely(component == 0))
1353       return false;
1354    CurrentFile->Component = component;
1355    CurrentFile->Flags = Flags;
1356    if (CurrentRlsFile != NULL)
1357       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1358    else
1359       CurrentFile->Release = 0;
1360    PkgFileName = File;
1361    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1362    Cache.HeaderP->PackageFileCount++;
1363
1364    if (Progress != 0)
1365       Progress->SubProgress(Index.Size());
1366    return true;
1367 }
1368                                                                         /*}}}*/
1369 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1370 // ---------------------------------------------------------------------
1371 /* This is used to create handles to strings. Given the same text it
1372    always returns the same number */
1373 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1374                                                  unsigned int Size)
1375 {
1376    auto strings = &strMixed;
1377    switch(type) {
1378       case MIXED: strings = &strMixed; break;
1379       case PKGNAME: strings = &strPkgNames; break;
1380       case VERSIONNUMBER: strings = &strVersions; break;
1381       case SECTION: strings = &strSections; break;
1382       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1383    }
1384
1385    auto const item = strings->find({S, Size, nullptr, 0});
1386    if (item != strings->end())
1387       return item->item;
1388
1389    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1390    strings->insert({nullptr, Size, this, idxString});
1391    return idxString;
1392 }
1393                                                                         /*}}}*/
1394 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1395 // ---------------------------------------------------------------------
1396 /* This just verifies that each file in the list of index files exists,
1397    has matching attributes with the cache and the cache does not have
1398    any extra files. */
1399 class APT_HIDDEN ScopedErrorRevert {
1400 public:
1401    ScopedErrorRevert() { _error->PushToStack(); }
1402    ~ScopedErrorRevert() { _error->RevertToStack(); }
1403 };
1404 static bool CheckValidity(const string &CacheFile,
1405                           pkgSourceList &List,
1406                           FileIterator const Start,
1407                           FileIterator const End,
1408                           MMap **OutMap = 0,
1409                           pkgCache **OutCache = 0)
1410 {
1411    ScopedErrorRevert ser;
1412    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1413    // No file, certainly invalid
1414    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1415    {
1416       if (Debug == true)
1417          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1418       return false;
1419    }
1420
1421    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1422    {
1423       if (Debug == true)
1424          std::clog << "sources.list is newer than the cache" << std::endl;
1425       return false;
1426    }
1427
1428    // Map it
1429    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1430    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1431    if (unlikely(Map->validData()) == false)
1432       return false;
1433    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1434    pkgCache &Cache = *CacheP.get();
1435    if (_error->PendingError() || Map->Size() == 0)
1436    {
1437       if (Debug == true)
1438          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1439       return _error->ReturnError();
1440    }
1441
1442    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1443    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1444    std::vector<pkgIndexFile *> Files;
1445    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1446    {
1447       if (Debug == true)
1448          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1449       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1450       if (RlsFile.end() == true)
1451       {
1452          if (Debug == true)
1453             std::clog << "FindInCache returned end-Pointer" << std::endl;
1454          return false;
1455       }
1456
1457       RlsVisited[RlsFile->ID] = true;
1458       if (Debug == true)
1459          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1460
1461       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1462       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1463             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1464    }
1465    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1466       if (RlsVisited[I] == false)
1467       {
1468          if (Debug == true)
1469             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1470          return false;
1471       }
1472
1473    std::copy(Start, End, std::back_inserter(Files));
1474
1475    /* Now we check every index file, see if it is in the cache,
1476       verify the IMS data and check that it is on the disk too.. */
1477    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1478    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1479    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1480    {
1481       if (Debug == true)
1482          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1483       if ((*PkgFile)->Exists() == false)
1484       {
1485          if (Debug == true)
1486             std::clog << "file doesn't exist" << std::endl;
1487          continue;
1488       }
1489
1490       // FindInCache is also expected to do an IMS check.
1491       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1492       if (File.end() == true)
1493       {
1494          if (Debug == true)
1495             std::clog << "FindInCache returned end-Pointer" << std::endl;
1496          return false;
1497       }
1498
1499       Visited[File->ID] = true;
1500       if (Debug == true)
1501          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1502    }
1503
1504    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1505       if (Visited[I] == false)
1506       {
1507          if (Debug == true)
1508             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1509          return false;
1510       }
1511
1512    if (_error->PendingError() == true)
1513    {
1514       if (Debug == true)
1515       {
1516          std::clog << "Validity failed because of pending errors:" << std::endl;
1517          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1518       }
1519       return _error->ReturnError();
1520    }
1521
1522    if (OutMap != 0)
1523       *OutMap = Map.release();
1524    if (OutCache != 0)
1525       *OutCache = CacheP.release();
1526    return true;
1527 }
1528                                                                         /*}}}*/
1529 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1530 // ---------------------------------------------------------------------
1531 /* Size is kind of an abstract notion that is only used for the progress
1532    meter */
1533 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1534 {
1535    map_filesize_t TotalSize = 0;
1536    if (List !=  NULL)
1537    {
1538       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1539       {
1540          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1541          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1542             if ((*j)->HasPackages() == true)
1543                TotalSize += (*j)->Size();
1544       }
1545    }
1546
1547    for (; Start < End; ++Start)
1548    {
1549       if ((*Start)->HasPackages() == false)
1550          continue;
1551       TotalSize += (*Start)->Size();
1552    }
1553    return TotalSize;
1554 }
1555                                                                         /*}}}*/
1556 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1557 static void BuildCache(pkgCacheGenerator &Gen,
1558                        OpProgress * const Progress,
1559                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1560                        pkgSourceList const * const List,
1561                        FileIterator const Start, FileIterator const End)
1562 {
1563    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1564       if (I->HasPackages() == false)
1565          return;
1566
1567       if (I->Exists() == false)
1568          return;
1569
1570       if (I->FindInCache(Gen.GetCache()).end() == false)
1571       {
1572          _error->Warning("Duplicate sources.list entry %s",
1573                I->Describe().c_str());
1574          return;
1575       }
1576
1577       map_filesize_t const Size = I->Size();
1578       if (Progress != NULL)
1579          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1580       CurrentSize += Size;
1581
1582       if (I->Merge(Gen,Progress) == false) {
1583          _error->ReturnError();
1584          return;
1585       }
1586    };
1587
1588    if (List !=  NULL)
1589    {
1590       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1591       {
1592          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1593          {
1594             _error->Warning("Duplicate sources.list entry %s",
1595                   (*i)->Describe().c_str());
1596             continue;
1597          }
1598
1599          if ((*i)->Merge(Gen, Progress) == false) {
1600             _error->ReturnError();
1601             continue;
1602          }
1603
1604          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1605          if (Indexes != NULL)
1606             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1607       }
1608    }
1609
1610    if (Start != End)
1611    {
1612       Gen.SelectReleaseFile("", "");
1613       std::for_each(Start, End, indexFileMerge);
1614    }
1615 }
1616                                                                         /*}}}*/
1617 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1618 // ---------------------------------------------------------------------
1619 /* This makes sure that the status cache (the cache that has all
1620    index files from the sources list and all local ones) is ready
1621    to be mmaped. If OutMap is not zero then a MMap object representing
1622    the cache will be stored there. This is pretty much mandetory if you
1623    are using AllowMem. AllowMem lets the function be run as non-root
1624    where it builds the cache 'fast' into a memory buffer. */
1625 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1626 {
1627    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1628    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1629    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1630    Flags |= MMap::Moveable;
1631    if (_config->FindB("APT::Cache-Fallback", false) == true)
1632       Flags |= MMap::Fallback;
1633    if (CacheF != NULL)
1634       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1635    else
1636       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1637 }
1638 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1639       std::string const &FileName)
1640 {
1641    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1642    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1643       return false;
1644
1645    fchmod(SCacheF.Fd(),0644);
1646
1647    // Write out the main data
1648    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1649       return _error->Error(_("IO Error saving source cache"));
1650
1651    // Write out the proper header
1652    Gen->GetCache().HeaderP->Dirty = false;
1653    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1654    if (SCacheF.Seek(0) == false ||
1655          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1656       return _error->Error(_("IO Error saving source cache"));
1657    Gen->GetCache().HeaderP->Dirty = true;
1658    return true;
1659 }
1660 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1661       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1662 {
1663    Map.reset(CreateDynamicMMap(NULL, 0));
1664    if (unlikely(Map->validData()) == false)
1665       return false;
1666    FileFd CacheF(FileName, FileFd::ReadOnly);
1667    if (CacheF.IsOpen() == false || CacheF.Failed())
1668       return false;
1669    _error->PushToStack();
1670    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1671    bool const newError = _error->PendingError();
1672    _error->MergeWithStack();
1673    if (alloc == 0 && newError)
1674       return _error->ReturnError();
1675    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1676       return false;
1677    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1678    return Gen->Start();
1679 }
1680 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1681                         MMap **OutMap, bool AllowMem)
1682    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1683 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1684                         MMap **OutMap,bool)
1685 {
1686    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1687 }
1688 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1689                         MMap **OutMap,pkgCache **OutCache, bool)
1690 {
1691    // FIXME: deprecate the ignored AllowMem parameter
1692    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1693
1694    std::vector<pkgIndexFile *> Files;
1695    if (_system->AddStatusFiles(Files) == false)
1696       return false;
1697
1698    // Decide if we can write to the files..
1699    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1700    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1701
1702    // ensure the cache directory exists
1703    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1704    {
1705       string dir = _config->FindDir("Dir::Cache");
1706       size_t const len = dir.size();
1707       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1708          dir = dir.substr(0, len - 5);
1709       if (CacheFile.empty() == false)
1710          CreateDirectory(dir, flNotFile(CacheFile));
1711       if (SrcCacheFile.empty() == false)
1712          CreateDirectory(dir, flNotFile(SrcCacheFile));
1713    }
1714
1715    if (Progress != NULL)
1716       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1717
1718    bool pkgcache_fine = false;
1719    bool srcpkgcache_fine = false;
1720    bool volatile_fine = List.GetVolatileFiles().empty();
1721
1722    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1723                      volatile_fine ? OutCache : NULL) == true)
1724    {
1725       if (Debug == true)
1726          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1727       pkgcache_fine = true;
1728       srcpkgcache_fine = true;
1729    }
1730    if (pkgcache_fine == false)
1731    {
1732       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1733       {
1734          if (Debug == true)
1735             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1736          srcpkgcache_fine = true;
1737       }
1738    }
1739
1740    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1741    {
1742       if (Progress != NULL)
1743          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1744       return true;
1745    }
1746
1747    bool Writeable = false;
1748    if (srcpkgcache_fine == false || pkgcache_fine == false)
1749    {
1750       if (CacheFile.empty() == false)
1751          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1752       else if (SrcCacheFile.empty() == false)
1753          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1754
1755       if (Debug == true)
1756          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1757    }
1758
1759    // At this point we know we need to construct something, so get storage ready
1760    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1761    if (unlikely(Map->validData()) == false)
1762       return false;
1763    if (Debug == true)
1764       std::clog << "Open memory Map (not filebased)" << std::endl;
1765
1766    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1767    map_filesize_t CurrentSize = 0;
1768    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1769    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1770    if (srcpkgcache_fine == true && pkgcache_fine == false)
1771    {
1772       if (Debug == true)
1773          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1774       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1775          return false;
1776       srcpkgcache_fine = true;
1777       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1778    }
1779    else if (srcpkgcache_fine == false)
1780    {
1781       if (Debug == true)
1782          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1783       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1784       if (Gen->Start() == false)
1785          return false;
1786
1787       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1788       BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1789                Files.end(),Files.end());
1790
1791       if (Writeable == true && SrcCacheFile.empty() == false)
1792          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1793             return false;
1794    }
1795
1796    if (pkgcache_fine == false)
1797    {
1798       if (Debug == true)
1799          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1800       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1801                Files.begin(), Files.end());
1802
1803       if (Writeable == true && CacheFile.empty() == false)
1804          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1805             return false;
1806    }
1807
1808    if (Debug == true)
1809       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1810
1811    if (volatile_fine == false)
1812    {
1813       if (Gen == nullptr)
1814       {
1815          if (Debug == true)
1816             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1817          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1818             return false;
1819       }
1820
1821       Files = List.GetVolatileFiles();
1822       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1823                Files.begin(), Files.end());
1824    }
1825
1826    if (OutMap != nullptr)
1827       *OutMap = Map.release();
1828
1829    if (Debug == true)
1830       std::clog << "Everything is ready for shipping" << std::endl;
1831    return true;
1832 }
1833                                                                         /*}}}*/
1834 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1835 class APT_HIDDEN ScopedErrorMerge {
1836 public:
1837    ScopedErrorMerge() { _error->PushToStack(); }
1838    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1839 };
1840 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1841    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1842 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1843 {
1844    std::vector<pkgIndexFile *> Files;
1845    if (_system->AddStatusFiles(Files) == false)
1846       return false;
1847
1848    ScopedErrorMerge sem;
1849    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1850    if (unlikely(Map->validData()) == false)
1851       return false;
1852    map_filesize_t CurrentSize = 0;
1853    map_filesize_t TotalSize = 0;
1854    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1855
1856    // Build the status cache
1857    if (Progress != NULL)
1858       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1859    pkgCacheGenerator Gen(Map.get(),Progress);
1860    if (Gen.Start() == false)
1861       return false;
1862    if (_error->PendingError() == true)
1863       return _error->ReturnError();
1864    BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1865                   Files.begin(), Files.end());
1866    // We've passed the point of no return
1867    _error->ReturnError();
1868
1869    *OutMap = Map.release();
1870
1871    return true;
1872 }
1873                                                                         /*}}}*/
1874 // IsDuplicateDescription                                               /*{{{*/
1875 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
1876                             APT::StringView CurMd5, std::string const &CurLang)
1877 {
1878    // Descriptions in the same link-list have all the same md5
1879    if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1880       return false;
1881    for (; Desc.end() == false; ++Desc)
1882       if (Desc.LanguageCode() == CurLang)
1883          return true;
1884    return false;
1885 }
1886                                                                         /*}}}*/
1887
1888 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1889 pkgCacheListParser::~pkgCacheListParser() {}