apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
  49                             APT::StringView CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return _error->ReturnError();
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 180         if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 184       if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 199       if (std::get<1>(seen.insert(ViewP)) == false)
 200          continue;
 201       // Ignore views outside of the cache.
 202       if (ViewP->data() < static_cast<const char*>(oldMap)
 203          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 204          continue;
 205       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 206       *ViewP = StringView(data , ViewP->size());
 207    }
 208 }                                                                       /*}}}*/
 209 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 210 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 211                                         const unsigned long &Len) {
 212    size_t oldSize = Map.Size();
 213    void const * const oldMap = Map.Data();
 214    map_stringitem_t const index = Map.WriteString(String, Len);
 215    if (index != 0)
 216       ReMap(oldMap, Map.Data(), oldSize);
 217    return index;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 221 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 222    size_t oldSize = Map.Size();
 223    void const * const oldMap = Map.Data();
 224    map_stringitem_t const index = Map.WriteString(String);
 225    if (index != 0)
 226       ReMap(oldMap, Map.Data(), oldSize);
 227    return index;
 228 }
 229                                                                         /*}}}*/
 230 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 231    size_t oldSize = Map.Size();
 232    void const * const oldMap = Map.Data();
 233    map_pointer_t const index = Map.Allocate(size);
 234    if (index != 0)
 235       ReMap(oldMap, Map.Data(), oldSize);
 236    return index;
 237 }
 238                                                                         /*}}}*/
 239 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This provides the generation of the entries in the cache. Each loop
 242    goes through a single package record from the underlying parse engine. */
 243 bool pkgCacheGenerator::MergeList(ListParser &List,
 244                                   pkgCache::VerIterator *OutVer)
 245 {
 246    List.Owner = this;
 247
 248    unsigned int Counter = 0;
 249    while (List.Step() == true)
 250    {
 251       string const PackageName = List.Package();
 252       if (PackageName.empty() == true)
 253           continue;
 254
 255       Counter++;
 256       if (Counter % 100 == 0 && Progress != 0)
 257          Progress->Progress(List.Offset());
 258
 259       APT::StringView Arch = List.Architecture();
 260       Dynamic<APT::StringView> DynArch(Arch);
 261       APT::StringView Version = List.Version();
 262       Dynamic<APT::StringView> DynVersion(Version);
 263       if (Version.empty() == true && Arch.empty() == true)
 264       {
 265          // package descriptions
 266          if (MergeListGroup(List, PackageName) == false)
 267             continue;
 268          continue;
 269       }
 270
 271       // Get a pointer to the package structure
 272       pkgCache::PkgIterator Pkg;
 273       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 274       if (NewPackage(Pkg, PackageName, Arch) == false) {
 275          // TRANSLATOR: The first placeholder is a package name,
 276          // the other two should be copied verbatim as they include debug info
 277          _error->Error(_("Error occurred while processing %s (%s%d)"),
 278                               PackageName.c_str(), "NewPackage", 1);
 279          continue;
 280       }
 281
 282
 283       if (Version.empty() == true)
 284       {
 285          if (MergeListPackage(List, Pkg) == false)
 286             continue;
 287       }
 288       else
 289       {
 290          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 291             return false;
 292       }
 293
 294       if (OutVer != 0)
 295          return true;
 296    }
 297
 298    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 299       return _error->Error(_("Wow, you exceeded the number of package "
 300                              "names this APT is capable of."));
 301    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 302       return _error->Error(_("Wow, you exceeded the number of versions "
 303                              "this APT is capable of."));
 304    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 305       return _error->Error(_("Wow, you exceeded the number of descriptions "
 306                              "this APT is capable of."));
 307    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 308       return _error->Error(_("Wow, you exceeded the number of dependencies "
 309                              "this APT is capable of."));
 310
 311    return true;
 312 }
 313 // CacheGenerator::MergeListGroup                                       /*{{{*/
 314 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 315 {
 316    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 317    // a group has no data on it's own, only packages have it but these
 318    // stanzas like this come from Translation- files to add descriptions,
 319    // but without a version we don't need a description for it…
 320    if (Grp.end() == true)
 321       return true;
 322    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 323
 324    pkgCache::PkgIterator Pkg;
 325    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 326    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 327       if (MergeListPackage(List, Pkg) == false)
 328          return false;
 329
 330    return true;
 331 }
 332                                                                         /*}}}*/
 333 // CacheGenerator::MergeListPackage                                     /*{{{*/
 334 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 335 {
 336    // we first process the package, then the descriptions
 337    // (for deb this package processing is in fact a no-op)
 338    pkgCache::VerIterator Ver(Cache);
 339    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 340    if (List.UsePackage(Pkg, Ver) == false)
 341       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 342                            Pkg.Name(), "UsePackage", 1);
 343
 344    // Find the right version to write the description
 345    StringView CurMd5 = List.Description_md5();
 346    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 347    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 348    {
 349       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 350
 351       // a version can only have one md5 describing it
 352       if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
 353          continue;
 354
 355       map_stringitem_t md5idx = VerDesc->md5sum;
 356       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 357       {
 358          // don't add a new description if we have one for the given
 359          // md5 && language
 360          if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
 361             continue;
 362
 363          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 364       }
 365
 366       // we can stop here as all "same" versions will share the description
 367       break;
 368    }
 369
 370    return true;
 371 }
 372                                                                         /*}}}*/
 373 // CacheGenerator::MergeListVersion                                     /*{{{*/
 374 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 375                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 376 {
 377    pkgCache::VerIterator Ver = Pkg.VersionList();
 378    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 379    map_pointer_t *LastVer = &Pkg->VersionList;
 380    void const * oldMap = Map.Data();
 381
 382    unsigned short const Hash = List.VersionHash();
 383    if (Ver.end() == false)
 384    {
 385       /* We know the list is sorted so we use that fact in the search.
 386          Insertion of new versions is done with correct sorting */
 387       int Res = 1;
 388       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 389       {
 390          char const * const VerStr = Ver.VerStr();
 391          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 392                VerStr, VerStr + strlen(VerStr));
 393          // Version is higher as current version - insert here
 394          if (Res > 0)
 395             break;
 396          // Versionstrings are equal - is hash also equal?
 397          if (Res == 0)
 398          {
 399             if (List.SameVersion(Hash, Ver) == true)
 400                break;
 401             // sort (volatile) sources above not-sources like the status file
 402             if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 403             {
 404                auto VF = Ver.FileList();
 405                for (; VF.end() == false; ++VF)
 406                   if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
 407                      break;
 408                if (VF.end() == true)
 409                   break;
 410             }
 411          }
 412          // proceed with the next till we have either the right
 413          // or we found another version (which will be lower)
 414       }
 415
 416       /* We already have a version for this item, record that we saw it */
 417       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 418       {
 419          if (List.UsePackage(Pkg,Ver) == false)
 420             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 421                                  Pkg.Name(), "UsePackage", 2);
 422
 423          if (NewFileVer(Ver,List) == false)
 424             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                                  Pkg.Name(), "NewFileVer", 1);
 426
 427          // Read only a single record and return
 428          if (OutVer != 0)
 429          {
 430             *OutVer = Ver;
 431             return true;
 432          }
 433
 434          return true;
 435       }
 436    }
 437
 438    // Add a new version
 439    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 440    if (unlikely(verindex == 0))
 441       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 442                            Pkg.Name(), "NewVersion", 1);
 443
 444    if (oldMap != Map.Data())
 445          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 446    *LastVer = verindex;
 447
 448    if (unlikely(List.NewVersion(Ver) == false))
 449       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 450                            Pkg.Name(), "NewVersion", 2);
 451
 452    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 453       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 454                            Pkg.Name(), "UsePackage", 3);
 455
 456    if (unlikely(NewFileVer(Ver,List) == false))
 457       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 458                            Pkg.Name(), "NewFileVer", 2);
 459
 460    pkgCache::GrpIterator Grp = Pkg.Group();
 461    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 462
 463    /* If it is the first version of this package we need to add implicit
 464       Multi-Arch dependencies to all other package versions in the group now -
 465       otherwise we just add them for this new version */
 466    if (Pkg.VersionList()->NextVer == 0)
 467    {
 468       pkgCache::PkgIterator P = Grp.PackageList();
 469       Dynamic<pkgCache::PkgIterator> DynP(P);
 470       for (; P.end() != true; P = Grp.NextPkg(P))
 471       {
 472          if (P->ID == Pkg->ID)
 473             continue;
 474          pkgCache::VerIterator V = P.VersionList();
 475          Dynamic<pkgCache::VerIterator> DynV(V);
 476          for (; V.end() != true; ++V)
 477             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 478                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 479                                     Pkg.Name(), "AddImplicitDepends", 1);
 480       }
 481    }
 482    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 483       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 484                            Pkg.Name(), "AddImplicitDepends", 2);
 485
 486    // Read only a single record and return
 487    if (OutVer != 0)
 488    {
 489       *OutVer = Ver;
 490       return true;
 491    }
 492
 493    /* Record the Description(s) based on their master md5sum */
 494    StringView CurMd5 = List.Description_md5();
 495
 496    /* Before we add a new description we first search in the group for
 497       a version with a description of the same MD5 - if so we reuse this
 498       description group instead of creating our own for this version */
 499    for (pkgCache::PkgIterator P = Grp.PackageList();
 500         P.end() == false; P = Grp.NextPkg(P))
 501    {
 502       for (pkgCache::VerIterator V = P.VersionList();
 503            V.end() == false; ++V)
 504       {
 505          if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
 506             continue;
 507          Ver->DescriptionList = V->DescriptionList;
 508       }
 509    }
 510
 511    // We haven't found reusable descriptions, so add the first description(s)
 512    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 513    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 514    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 515       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 516          return false;
 517    return true;
 518 }
 519                                                                         /*}}}*/
 520 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 521 {
 522    pkgCache::DescIterator Desc;
 523    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 524
 525    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 526    if (unlikely(descindex == 0))
 527       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 528             Ver.ParentPkg().Name(), "NewDescription", 1);
 529
 530    md5idx = Desc->md5sum;
 531    Desc->ParentPkg = Ver.ParentPkg().Index();
 532
 533    // we add at the end, so that the start is constant as we need
 534    // that to be able to efficiently share these lists
 535    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 536    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 537    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 538    *LastNextDesc = descindex;
 539
 540    if (NewFileDesc(Desc,List) == false)
 541       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 542             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 543
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547                                                                         /*}}}*/
 548 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* This creates a new group structure and adds it to the hash table */
 551 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 552 {
 553    Dynamic<StringView> DName(Name);
 554    Grp = Cache.FindGrp(Name);
 555    if (Grp.end() == false)
 556       return true;
 557
 558    // Get a structure
 559    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 560    if (unlikely(Group == 0))
 561       return false;
 562
 563    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 564    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 565    if (unlikely(idxName == 0))
 566       return false;
 567    Grp->Name = idxName;
 568
 569    // Insert it into the hash table
 570    unsigned long const Hash = Cache.Hash(Name);
 571    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 572
 573    while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 574       insertAt = &(Cache.GrpP + *insertAt)->Next;
 575    Grp->Next = *insertAt;
 576    *insertAt = Group;
 577
 578    Grp->ID = Cache.HeaderP->GroupCount++;
 579    return true;
 580 }
 581                                                                         /*}}}*/
 582 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 583 // ---------------------------------------------------------------------
 584 /* This creates a new package structure and adds it to the hash table */
 585 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 586                                         StringView Arch) {
 587    pkgCache::GrpIterator Grp;
 588    Dynamic<StringView> DName(Name);
 589    Dynamic<StringView> DArch(Arch);
 590    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 591    if (unlikely(NewGroup(Grp, Name) == false))
 592       return false;
 593
 594    Pkg = Grp.FindPkg(Arch);
 595       if (Pkg.end() == false)
 596          return true;
 597
 598    // Get a structure
 599    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 600    if (unlikely(Package == 0))
 601       return false;
 602    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 603
 604    // Set the name, arch and the ID
 605    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 606    Pkg->Group = Grp.Index();
 607    // all is mapped to the native architecture
 608    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 609    if (unlikely(idxArch == 0))
 610       return false;
 611    Pkg->Arch = idxArch;
 612    Pkg->ID = Cache.HeaderP->PackageCount++;
 613
 614    // Insert the package into our package list
 615    if (Grp->FirstPackage == 0) // the group is new
 616    {
 617       Grp->FirstPackage = Package;
 618       // Insert it into the hash table
 619       map_id_t const Hash = Cache.Hash(Name);
 620       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 621       while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
 622          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 623       Pkg->NextPackage = *insertAt;
 624       *insertAt = Package;
 625    }
 626    else // Group the Packages together
 627    {
 628       // if sibling is provided by another package, this one is too
 629       {
 630          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 631          if (M.end() == false) {
 632             pkgCache::PrvIterator Prv;
 633             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 634             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 635             {
 636                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 637                   continue;
 638                pkgCache::VerIterator Ver = Prv.OwnerVer();
 639                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 640                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 641                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 642                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 643                {
 644                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 645                      continue;
 646                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 647                      return false;
 648                }
 649             }
 650          }
 651       }
 652       // let M-A:foreign package siblings provide this package
 653       {
 654          pkgCache::PkgIterator P;
 655          pkgCache::VerIterator Ver;
 656          Dynamic<pkgCache::PkgIterator> DynP(P);
 657          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 658
 659          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 660          {
 661             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 662                continue;
 663             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 664                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 665                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 666                      return false;
 667          }
 668       }
 669       // and negative dependencies, don't forget negative dependencies
 670       {
 671          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 672          if (M.end() == false) {
 673             pkgCache::DepIterator Dep;
 674             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 675             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 676             {
 677                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 678                   continue;
 679                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 680                      Dep->Type != pkgCache::Dep::Replaces)
 681                   continue;
 682                pkgCache::VerIterator Ver = Dep.ParentVer();
 683                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 684                map_pointer_t * unused = NULL;
 685                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 686                   return false;
 687             }
 688          }
 689       }
 690
 691       // this package is the new last package
 692       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 693       Pkg->NextPackage = LastPkg->NextPackage;
 694       LastPkg->NextPackage = Package;
 695    }
 696    Grp->LastPackage = Package;
 697
 698    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 699    if (Arch == "any")
 700    {
 701       size_t const found = Name.rfind(':');
 702       StringView ArchA = Name.substr(found + 1);
 703       if (ArchA != "any")
 704       {
 705          // ArchA is used inside the loop which might remap (NameA is not used)
 706          Dynamic<StringView> DynArchA(ArchA);
 707          StringView NameA = Name.substr(0, found);
 708          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 709          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 710          if (PkgA.end())
 711          {
 712             Dynamic<StringView> DynNameA(NameA);
 713             if (NewPackage(PkgA, NameA, ArchA) == false)
 714                return false;
 715          }
 716          if (unlikely(PkgA.end()))
 717             return _error->Fatal("NewPackage was successful for %s:%s,"
 718                   "but the package doesn't exist anyhow!",
 719                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 720          else
 721          {
 722             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 723             for (; Prv.end() == false; ++Prv)
 724             {
 725                if (Prv.IsMultiArchImplicit())
 726                   continue;
 727                pkgCache::VerIterator V = Prv.OwnerVer();
 728                if (ArchA != V.ParentPkg().Arch())
 729                   continue;
 730                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 731                   return false;
 732             }
 733             pkgCache::VerIterator V = PkgA.VersionList();
 734             Dynamic<pkgCache::VerIterator> DynV(V);
 735             for (; V.end() == false; ++V)
 736             {
 737                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 738                   return false;
 739             }
 740          }
 741       }
 742    }
 743    return true;
 744 }
 745                                                                         /*}}}*/
 746 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 747 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 748                                            pkgCache::PkgIterator &P,
 749                                            pkgCache::VerIterator &V)
 750 {
 751    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 752    Dynamic<APT::StringView> DynArch(Arch);
 753    map_pointer_t *OldDepLast = NULL;
 754    /* MultiArch handling introduces a lot of implicit Dependencies:
 755       - MultiArch: same → Co-Installable if they have the same version
 756       - All others conflict with all other group members */
 757    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 758    pkgCache::PkgIterator D = G.PackageList();
 759    Dynamic<pkgCache::PkgIterator> DynD(D);
 760    map_stringitem_t const VerStrIdx = V->VerStr;
 761    for (; D.end() != true; D = G.NextPkg(D))
 762    {
 763       if (Arch == D.Arch() || D->VersionList == 0)
 764          continue;
 765       /* We allow only one installed arch at the time
 766          per group, therefore each group member conflicts
 767          with all other group members */
 768       if (coInstall == true)
 769       {
 770          // Replaces: ${self}:other ( << ${binary:Version})
 771          NewDepends(D, V, VerStrIdx,
 772                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 773                     OldDepLast);
 774          // Breaks: ${self}:other (!= ${binary:Version})
 775          NewDepends(D, V, VerStrIdx,
 776                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 777                     OldDepLast);
 778       } else {
 779          // Conflicts: ${self}:other
 780          NewDepends(D, V, 0,
 781                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 782                     OldDepLast);
 783       }
 784    }
 785    return true;
 786 }
 787 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 788                                            pkgCache::PkgIterator &D)
 789 {
 790    /* MultiArch handling introduces a lot of implicit Dependencies:
 791       - MultiArch: same → Co-Installable if they have the same version
 792       - All others conflict with all other group members */
 793    map_pointer_t *OldDepLast = NULL;
 794    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 795    if (coInstall == true)
 796    {
 797       map_stringitem_t const VerStrIdx = V->VerStr;
 798       // Replaces: ${self}:other ( << ${binary:Version})
 799       NewDepends(D, V, VerStrIdx,
 800                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 801                  OldDepLast);
 802       // Breaks: ${self}:other (!= ${binary:Version})
 803       NewDepends(D, V, VerStrIdx,
 804                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 805                  OldDepLast);
 806    } else {
 807       // Conflicts: ${self}:other
 808       NewDepends(D, V, 0,
 809                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 810                  OldDepLast);
 811    }
 812    return true;
 813 }
 814
 815                                                                         /*}}}*/
 816 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* */
 819 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 820                                    ListParser &List)
 821 {
 822    if (CurrentFile == 0)
 823       return true;
 824
 825    // Get a structure
 826    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 827    if (VerFile == 0)
 828       return false;
 829
 830    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 831    VF->File = CurrentFile - Cache.PkgFileP;
 832
 833    // Link it to the end of the list
 834    map_pointer_t *Last = &Ver->FileList;
 835    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 836       Last = &V->NextFile;
 837    VF->NextFile = *Last;
 838    *Last = VF.Index();
 839
 840    VF->Offset = List.Offset();
 841    VF->Size = List.Size();
 842    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 843       Cache.HeaderP->MaxVerFileSize = VF->Size;
 844    Cache.HeaderP->VerFileCount++;
 845
 846    return true;
 847 }
 848                                                                         /*}}}*/
 849 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 850 // ---------------------------------------------------------------------
 851 /* This puts a version structure in the linked list */
 852 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 853                                             APT::StringView const &VerStr,
 854                                             map_pointer_t const ParentPkg,
 855                                             unsigned short const Hash,
 856                                             map_pointer_t const Next)
 857 {
 858    // Get a structure
 859    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 860    if (Version == 0)
 861       return 0;
 862
 863    // Fill it in
 864    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 865    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 866    Ver->NextVer = Next;
 867    Ver->ParentPkg = ParentPkg;
 868    Ver->Hash = Hash;
 869    Ver->ID = Cache.HeaderP->VersionCount++;
 870
 871    // try to find the version string in the group for reuse
 872    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 873    pkgCache::GrpIterator Grp = Pkg.Group();
 874    if (Pkg.end() == false && Grp.end() == false)
 875    {
 876       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 877       {
 878          if (Pkg == P)
 879             continue;
 880          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 881          {
 882             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 883             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 884             {
 885                Ver->VerStr = V->VerStr;
 886                return Version;
 887             }
 888             else if (cmp < 0)
 889                break;
 890          }
 891       }
 892    }
 893    // haven't found the version string, so create
 894    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 895    if (unlikely(idxVerStr == 0))
 896       return 0;
 897    Ver->VerStr = idxVerStr;
 898    return Version;
 899 }
 900                                                                         /*}}}*/
 901 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 902 // ---------------------------------------------------------------------
 903 /* */
 904 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 905                                    ListParser &List)
 906 {
 907    if (CurrentFile == 0)
 908       return true;
 909
 910    // Get a structure
 911    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 912    if (DescFile == 0)
 913       return false;
 914
 915    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 916    DF->File = CurrentFile - Cache.PkgFileP;
 917
 918    // Link it to the end of the list
 919    map_pointer_t *Last = &Desc->FileList;
 920    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 921       Last = &D->NextFile;
 922
 923    DF->NextFile = *Last;
 924    *Last = DF.Index();
 925
 926    DF->Offset = List.Offset();
 927    DF->Size = List.Size();
 928    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 929       Cache.HeaderP->MaxDescFileSize = DF->Size;
 930    Cache.HeaderP->DescFileCount++;
 931
 932    return true;
 933 }
 934                                                                         /*}}}*/
 935 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* This puts a description structure in the linked list */
 938 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 939                                             const string &Lang,
 940                                             APT::StringView md5sum,
 941                                             map_stringitem_t const idxmd5str)
 942 {
 943    // Get a structure
 944    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 945    if (Description == 0)
 946       return 0;
 947
 948    // Fill it in
 949    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 950    Desc->ID = Cache.HeaderP->DescriptionCount++;
 951    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 952    if (unlikely(idxlanguage_code == 0))
 953       return 0;
 954    Desc->language_code = idxlanguage_code;
 955
 956    if (idxmd5str != 0)
 957       Desc->md5sum = idxmd5str;
 958    else
 959    {
 960       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
 961       if (unlikely(idxmd5sum == 0))
 962          return 0;
 963       Desc->md5sum = idxmd5sum;
 964    }
 965
 966    return Description;
 967 }
 968                                                                         /*}}}*/
 969 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 970 // ---------------------------------------------------------------------
 971 /* This creates a dependency element in the tree. It is linked to the
 972    version and to the package that it is pointing to. */
 973 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 974                                    pkgCache::VerIterator &Ver,
 975                                    map_pointer_t const Version,
 976                                    uint8_t const Op,
 977                                    uint8_t const Type,
 978                                    map_pointer_t* &OldDepLast)
 979 {
 980    void const * const oldMap = Map.Data();
 981    // Get a structure
 982    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 983    if (unlikely(Dependency == 0))
 984       return false;
 985
 986    bool isDuplicate = false;
 987    map_pointer_t DependencyData = 0;
 988    map_pointer_t PreviousData = 0;
 989    if (Pkg->RevDepends != 0)
 990    {
 991       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 992       DependencyData = L->DependencyData;
 993       do {
 994          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 995          if (Version > D->Version)
 996             break;
 997          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 998          {
 999             isDuplicate = true;
1000             break;
1001          }
1002          PreviousData = DependencyData;
1003          DependencyData = D->NextData;
1004       } while (DependencyData != 0);
1005    }
1006
1007    if (isDuplicate == false)
1008    {
1009       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1010       if (unlikely(DependencyData == 0))
1011         return false;
1012    }
1013
1014    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1015    Link->ParentVer = Ver.Index();
1016    Link->DependencyData = DependencyData;
1017    Link->ID = Cache.HeaderP->DependsCount++;
1018
1019    pkgCache::DepIterator Dep(Cache, Link);
1020    if (isDuplicate == false)
1021    {
1022       Dep->Type = Type;
1023       Dep->CompareOp = Op;
1024       Dep->Version = Version;
1025       Dep->Package = Pkg.Index();
1026       ++Cache.HeaderP->DependsDataCount;
1027       if (PreviousData != 0)
1028       {
1029          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1030          Dep->NextData = D->NextData;
1031          D->NextData = DependencyData;
1032       }
1033       else if (Pkg->RevDepends != 0)
1034       {
1035          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1036          Dep->NextData = D->DependencyData;
1037       }
1038    }
1039
1040    if (isDuplicate == true || PreviousData != 0)
1041    {
1042       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1043       Link->NextRevDepends = L->NextRevDepends;
1044       L->NextRevDepends = Dependency;
1045    }
1046    else
1047    {
1048       Link->NextRevDepends = Pkg->RevDepends;
1049       Pkg->RevDepends = Dependency;
1050    }
1051
1052
1053    // Do we know where to link the Dependency to?
1054    if (OldDepLast == NULL)
1055    {
1056       OldDepLast = &Ver->DependsList;
1057       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1058          OldDepLast = &D->NextDepends;
1059    } else if (oldMap != Map.Data())
1060       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1061
1062    Dep->NextDepends = *OldDepLast;
1063    *OldDepLast = Dependency;
1064    OldDepLast = &Dep->NextDepends;
1065    return true;
1066 }
1067                                                                         /*}}}*/
1068 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1069 // ---------------------------------------------------------------------
1070 /* This creates a Group and the Package to link this dependency to if
1071    needed and handles also the caching of the old endpoint */
1072 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1073                                                StringView PackageName,
1074                                                StringView Arch,
1075                                                StringView Version,
1076                                                uint8_t const Op,
1077                                                uint8_t const Type)
1078 {
1079    pkgCache::GrpIterator Grp;
1080    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1081    Dynamic<StringView> DynPackageName(PackageName);
1082    Dynamic<StringView> DynArch(Arch);
1083    Dynamic<StringView> DynVersion(Version);
1084    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1085       return false;
1086
1087    map_stringitem_t idxVersion = 0;
1088    if (Version.empty() == false)
1089    {
1090       int const CmpOp = Op & 0x0F;
1091       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1092       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1093          idxVersion = Ver->VerStr;
1094
1095       if (idxVersion == 0)
1096       {
1097          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1098          if (unlikely(idxVersion == 0))
1099             return false;
1100       }
1101    }
1102
1103    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1104          Type == pkgCache::Dep::Conflicts ||
1105          Type == pkgCache::Dep::Replaces);
1106
1107    pkgCache::PkgIterator Pkg;
1108    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1109    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1110    {
1111       // Locate the target package
1112       Pkg = Grp.FindPkg(Arch);
1113       if (Pkg.end() == true) {
1114          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1115             return false;
1116       }
1117
1118       /* Caching the old end point speeds up generation substantially */
1119       if (OldDepVer != Ver) {
1120          OldDepLast = NULL;
1121          OldDepVer = Ver;
1122       }
1123
1124       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1125    }
1126    else
1127    {
1128       /* Caching the old end point speeds up generation substantially */
1129       if (OldDepVer != Ver) {
1130          OldDepLast = NULL;
1131          OldDepVer = Ver;
1132       }
1133
1134       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1135       {
1136          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1137                return false;
1138       }
1139    }
1140    return true;
1141 }
1142                                                                         /*}}}*/
1143 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1144 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1145                                                 StringView PkgName,
1146                                                 StringView PkgArch,
1147                                                 StringView Version,
1148                                                 uint8_t const Flags)
1149 {
1150    pkgCache const &Cache = Owner->Cache;
1151    Dynamic<StringView> DynPkgName(PkgName);
1152    Dynamic<StringView> DynArch(PkgArch);
1153    Dynamic<StringView> DynVersion(Version);
1154
1155    // We do not add self referencing provides
1156    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1157         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1158          (Version.empty() || Version == Ver.VerStr()))
1159       return true;
1160
1161    // Locate the target package
1162    pkgCache::PkgIterator Pkg;
1163    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1164    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1165       return false;
1166
1167    map_stringitem_t idxProvideVersion = 0;
1168    if (Version.empty() == false) {
1169       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1170       if (unlikely(idxProvideVersion == 0))
1171          return false;
1172    }
1173    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1174 }
1175 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1176                                     pkgCache::PkgIterator &Pkg,
1177                                     map_pointer_t const ProvideVersion,
1178                                     uint8_t const Flags)
1179 {
1180    // Get a structure
1181    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1182    if (unlikely(Provides == 0))
1183       return false;
1184    ++Cache.HeaderP->ProvidesCount;
1185
1186    // Fill it in
1187    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1188    Prv->Version = Ver.Index();
1189    Prv->ProvideVersion = ProvideVersion;
1190    Prv->Flags = Flags;
1191    Prv->NextPkgProv = Ver->ProvidesList;
1192    Ver->ProvidesList = Prv.Index();
1193
1194    // Link it to the package
1195    Prv->ParentPkg = Pkg.Index();
1196    Prv->NextProvides = Pkg->ProvidesList;
1197    Pkg->ProvidesList = Prv.Index();
1198    return true;
1199 }
1200                                                                         /*}}}*/
1201 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1202 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1203                                 StringView Version, uint8_t const Flags) {
1204    pkgCache &Cache = Owner->Cache;
1205    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1206    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1207    Dynamic<StringView> DynPackage(Package);
1208    Dynamic<StringView> DynVersion(Version);
1209
1210    if (Grp.end() == true)
1211       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1212    else
1213    {
1214       map_stringitem_t idxProvideVersion = 0;
1215       if (Version.empty() == false) {
1216          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1217          if (unlikely(idxProvideVersion == 0))
1218             return false;
1219       }
1220
1221       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1222       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1223       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1224       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1225       pkgCache::PkgIterator Pkg;
1226       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1227       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1228       {
1229          if (isImplicit && OwnerPkg == Pkg)
1230             continue;
1231          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1232             continue;
1233          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1234             return false;
1235       }
1236    }
1237    return true;
1238 }
1239                                                                         /*}}}*/
1240 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1241       pkgCache::VerIterator const &Ver)
1242 {
1243    return Hash == Ver->Hash;
1244 }
1245                                                                         /*}}}*/
1246 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1247 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1248                                    unsigned long Flags)
1249 {
1250    if (File.empty() && Site.empty())
1251    {
1252       CurrentRlsFile = NULL;
1253       return true;
1254    }
1255
1256    // Get some space for the structure
1257    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1258    if (unlikely(idxFile == 0))
1259       return false;
1260    CurrentRlsFile = Cache.RlsFileP + idxFile;
1261
1262    // Fill it in
1263    map_stringitem_t const idxFileName = WriteStringInMap(File);
1264    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1265    if (unlikely(idxFileName == 0 || idxSite == 0))
1266       return false;
1267    CurrentRlsFile->FileName = idxFileName;
1268    CurrentRlsFile->Site = idxSite;
1269    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1270    CurrentRlsFile->Flags = Flags;
1271    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1272    RlsFileName = File;
1273    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1274    Cache.HeaderP->ReleaseFileCount++;
1275
1276    return true;
1277 }
1278                                                                         /*}}}*/
1279 // ListParser::NewTag - Create a Tag element                            /*{{{*/
1280 // ---------------------------------------------------------------------
1281 /* */
1282 bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
1283                                            const char *NameStart,
1284                                            unsigned int NameSize)
1285 {
1286    return Owner->NewTag(Ver, NameStart, NameSize);
1287 }
1288 bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
1289                                            const char *NameStart,
1290                                            unsigned int NameSize)
1291 {
1292    // Get a structure
1293    unsigned long Tagg = AllocateInMap(sizeof(pkgCache::Tag));
1294    if (Tagg == 0)
1295       return false;
1296    Cache.HeaderP->TagCount++;
1297
1298    // Fill it in
1299    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
1300    Tg->Name = WriteStringInMap(NameStart,NameSize);
1301    if (Tg->Name == 0)
1302       return false;
1303    Tg->NextTag = Ver->TagList;
1304    Ver->TagList = Tg.Index();
1305
1306    return true;
1307 }
1308                                                                         /*}}}*/
1309 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1310 // ---------------------------------------------------------------------
1311 /* This is used to select which file is to be associated with all newly
1312    added versions. The caller is responsible for setting the IMS fields. */
1313 bool pkgCacheGenerator::SelectFile(std::string const &File,
1314                                    pkgIndexFile const &Index,
1315                                    std::string const &Architecture,
1316                                    std::string const &Component,
1317                                    unsigned long const Flags)
1318 {
1319    // Get some space for the structure
1320    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1321    if (unlikely(idxFile == 0))
1322       return false;
1323    CurrentFile = Cache.PkgFileP + idxFile;
1324
1325    // Fill it in
1326    map_stringitem_t const idxFileName = WriteStringInMap(File);
1327    if (unlikely(idxFileName == 0))
1328       return false;
1329    CurrentFile->FileName = idxFileName;
1330    CurrentFile->NextFile = Cache.HeaderP->FileList;
1331    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1332    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1333    if (unlikely(idxIndexType == 0))
1334       return false;
1335    CurrentFile->IndexType = idxIndexType;
1336    if (Architecture.empty())
1337       CurrentFile->Architecture = 0;
1338    else
1339    {
1340       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1341       if (unlikely(arch == 0))
1342          return false;
1343       CurrentFile->Architecture = arch;
1344    }
1345    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1346    if (unlikely(component == 0))
1347       return false;
1348    CurrentFile->Component = component;
1349    CurrentFile->Flags = Flags;
1350    if (CurrentRlsFile != NULL)
1351       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1352    else
1353       CurrentFile->Release = 0;
1354    PkgFileName = File;
1355    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1356    Cache.HeaderP->PackageFileCount++;
1357
1358    if (Progress != 0)
1359       Progress->SubProgress(Index.Size());
1360    return true;
1361 }
1362                                                                         /*}}}*/
1363 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1364 // ---------------------------------------------------------------------
1365 /* This is used to create handles to strings. Given the same text it
1366    always returns the same number */
1367 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1368                                                  unsigned int Size)
1369 {
1370    auto strings = &strMixed;
1371    switch(type) {
1372       case MIXED: strings = &strMixed; break;
1373       case PKGNAME: strings = &strPkgNames; break;
1374       case VERSIONNUMBER: strings = &strVersions; break;
1375       case SECTION: strings = &strSections; break;
1376       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1377    }
1378
1379    auto const item = strings->find({S, Size, nullptr, 0});
1380    if (item != strings->end())
1381       return item->item;
1382
1383    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1384    strings->insert({nullptr, Size, this, idxString});
1385    return idxString;
1386 }
1387                                                                         /*}}}*/
1388 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1389 // ---------------------------------------------------------------------
1390 /* This just verifies that each file in the list of index files exists,
1391    has matching attributes with the cache and the cache does not have
1392    any extra files. */
1393 class APT_HIDDEN ScopedErrorRevert {
1394 public:
1395    ScopedErrorRevert() { _error->PushToStack(); }
1396    ~ScopedErrorRevert() { _error->RevertToStack(); }
1397 };
1398 static bool CheckValidity(const string &CacheFile,
1399                           pkgSourceList &List,
1400                           FileIterator const Start,
1401                           FileIterator const End,
1402                           MMap **OutMap = 0,
1403                           pkgCache **OutCache = 0)
1404 {
1405    ScopedErrorRevert ser;
1406    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1407    // No file, certainly invalid
1408    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1409    {
1410       if (Debug == true)
1411          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1412       return false;
1413    }
1414
1415    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1416    {
1417       if (Debug == true)
1418          std::clog << "sources.list is newer than the cache" << std::endl;
1419       return false;
1420    }
1421
1422    // Map it
1423    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1424    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1425    if (unlikely(Map->validData()) == false)
1426       return false;
1427    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1428    pkgCache &Cache = *CacheP.get();
1429    if (_error->PendingError() || Map->Size() == 0)
1430    {
1431       if (Debug == true)
1432          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1433       return _error->ReturnError();
1434    }
1435
1436    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1437    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1438    std::vector<pkgIndexFile *> Files;
1439    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1440    {
1441       if (Debug == true)
1442          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1443       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1444       if (RlsFile.end() == true)
1445       {
1446          if (Debug == true)
1447             std::clog << "FindInCache returned end-Pointer" << std::endl;
1448          return false;
1449       }
1450
1451       RlsVisited[RlsFile->ID] = true;
1452       if (Debug == true)
1453          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1454
1455       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1456       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1457             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1458    }
1459    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1460       if (RlsVisited[I] == false)
1461       {
1462          if (Debug == true)
1463             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1464          return false;
1465       }
1466
1467    std::copy(Start, End, std::back_inserter(Files));
1468
1469    /* Now we check every index file, see if it is in the cache,
1470       verify the IMS data and check that it is on the disk too.. */
1471    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1472    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1473    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1474    {
1475       if (Debug == true)
1476          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1477       if ((*PkgFile)->Exists() == false)
1478       {
1479          if (Debug == true)
1480             std::clog << "file doesn't exist" << std::endl;
1481          continue;
1482       }
1483
1484       // FindInCache is also expected to do an IMS check.
1485       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1486       if (File.end() == true)
1487       {
1488          if (Debug == true)
1489             std::clog << "FindInCache returned end-Pointer" << std::endl;
1490          return false;
1491       }
1492
1493       Visited[File->ID] = true;
1494       if (Debug == true)
1495          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1496    }
1497
1498    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1499       if (Visited[I] == false)
1500       {
1501          if (Debug == true)
1502             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1503          return false;
1504       }
1505
1506    if (_error->PendingError() == true)
1507    {
1508       if (Debug == true)
1509       {
1510          std::clog << "Validity failed because of pending errors:" << std::endl;
1511          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1512       }
1513       return _error->ReturnError();
1514    }
1515
1516    if (OutMap != 0)
1517       *OutMap = Map.release();
1518    if (OutCache != 0)
1519       *OutCache = CacheP.release();
1520    return true;
1521 }
1522                                                                         /*}}}*/
1523 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1524 // ---------------------------------------------------------------------
1525 /* Size is kind of an abstract notion that is only used for the progress
1526    meter */
1527 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1528 {
1529    map_filesize_t TotalSize = 0;
1530    if (List !=  NULL)
1531    {
1532       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1533       {
1534          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1535          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1536             if ((*j)->HasPackages() == true)
1537                TotalSize += (*j)->Size();
1538       }
1539    }
1540
1541    for (; Start < End; ++Start)
1542    {
1543       if ((*Start)->HasPackages() == false)
1544          continue;
1545       TotalSize += (*Start)->Size();
1546    }
1547    return TotalSize;
1548 }
1549                                                                         /*}}}*/
1550 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1551 static void BuildCache(pkgCacheGenerator &Gen,
1552                        OpProgress * const Progress,
1553                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1554                        pkgSourceList const * const List,
1555                        FileIterator const Start, FileIterator const End)
1556 {
1557    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1558       if (I->HasPackages() == false)
1559          return;
1560
1561       if (I->Exists() == false)
1562          return;
1563
1564       if (I->FindInCache(Gen.GetCache()).end() == false)
1565       {
1566          _error->Warning("Duplicate sources.list entry %s",
1567                I->Describe().c_str());
1568          return;
1569       }
1570
1571       map_filesize_t const Size = I->Size();
1572       if (Progress != NULL)
1573          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1574       CurrentSize += Size;
1575
1576       if (I->Merge(Gen,Progress) == false) {
1577          _error->ReturnError();
1578          return;
1579       }
1580    };
1581
1582    if (List !=  NULL)
1583    {
1584       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1585       {
1586          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1587          {
1588             _error->Warning("Duplicate sources.list entry %s",
1589                   (*i)->Describe().c_str());
1590             continue;
1591          }
1592
1593          if ((*i)->Merge(Gen, Progress) == false) {
1594             _error->ReturnError();
1595             continue;
1596          }
1597
1598          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1599          if (Indexes != NULL)
1600             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1601       }
1602    }
1603
1604    if (Start != End)
1605    {
1606       Gen.SelectReleaseFile("", "");
1607       std::for_each(Start, End, indexFileMerge);
1608    }
1609 }
1610                                                                         /*}}}*/
1611 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1612 // ---------------------------------------------------------------------
1613 /* This makes sure that the status cache (the cache that has all
1614    index files from the sources list and all local ones) is ready
1615    to be mmaped. If OutMap is not zero then a MMap object representing
1616    the cache will be stored there. This is pretty much mandetory if you
1617    are using AllowMem. AllowMem lets the function be run as non-root
1618    where it builds the cache 'fast' into a memory buffer. */
1619 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1620 {
1621    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1622    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1623    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1624    Flags |= MMap::Moveable;
1625    if (_config->FindB("APT::Cache-Fallback", false) == true)
1626       Flags |= MMap::Fallback;
1627    if (CacheF != NULL)
1628       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1629    else
1630       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1631 }
1632 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1633       std::string const &FileName)
1634 {
1635    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1636    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1637       return false;
1638
1639    fchmod(SCacheF.Fd(),0644);
1640
1641    // Write out the main data
1642    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1643       return _error->Error(_("IO Error saving source cache"));
1644
1645    // Write out the proper header
1646    Gen->GetCache().HeaderP->Dirty = false;
1647    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1648    if (SCacheF.Seek(0) == false ||
1649          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1650       return _error->Error(_("IO Error saving source cache"));
1651    Gen->GetCache().HeaderP->Dirty = true;
1652    return true;
1653 }
1654 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1655       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1656 {
1657    Map.reset(CreateDynamicMMap(NULL, 0));
1658    if (unlikely(Map->validData()) == false)
1659       return false;
1660    FileFd CacheF(FileName, FileFd::ReadOnly);
1661    if (CacheF.IsOpen() == false || CacheF.Failed())
1662       return false;
1663    _error->PushToStack();
1664    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1665    bool const newError = _error->PendingError();
1666    _error->MergeWithStack();
1667    if (alloc == 0 && newError)
1668       return _error->ReturnError();
1669    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1670       return false;
1671    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1672    return Gen->Start();
1673 }
1674 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1675                         MMap **OutMap, bool AllowMem)
1676    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1677 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1678                         MMap **OutMap,bool)
1679 {
1680    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1681 }
1682 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1683                         MMap **OutMap,pkgCache **OutCache, bool)
1684 {
1685    // FIXME: deprecate the ignored AllowMem parameter
1686    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1687
1688    std::vector<pkgIndexFile *> Files;
1689    if (_system->AddStatusFiles(Files) == false)
1690       return false;
1691
1692    // Decide if we can write to the files..
1693    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1694    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1695
1696    // ensure the cache directory exists
1697    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1698    {
1699       string dir = _config->FindDir("Dir::Cache");
1700       size_t const len = dir.size();
1701       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1702          dir = dir.substr(0, len - 5);
1703       if (CacheFile.empty() == false)
1704          CreateDirectory(dir, flNotFile(CacheFile));
1705       if (SrcCacheFile.empty() == false)
1706          CreateDirectory(dir, flNotFile(SrcCacheFile));
1707    }
1708
1709    if (Progress != NULL)
1710       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1711
1712    bool pkgcache_fine = false;
1713    bool srcpkgcache_fine = false;
1714    bool volatile_fine = List.GetVolatileFiles().empty();
1715
1716    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1717                      volatile_fine ? OutCache : NULL) == true)
1718    {
1719       if (Debug == true)
1720          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1721       pkgcache_fine = true;
1722       srcpkgcache_fine = true;
1723    }
1724    if (pkgcache_fine == false)
1725    {
1726       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1727       {
1728          if (Debug == true)
1729             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1730          srcpkgcache_fine = true;
1731       }
1732    }
1733
1734    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1735    {
1736       if (Progress != NULL)
1737          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1738       return true;
1739    }
1740
1741    bool Writeable = false;
1742    if (srcpkgcache_fine == false || pkgcache_fine == false)
1743    {
1744       if (CacheFile.empty() == false)
1745          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1746       else if (SrcCacheFile.empty() == false)
1747          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1748
1749       if (Debug == true)
1750          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1751    }
1752
1753    // At this point we know we need to construct something, so get storage ready
1754    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1755    if (unlikely(Map->validData()) == false)
1756       return false;
1757    if (Debug == true)
1758       std::clog << "Open memory Map (not filebased)" << std::endl;
1759
1760    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1761    map_filesize_t CurrentSize = 0;
1762    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1763    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1764    if (srcpkgcache_fine == true && pkgcache_fine == false)
1765    {
1766       if (Debug == true)
1767          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1768       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1769          return false;
1770       srcpkgcache_fine = true;
1771       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1772    }
1773    else if (srcpkgcache_fine == false)
1774    {
1775       if (Debug == true)
1776          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1777       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1778       if (Gen->Start() == false)
1779          return false;
1780
1781       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1782       BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1783                Files.end(),Files.end());
1784
1785       if (Writeable == true && SrcCacheFile.empty() == false)
1786          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1787             return false;
1788    }
1789
1790    if (pkgcache_fine == false)
1791    {
1792       if (Debug == true)
1793          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1794       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1795                Files.begin(), Files.end());
1796
1797       if (Writeable == true && CacheFile.empty() == false)
1798          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1799             return false;
1800    }
1801
1802    if (Debug == true)
1803       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1804
1805    if (volatile_fine == false)
1806    {
1807       if (Gen == nullptr)
1808       {
1809          if (Debug == true)
1810             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1811          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1812             return false;
1813       }
1814
1815       Files = List.GetVolatileFiles();
1816       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1817                Files.begin(), Files.end());
1818    }
1819
1820    if (OutMap != nullptr)
1821       *OutMap = Map.release();
1822
1823    if (Debug == true)
1824       std::clog << "Everything is ready for shipping" << std::endl;
1825    return true;
1826 }
1827                                                                         /*}}}*/
1828 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1829 class APT_HIDDEN ScopedErrorMerge {
1830 public:
1831    ScopedErrorMerge() { _error->PushToStack(); }
1832    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1833 };
1834 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1835    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1836 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1837 {
1838    std::vector<pkgIndexFile *> Files;
1839    if (_system->AddStatusFiles(Files) == false)
1840       return false;
1841
1842    ScopedErrorMerge sem;
1843    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1844    if (unlikely(Map->validData()) == false)
1845       return false;
1846    map_filesize_t CurrentSize = 0;
1847    map_filesize_t TotalSize = 0;
1848    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1849
1850    // Build the status cache
1851    if (Progress != NULL)
1852       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1853    pkgCacheGenerator Gen(Map.get(),Progress);
1854    if (Gen.Start() == false)
1855       return false;
1856    if (_error->PendingError() == true)
1857       return _error->ReturnError();
1858    BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1859                   Files.begin(), Files.end());
1860    // We've passed the point of no return
1861    _error->ReturnError();
1862
1863    *OutMap = Map.release();
1864
1865    return true;
1866 }
1867                                                                         /*}}}*/
1868 // IsDuplicateDescription                                               /*{{{*/
1869 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
1870                             APT::StringView CurMd5, std::string const &CurLang)
1871 {
1872    // Descriptions in the same link-list have all the same md5
1873    if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1874       return false;
1875    for (; Desc.end() == false; ++Desc)
1876       if (Desc.LanguageCode() == CurLang)
1877          return true;
1878    return false;
1879 }
1880                                                                         /*}}}*/
1881
1882 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1883 pkgCacheListParser::~pkgCacheListParser() {}