apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
  49                             APT::StringView CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return false;
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (_error->PendingError() == true || Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 180         if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 184       if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 199       if (std::get<1>(seen.insert(ViewP)) == false)
 200          continue;
 201       // Ignore views outside of the cache.
 202       if (ViewP->data() < static_cast<const char*>(oldMap)
 203          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 204          continue;
 205       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 206       *ViewP = StringView(data , ViewP->size());
 207    }
 208 }                                                                       /*}}}*/
 209 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 210 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 211                                         const unsigned long &Len) {
 212    size_t oldSize = Map.Size();
 213    void const * const oldMap = Map.Data();
 214    map_stringitem_t const index = Map.WriteString(String, Len);
 215    if (index != 0)
 216       ReMap(oldMap, Map.Data(), oldSize);
 217    return index;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 221 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 222    size_t oldSize = Map.Size();
 223    void const * const oldMap = Map.Data();
 224    map_stringitem_t const index = Map.WriteString(String);
 225    if (index != 0)
 226       ReMap(oldMap, Map.Data(), oldSize);
 227    return index;
 228 }
 229                                                                         /*}}}*/
 230 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 231    size_t oldSize = Map.Size();
 232    void const * const oldMap = Map.Data();
 233    map_pointer_t const index = Map.Allocate(size);
 234    if (index != 0)
 235       ReMap(oldMap, Map.Data(), oldSize);
 236    return index;
 237 }
 238                                                                         /*}}}*/
 239 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This provides the generation of the entries in the cache. Each loop
 242    goes through a single package record from the underlying parse engine. */
 243 bool pkgCacheGenerator::MergeList(ListParser &List,
 244                                   pkgCache::VerIterator *OutVer)
 245 {
 246    List.Owner = this;
 247
 248    unsigned int Counter = 0;
 249    while (List.Step() == true)
 250    {
 251       string const PackageName = List.Package();
 252       if (PackageName.empty() == true)
 253          return false;
 254
 255       Counter++;
 256       if (Counter % 100 == 0 && Progress != 0)
 257          Progress->Progress(List.Offset());
 258
 259       APT::StringView Arch = List.Architecture();
 260       Dynamic<APT::StringView> DynArch(Arch);
 261       APT::StringView Version = List.Version();
 262       Dynamic<APT::StringView> DynVersion(Version);
 263       if (Version.empty() == true && Arch.empty() == true)
 264       {
 265          // package descriptions
 266          if (MergeListGroup(List, PackageName) == false)
 267             return false;
 268          continue;
 269       }
 270
 271       // Get a pointer to the package structure
 272       pkgCache::PkgIterator Pkg;
 273       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 274       if (NewPackage(Pkg, PackageName, Arch) == false)
 275          // TRANSLATOR: The first placeholder is a package name,
 276          // the other two should be copied verbatim as they include debug info
 277          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 278                               PackageName.c_str(), "NewPackage", 1);
 279
 280
 281       if (Version.empty() == true)
 282       {
 283          if (MergeListPackage(List, Pkg) == false)
 284             return false;
 285       }
 286       else
 287       {
 288          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 289             return false;
 290       }
 291
 292       if (OutVer != 0)
 293          return true;
 294    }
 295
 296    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 297       return _error->Error(_("Wow, you exceeded the number of package "
 298                              "names this APT is capable of."));
 299    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 300       return _error->Error(_("Wow, you exceeded the number of versions "
 301                              "this APT is capable of."));
 302    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 303       return _error->Error(_("Wow, you exceeded the number of descriptions "
 304                              "this APT is capable of."));
 305    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 306       return _error->Error(_("Wow, you exceeded the number of dependencies "
 307                              "this APT is capable of."));
 308
 309    return true;
 310 }
 311 // CacheGenerator::MergeListGroup                                       /*{{{*/
 312 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 313 {
 314    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 315    // a group has no data on it's own, only packages have it but these
 316    // stanzas like this come from Translation- files to add descriptions,
 317    // but without a version we don't need a description for it…
 318    if (Grp.end() == true)
 319       return true;
 320    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 321
 322    pkgCache::PkgIterator Pkg;
 323    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 324    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 325       if (MergeListPackage(List, Pkg) == false)
 326          return false;
 327
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // CacheGenerator::MergeListPackage                                     /*{{{*/
 332 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 333 {
 334    // we first process the package, then the descriptions
 335    // (for deb this package processing is in fact a no-op)
 336    pkgCache::VerIterator Ver(Cache);
 337    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 338    if (List.UsePackage(Pkg, Ver) == false)
 339       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 340                            Pkg.Name(), "UsePackage", 1);
 341
 342    // Find the right version to write the description
 343    StringView CurMd5 = List.Description_md5();
 344    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 345    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 346    {
 347       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 348
 349       // a version can only have one md5 describing it
 350       if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
 351          continue;
 352
 353       map_stringitem_t md5idx = VerDesc->md5sum;
 354       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 355       {
 356          // don't add a new description if we have one for the given
 357          // md5 && language
 358          if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
 359             continue;
 360
 361          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 362       }
 363
 364       // we can stop here as all "same" versions will share the description
 365       break;
 366    }
 367
 368    return true;
 369 }
 370                                                                         /*}}}*/
 371 // CacheGenerator::MergeListVersion                                     /*{{{*/
 372 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 373                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 374 {
 375    pkgCache::VerIterator Ver = Pkg.VersionList();
 376    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 377    map_pointer_t *LastVer = &Pkg->VersionList;
 378    void const * oldMap = Map.Data();
 379
 380    unsigned short const Hash = List.VersionHash();
 381    if (Ver.end() == false)
 382    {
 383       /* We know the list is sorted so we use that fact in the search.
 384          Insertion of new versions is done with correct sorting */
 385       int Res = 1;
 386       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 387       {
 388          char const * const VerStr = Ver.VerStr();
 389          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 390                VerStr, VerStr + strlen(VerStr));
 391          // Version is higher as current version - insert here
 392          if (Res > 0)
 393             break;
 394          // Versionstrings are equal - is hash also equal?
 395          if (Res == 0)
 396          {
 397             if (List.SameVersion(Hash, Ver) == true)
 398                break;
 399             // sort (volatile) sources above not-sources like the status file
 400             if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 401             {
 402                auto VF = Ver.FileList();
 403                for (; VF.end() == false; ++VF)
 404                   if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
 405                      break;
 406                if (VF.end() == true)
 407                   break;
 408             }
 409          }
 410          // proceed with the next till we have either the right
 411          // or we found another version (which will be lower)
 412       }
 413
 414       /* We already have a version for this item, record that we saw it */
 415       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 416       {
 417          if (List.UsePackage(Pkg,Ver) == false)
 418             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                                  Pkg.Name(), "UsePackage", 2);
 420
 421          if (NewFileVer(Ver,List) == false)
 422             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 423                                  Pkg.Name(), "NewFileVer", 1);
 424
 425          // Read only a single record and return
 426          if (OutVer != 0)
 427          {
 428             *OutVer = Ver;
 429             return true;
 430          }
 431
 432          return true;
 433       }
 434    }
 435
 436    // Add a new version
 437    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 438    if (unlikely(verindex == 0))
 439       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 440                            Pkg.Name(), "NewVersion", 1);
 441
 442    if (oldMap != Map.Data())
 443          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 444    *LastVer = verindex;
 445
 446    if (unlikely(List.NewVersion(Ver) == false))
 447       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 448                            Pkg.Name(), "NewVersion", 2);
 449
 450    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 451       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 452                            Pkg.Name(), "UsePackage", 3);
 453
 454    if (unlikely(NewFileVer(Ver,List) == false))
 455       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 456                            Pkg.Name(), "NewFileVer", 2);
 457
 458    pkgCache::GrpIterator Grp = Pkg.Group();
 459    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 460
 461    /* If it is the first version of this package we need to add implicit
 462       Multi-Arch dependencies to all other package versions in the group now -
 463       otherwise we just add them for this new version */
 464    if (Pkg.VersionList()->NextVer == 0)
 465    {
 466       pkgCache::PkgIterator P = Grp.PackageList();
 467       Dynamic<pkgCache::PkgIterator> DynP(P);
 468       for (; P.end() != true; P = Grp.NextPkg(P))
 469       {
 470          if (P->ID == Pkg->ID)
 471             continue;
 472          pkgCache::VerIterator V = P.VersionList();
 473          Dynamic<pkgCache::VerIterator> DynV(V);
 474          for (; V.end() != true; ++V)
 475             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 476                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 477                                     Pkg.Name(), "AddImplicitDepends", 1);
 478       }
 479    }
 480    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482                            Pkg.Name(), "AddImplicitDepends", 2);
 483
 484    // Read only a single record and return
 485    if (OutVer != 0)
 486    {
 487       *OutVer = Ver;
 488       return true;
 489    }
 490
 491    /* Record the Description(s) based on their master md5sum */
 492    StringView CurMd5 = List.Description_md5();
 493
 494    /* Before we add a new description we first search in the group for
 495       a version with a description of the same MD5 - if so we reuse this
 496       description group instead of creating our own for this version */
 497    for (pkgCache::PkgIterator P = Grp.PackageList();
 498         P.end() == false; P = Grp.NextPkg(P))
 499    {
 500       for (pkgCache::VerIterator V = P.VersionList();
 501            V.end() == false; ++V)
 502       {
 503          if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
 504             continue;
 505          Ver->DescriptionList = V->DescriptionList;
 506       }
 507    }
 508
 509    // We haven't found reusable descriptions, so add the first description(s)
 510    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 511    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 512    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 513       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 514          return false;
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 519 {
 520    pkgCache::DescIterator Desc;
 521    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 522
 523    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 524    if (unlikely(descindex == 0))
 525       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 526             Ver.ParentPkg().Name(), "NewDescription", 1);
 527
 528    md5idx = Desc->md5sum;
 529    Desc->ParentPkg = Ver.ParentPkg().Index();
 530
 531    // we add at the end, so that the start is constant as we need
 532    // that to be able to efficiently share these lists
 533    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 534    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 535    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 536    *LastNextDesc = descindex;
 537
 538    if (NewFileDesc(Desc,List) == false)
 539       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 540             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 541
 542    return true;
 543 }
 544                                                                         /*}}}*/
 545                                                                         /*}}}*/
 546 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* This creates a new group structure and adds it to the hash table */
 549 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 550 {
 551    Dynamic<StringView> DName(Name);
 552    Grp = Cache.FindGrp(Name);
 553    if (Grp.end() == false)
 554       return true;
 555
 556    // Get a structure
 557    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 558    if (unlikely(Group == 0))
 559       return false;
 560
 561    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 562    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 563    if (unlikely(idxName == 0))
 564       return false;
 565    Grp->Name = idxName;
 566
 567    // Insert it into the hash table
 568    unsigned long const Hash = Cache.Hash(Name);
 569    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 570
 571    while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 572       insertAt = &(Cache.GrpP + *insertAt)->Next;
 573    Grp->Next = *insertAt;
 574    *insertAt = Group;
 575
 576    Grp->ID = Cache.HeaderP->GroupCount++;
 577    return true;
 578 }
 579                                                                         /*}}}*/
 580 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* This creates a new package structure and adds it to the hash table */
 583 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 584                                         StringView Arch) {
 585    pkgCache::GrpIterator Grp;
 586    Dynamic<StringView> DName(Name);
 587    Dynamic<StringView> DArch(Arch);
 588    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 589    if (unlikely(NewGroup(Grp, Name) == false))
 590       return false;
 591
 592    Pkg = Grp.FindPkg(Arch);
 593       if (Pkg.end() == false)
 594          return true;
 595
 596    // Get a structure
 597    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 598    if (unlikely(Package == 0))
 599       return false;
 600    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 601
 602    // Set the name, arch and the ID
 603    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 604    Pkg->Group = Grp.Index();
 605    // all is mapped to the native architecture
 606    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 607    if (unlikely(idxArch == 0))
 608       return false;
 609    Pkg->Arch = idxArch;
 610    Pkg->ID = Cache.HeaderP->PackageCount++;
 611
 612    // Insert the package into our package list
 613    if (Grp->FirstPackage == 0) // the group is new
 614    {
 615       Grp->FirstPackage = Package;
 616       // Insert it into the hash table
 617       map_id_t const Hash = Cache.Hash(Name);
 618       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 619       while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
 620          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 621       Pkg->NextPackage = *insertAt;
 622       *insertAt = Package;
 623    }
 624    else // Group the Packages together
 625    {
 626       // if sibling is provided by another package, this one is too
 627       {
 628          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 629          if (M.end() == false) {
 630             pkgCache::PrvIterator Prv;
 631             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 632             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 633             {
 634                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 635                   continue;
 636                pkgCache::VerIterator Ver = Prv.OwnerVer();
 637                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 638                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 639                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 640                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 641                {
 642                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 643                      continue;
 644                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 645                      return false;
 646                }
 647             }
 648          }
 649       }
 650       // let M-A:foreign package siblings provide this package
 651       {
 652          pkgCache::PkgIterator P;
 653          pkgCache::VerIterator Ver;
 654          Dynamic<pkgCache::PkgIterator> DynP(P);
 655          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 656
 657          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 658          {
 659             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 660                continue;
 661             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 662                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 663                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 664                      return false;
 665          }
 666       }
 667       // and negative dependencies, don't forget negative dependencies
 668       {
 669          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 670          if (M.end() == false) {
 671             pkgCache::DepIterator Dep;
 672             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 673             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 674             {
 675                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 676                   continue;
 677                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 678                      Dep->Type != pkgCache::Dep::Replaces)
 679                   continue;
 680                pkgCache::VerIterator Ver = Dep.ParentVer();
 681                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 682                map_pointer_t * unused = NULL;
 683                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 684                   return false;
 685             }
 686          }
 687       }
 688
 689       // this package is the new last package
 690       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 691       Pkg->NextPackage = LastPkg->NextPackage;
 692       LastPkg->NextPackage = Package;
 693    }
 694    Grp->LastPackage = Package;
 695
 696    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 697    if (Arch == "any")
 698    {
 699       size_t const found = Name.rfind(':');
 700       StringView ArchA = Name.substr(found + 1);
 701       if (ArchA != "any")
 702       {
 703          // ArchA is used inside the loop which might remap (NameA is not used)
 704          Dynamic<StringView> DynArchA(ArchA);
 705          StringView NameA = Name.substr(0, found);
 706          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 707          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 708          if (PkgA.end())
 709          {
 710             Dynamic<StringView> DynNameA(NameA);
 711             if (NewPackage(PkgA, NameA, ArchA) == false)
 712                return false;
 713          }
 714          if (unlikely(PkgA.end()))
 715             return _error->Fatal("NewPackage was successful for %s:%s,"
 716                   "but the package doesn't exist anyhow!",
 717                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 718          else
 719          {
 720             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 721             for (; Prv.end() == false; ++Prv)
 722             {
 723                if (Prv.IsMultiArchImplicit())
 724                   continue;
 725                pkgCache::VerIterator V = Prv.OwnerVer();
 726                if (ArchA != V.ParentPkg().Arch())
 727                   continue;
 728                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 729                   return false;
 730             }
 731             pkgCache::VerIterator V = PkgA.VersionList();
 732             Dynamic<pkgCache::VerIterator> DynV(V);
 733             for (; V.end() == false; ++V)
 734             {
 735                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 736                   return false;
 737             }
 738          }
 739       }
 740    }
 741    return true;
 742 }
 743                                                                         /*}}}*/
 744 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 745 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 746                                            pkgCache::PkgIterator &P,
 747                                            pkgCache::VerIterator &V)
 748 {
 749    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 750    Dynamic<APT::StringView> DynArch(Arch);
 751    map_pointer_t *OldDepLast = NULL;
 752    /* MultiArch handling introduces a lot of implicit Dependencies:
 753       - MultiArch: same → Co-Installable if they have the same version
 754       - All others conflict with all other group members */
 755    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 756    pkgCache::PkgIterator D = G.PackageList();
 757    Dynamic<pkgCache::PkgIterator> DynD(D);
 758    map_stringitem_t const VerStrIdx = V->VerStr;
 759    for (; D.end() != true; D = G.NextPkg(D))
 760    {
 761       if (Arch == D.Arch() || D->VersionList == 0)
 762          continue;
 763       /* We allow only one installed arch at the time
 764          per group, therefore each group member conflicts
 765          with all other group members */
 766       if (coInstall == true)
 767       {
 768          // Replaces: ${self}:other ( << ${binary:Version})
 769          NewDepends(D, V, VerStrIdx,
 770                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 771                     OldDepLast);
 772          // Breaks: ${self}:other (!= ${binary:Version})
 773          NewDepends(D, V, VerStrIdx,
 774                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 775                     OldDepLast);
 776       } else {
 777          // Conflicts: ${self}:other
 778          NewDepends(D, V, 0,
 779                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 780                     OldDepLast);
 781       }
 782    }
 783    return true;
 784 }
 785 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 786                                            pkgCache::PkgIterator &D)
 787 {
 788    /* MultiArch handling introduces a lot of implicit Dependencies:
 789       - MultiArch: same → Co-Installable if they have the same version
 790       - All others conflict with all other group members */
 791    map_pointer_t *OldDepLast = NULL;
 792    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 793    if (coInstall == true)
 794    {
 795       map_stringitem_t const VerStrIdx = V->VerStr;
 796       // Replaces: ${self}:other ( << ${binary:Version})
 797       NewDepends(D, V, VerStrIdx,
 798                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 799                  OldDepLast);
 800       // Breaks: ${self}:other (!= ${binary:Version})
 801       NewDepends(D, V, VerStrIdx,
 802                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 803                  OldDepLast);
 804    } else {
 805       // Conflicts: ${self}:other
 806       NewDepends(D, V, 0,
 807                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 808                  OldDepLast);
 809    }
 810    return true;
 811 }
 812
 813                                                                         /*}}}*/
 814 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 815 // ---------------------------------------------------------------------
 816 /* */
 817 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 818                                    ListParser &List)
 819 {
 820    if (CurrentFile == 0)
 821       return true;
 822
 823    // Get a structure
 824    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 825    if (VerFile == 0)
 826       return false;
 827
 828    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 829    VF->File = CurrentFile - Cache.PkgFileP;
 830
 831    // Link it to the end of the list
 832    map_pointer_t *Last = &Ver->FileList;
 833    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 834       Last = &V->NextFile;
 835    VF->NextFile = *Last;
 836    *Last = VF.Index();
 837
 838    VF->Offset = List.Offset();
 839    VF->Size = List.Size();
 840    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 841       Cache.HeaderP->MaxVerFileSize = VF->Size;
 842    Cache.HeaderP->VerFileCount++;
 843
 844    return true;
 845 }
 846                                                                         /*}}}*/
 847 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This puts a version structure in the linked list */
 850 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 851                                             APT::StringView const &VerStr,
 852                                             map_pointer_t const ParentPkg,
 853                                             unsigned short const Hash,
 854                                             map_pointer_t const Next)
 855 {
 856    // Get a structure
 857    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 858    if (Version == 0)
 859       return 0;
 860
 861    // Fill it in
 862    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 863    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 864    Ver->NextVer = Next;
 865    Ver->ParentPkg = ParentPkg;
 866    Ver->Hash = Hash;
 867    Ver->ID = Cache.HeaderP->VersionCount++;
 868
 869    // try to find the version string in the group for reuse
 870    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 871    pkgCache::GrpIterator Grp = Pkg.Group();
 872    if (Pkg.end() == false && Grp.end() == false)
 873    {
 874       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 875       {
 876          if (Pkg == P)
 877             continue;
 878          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 879          {
 880             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 881             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 882             {
 883                Ver->VerStr = V->VerStr;
 884                return Version;
 885             }
 886             else if (cmp < 0)
 887                break;
 888          }
 889       }
 890    }
 891    // haven't found the version string, so create
 892    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 893    if (unlikely(idxVerStr == 0))
 894       return 0;
 895    Ver->VerStr = idxVerStr;
 896    return Version;
 897 }
 898                                                                         /*}}}*/
 899 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 900 // ---------------------------------------------------------------------
 901 /* */
 902 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 903                                    ListParser &List)
 904 {
 905    if (CurrentFile == 0)
 906       return true;
 907
 908    // Get a structure
 909    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 910    if (DescFile == 0)
 911       return false;
 912
 913    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 914    DF->File = CurrentFile - Cache.PkgFileP;
 915
 916    // Link it to the end of the list
 917    map_pointer_t *Last = &Desc->FileList;
 918    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 919       Last = &D->NextFile;
 920
 921    DF->NextFile = *Last;
 922    *Last = DF.Index();
 923
 924    DF->Offset = List.Offset();
 925    DF->Size = List.Size();
 926    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 927       Cache.HeaderP->MaxDescFileSize = DF->Size;
 928    Cache.HeaderP->DescFileCount++;
 929
 930    return true;
 931 }
 932                                                                         /*}}}*/
 933 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 934 // ---------------------------------------------------------------------
 935 /* This puts a description structure in the linked list */
 936 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 937                                             const string &Lang,
 938                                             APT::StringView md5sum,
 939                                             map_stringitem_t const idxmd5str)
 940 {
 941    // Get a structure
 942    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 943    if (Description == 0)
 944       return 0;
 945
 946    // Fill it in
 947    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 948    Desc->ID = Cache.HeaderP->DescriptionCount++;
 949    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 950    if (unlikely(idxlanguage_code == 0))
 951       return 0;
 952    Desc->language_code = idxlanguage_code;
 953
 954    if (idxmd5str != 0)
 955       Desc->md5sum = idxmd5str;
 956    else
 957    {
 958       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
 959       if (unlikely(idxmd5sum == 0))
 960          return 0;
 961       Desc->md5sum = idxmd5sum;
 962    }
 963
 964    return Description;
 965 }
 966                                                                         /*}}}*/
 967 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 968 // ---------------------------------------------------------------------
 969 /* This creates a dependency element in the tree. It is linked to the
 970    version and to the package that it is pointing to. */
 971 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 972                                    pkgCache::VerIterator &Ver,
 973                                    map_pointer_t const Version,
 974                                    uint8_t const Op,
 975                                    uint8_t const Type,
 976                                    map_pointer_t* &OldDepLast)
 977 {
 978    void const * const oldMap = Map.Data();
 979    // Get a structure
 980    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 981    if (unlikely(Dependency == 0))
 982       return false;
 983
 984    bool isDuplicate = false;
 985    map_pointer_t DependencyData = 0;
 986    map_pointer_t PreviousData = 0;
 987    if (Pkg->RevDepends != 0)
 988    {
 989       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 990       DependencyData = L->DependencyData;
 991       do {
 992          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 993          if (Version > D->Version)
 994             break;
 995          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 996          {
 997             isDuplicate = true;
 998             break;
 999          }
1000          PreviousData = DependencyData;
1001          DependencyData = D->NextData;
1002       } while (DependencyData != 0);
1003    }
1004
1005    if (isDuplicate == false)
1006    {
1007       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1008       if (unlikely(DependencyData == 0))
1009         return false;
1010    }
1011
1012    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1013    Link->ParentVer = Ver.Index();
1014    Link->DependencyData = DependencyData;
1015    Link->ID = Cache.HeaderP->DependsCount++;
1016
1017    pkgCache::DepIterator Dep(Cache, Link);
1018    if (isDuplicate == false)
1019    {
1020       Dep->Type = Type;
1021       Dep->CompareOp = Op;
1022       Dep->Version = Version;
1023       Dep->Package = Pkg.Index();
1024       ++Cache.HeaderP->DependsDataCount;
1025       if (PreviousData != 0)
1026       {
1027          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1028          Dep->NextData = D->NextData;
1029          D->NextData = DependencyData;
1030       }
1031       else if (Pkg->RevDepends != 0)
1032       {
1033          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1034          Dep->NextData = D->DependencyData;
1035       }
1036    }
1037
1038    if (isDuplicate == true || PreviousData != 0)
1039    {
1040       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1041       Link->NextRevDepends = L->NextRevDepends;
1042       L->NextRevDepends = Dependency;
1043    }
1044    else
1045    {
1046       Link->NextRevDepends = Pkg->RevDepends;
1047       Pkg->RevDepends = Dependency;
1048    }
1049
1050
1051    // Do we know where to link the Dependency to?
1052    if (OldDepLast == NULL)
1053    {
1054       OldDepLast = &Ver->DependsList;
1055       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1056          OldDepLast = &D->NextDepends;
1057    } else if (oldMap != Map.Data())
1058       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1059
1060    Dep->NextDepends = *OldDepLast;
1061    *OldDepLast = Dependency;
1062    OldDepLast = &Dep->NextDepends;
1063    return true;
1064 }
1065                                                                         /*}}}*/
1066 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1067 // ---------------------------------------------------------------------
1068 /* This creates a Group and the Package to link this dependency to if
1069    needed and handles also the caching of the old endpoint */
1070 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1071                                                StringView PackageName,
1072                                                StringView Arch,
1073                                                StringView Version,
1074                                                uint8_t const Op,
1075                                                uint8_t const Type)
1076 {
1077    pkgCache::GrpIterator Grp;
1078    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1079    Dynamic<StringView> DynPackageName(PackageName);
1080    Dynamic<StringView> DynArch(Arch);
1081    Dynamic<StringView> DynVersion(Version);
1082    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1083       return false;
1084
1085    map_stringitem_t idxVersion = 0;
1086    if (Version.empty() == false)
1087    {
1088       int const CmpOp = Op & 0x0F;
1089       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1090       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1091          idxVersion = Ver->VerStr;
1092
1093       if (idxVersion == 0)
1094       {
1095          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1096          if (unlikely(idxVersion == 0))
1097             return false;
1098       }
1099    }
1100
1101    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1102          Type == pkgCache::Dep::Conflicts ||
1103          Type == pkgCache::Dep::Replaces);
1104
1105    pkgCache::PkgIterator Pkg;
1106    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1107    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1108    {
1109       // Locate the target package
1110       Pkg = Grp.FindPkg(Arch);
1111       if (Pkg.end() == true) {
1112          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1113             return false;
1114       }
1115
1116       /* Caching the old end point speeds up generation substantially */
1117       if (OldDepVer != Ver) {
1118          OldDepLast = NULL;
1119          OldDepVer = Ver;
1120       }
1121
1122       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1123    }
1124    else
1125    {
1126       /* Caching the old end point speeds up generation substantially */
1127       if (OldDepVer != Ver) {
1128          OldDepLast = NULL;
1129          OldDepVer = Ver;
1130       }
1131
1132       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1133       {
1134          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1135                return false;
1136       }
1137    }
1138    return true;
1139 }
1140                                                                         /*}}}*/
1141 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1142 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1143                                                 StringView PkgName,
1144                                                 StringView PkgArch,
1145                                                 StringView Version,
1146                                                 uint8_t const Flags)
1147 {
1148    pkgCache const &Cache = Owner->Cache;
1149    Dynamic<StringView> DynPkgName(PkgName);
1150    Dynamic<StringView> DynArch(PkgArch);
1151    Dynamic<StringView> DynVersion(Version);
1152
1153    // We do not add self referencing provides
1154    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1155         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1156          (Version.empty() || Version == Ver.VerStr()))
1157       return true;
1158
1159    // Locate the target package
1160    pkgCache::PkgIterator Pkg;
1161    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1162    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1163       return false;
1164
1165    map_stringitem_t idxProvideVersion = 0;
1166    if (Version.empty() == false) {
1167       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1168       if (unlikely(idxProvideVersion == 0))
1169          return false;
1170    }
1171    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1172 }
1173 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1174                                     pkgCache::PkgIterator &Pkg,
1175                                     map_pointer_t const ProvideVersion,
1176                                     uint8_t const Flags)
1177 {
1178    // Get a structure
1179    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1180    if (unlikely(Provides == 0))
1181       return false;
1182    ++Cache.HeaderP->ProvidesCount;
1183
1184    // Fill it in
1185    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1186    Prv->Version = Ver.Index();
1187    Prv->ProvideVersion = ProvideVersion;
1188    Prv->Flags = Flags;
1189    Prv->NextPkgProv = Ver->ProvidesList;
1190    Ver->ProvidesList = Prv.Index();
1191
1192    // Link it to the package
1193    Prv->ParentPkg = Pkg.Index();
1194    Prv->NextProvides = Pkg->ProvidesList;
1195    Pkg->ProvidesList = Prv.Index();
1196    return true;
1197 }
1198                                                                         /*}}}*/
1199 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1200 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1201                                 StringView Version, uint8_t const Flags) {
1202    pkgCache &Cache = Owner->Cache;
1203    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1204    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1205    Dynamic<StringView> DynPackage(Package);
1206    Dynamic<StringView> DynVersion(Version);
1207
1208    if (Grp.end() == true)
1209       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1210    else
1211    {
1212       map_stringitem_t idxProvideVersion = 0;
1213       if (Version.empty() == false) {
1214          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1215          if (unlikely(idxProvideVersion == 0))
1216             return false;
1217       }
1218
1219       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1220       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1221       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1222       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1223       pkgCache::PkgIterator Pkg;
1224       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1225       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1226       {
1227          if (isImplicit && OwnerPkg == Pkg)
1228             continue;
1229          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1230             continue;
1231          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1232             return false;
1233       }
1234    }
1235    return true;
1236 }
1237                                                                         /*}}}*/
1238 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1239       pkgCache::VerIterator const &Ver)
1240 {
1241    return Hash == Ver->Hash;
1242 }
1243                                                                         /*}}}*/
1244 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1245 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1246                                    unsigned long Flags)
1247 {
1248    if (File.empty() && Site.empty())
1249    {
1250       CurrentRlsFile = NULL;
1251       return true;
1252    }
1253
1254    // Get some space for the structure
1255    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1256    if (unlikely(idxFile == 0))
1257       return false;
1258    CurrentRlsFile = Cache.RlsFileP + idxFile;
1259
1260    // Fill it in
1261    map_stringitem_t const idxFileName = WriteStringInMap(File);
1262    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1263    if (unlikely(idxFileName == 0 || idxSite == 0))
1264       return false;
1265    CurrentRlsFile->FileName = idxFileName;
1266    CurrentRlsFile->Site = idxSite;
1267    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1268    CurrentRlsFile->Flags = Flags;
1269    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1270    RlsFileName = File;
1271    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1272    Cache.HeaderP->ReleaseFileCount++;
1273
1274    return true;
1275 }
1276                                                                         /*}}}*/
1277 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1278 // ---------------------------------------------------------------------
1279 /* This is used to select which file is to be associated with all newly
1280    added versions. The caller is responsible for setting the IMS fields. */
1281 bool pkgCacheGenerator::SelectFile(std::string const &File,
1282                                    pkgIndexFile const &Index,
1283                                    std::string const &Architecture,
1284                                    std::string const &Component,
1285                                    unsigned long const Flags)
1286 {
1287    // Get some space for the structure
1288    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1289    if (unlikely(idxFile == 0))
1290       return false;
1291    CurrentFile = Cache.PkgFileP + idxFile;
1292
1293    // Fill it in
1294    map_stringitem_t const idxFileName = WriteStringInMap(File);
1295    if (unlikely(idxFileName == 0))
1296       return false;
1297    CurrentFile->FileName = idxFileName;
1298    CurrentFile->NextFile = Cache.HeaderP->FileList;
1299    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1300    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1301    if (unlikely(idxIndexType == 0))
1302       return false;
1303    CurrentFile->IndexType = idxIndexType;
1304    if (Architecture.empty())
1305       CurrentFile->Architecture = 0;
1306    else
1307    {
1308       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1309       if (unlikely(arch == 0))
1310          return false;
1311       CurrentFile->Architecture = arch;
1312    }
1313    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1314    if (unlikely(component == 0))
1315       return false;
1316    CurrentFile->Component = component;
1317    CurrentFile->Flags = Flags;
1318    if (CurrentRlsFile != NULL)
1319       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1320    else
1321       CurrentFile->Release = 0;
1322    PkgFileName = File;
1323    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1324    Cache.HeaderP->PackageFileCount++;
1325
1326    if (Progress != 0)
1327       Progress->SubProgress(Index.Size());
1328    return true;
1329 }
1330                                                                         /*}}}*/
1331 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1332 // ---------------------------------------------------------------------
1333 /* This is used to create handles to strings. Given the same text it
1334    always returns the same number */
1335 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1336                                                  unsigned int Size)
1337 {
1338    auto strings = &strMixed;
1339    switch(type) {
1340       case MIXED: strings = &strMixed; break;
1341       case PKGNAME: strings = &strPkgNames; break;
1342       case VERSIONNUMBER: strings = &strVersions; break;
1343       case SECTION: strings = &strSections; break;
1344       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1345    }
1346
1347    auto const item = strings->find({S, Size, nullptr, 0});
1348    if (item != strings->end())
1349       return item->item;
1350
1351    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1352    strings->insert({nullptr, Size, this, idxString});
1353    return idxString;
1354 }
1355                                                                         /*}}}*/
1356 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1357 // ---------------------------------------------------------------------
1358 /* This just verifies that each file in the list of index files exists,
1359    has matching attributes with the cache and the cache does not have
1360    any extra files. */
1361 class APT_HIDDEN ScopedErrorRevert {
1362 public:
1363    ScopedErrorRevert() { _error->PushToStack(); }
1364    ~ScopedErrorRevert() { _error->RevertToStack(); }
1365 };
1366 static bool CheckValidity(const string &CacheFile,
1367                           pkgSourceList &List,
1368                           FileIterator const Start,
1369                           FileIterator const End,
1370                           MMap **OutMap = 0,
1371                           pkgCache **OutCache = 0)
1372 {
1373    ScopedErrorRevert ser;
1374    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1375    // No file, certainly invalid
1376    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1377    {
1378       if (Debug == true)
1379          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1380       return false;
1381    }
1382
1383    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1384    {
1385       if (Debug == true)
1386          std::clog << "sources.list is newer than the cache" << std::endl;
1387       return false;
1388    }
1389
1390    // Map it
1391    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1392    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1393    if (unlikely(Map->validData()) == false)
1394       return false;
1395    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1396    pkgCache &Cache = *CacheP.get();
1397    if (_error->PendingError() || Map->Size() == 0)
1398    {
1399       if (Debug == true)
1400          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1401       return false;
1402    }
1403
1404    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1405    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1406    std::vector<pkgIndexFile *> Files;
1407    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1408    {
1409       if (Debug == true)
1410          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1411       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1412       if (RlsFile.end() == true)
1413       {
1414          if (Debug == true)
1415             std::clog << "FindInCache returned end-Pointer" << std::endl;
1416          return false;
1417       }
1418
1419       RlsVisited[RlsFile->ID] = true;
1420       if (Debug == true)
1421          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1422
1423       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1424       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1425             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1426    }
1427    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1428       if (RlsVisited[I] == false)
1429       {
1430          if (Debug == true)
1431             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1432          return false;
1433       }
1434
1435    std::copy(Start, End, std::back_inserter(Files));
1436
1437    /* Now we check every index file, see if it is in the cache,
1438       verify the IMS data and check that it is on the disk too.. */
1439    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1440    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1441    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1442    {
1443       if (Debug == true)
1444          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1445       if ((*PkgFile)->Exists() == false)
1446       {
1447          if (Debug == true)
1448             std::clog << "file doesn't exist" << std::endl;
1449          continue;
1450       }
1451
1452       // FindInCache is also expected to do an IMS check.
1453       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1454       if (File.end() == true)
1455       {
1456          if (Debug == true)
1457             std::clog << "FindInCache returned end-Pointer" << std::endl;
1458          return false;
1459       }
1460
1461       Visited[File->ID] = true;
1462       if (Debug == true)
1463          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1464    }
1465
1466    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1467       if (Visited[I] == false)
1468       {
1469          if (Debug == true)
1470             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1471          return false;
1472       }
1473
1474    if (_error->PendingError() == true)
1475    {
1476       if (Debug == true)
1477       {
1478          std::clog << "Validity failed because of pending errors:" << std::endl;
1479          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1480       }
1481       return false;
1482    }
1483
1484    if (OutMap != 0)
1485       *OutMap = Map.release();
1486    if (OutCache != 0)
1487       *OutCache = CacheP.release();
1488    return true;
1489 }
1490                                                                         /*}}}*/
1491 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1492 // ---------------------------------------------------------------------
1493 /* Size is kind of an abstract notion that is only used for the progress
1494    meter */
1495 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1496 {
1497    map_filesize_t TotalSize = 0;
1498    if (List !=  NULL)
1499    {
1500       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1501       {
1502          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1503          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1504             if ((*j)->HasPackages() == true)
1505                TotalSize += (*j)->Size();
1506       }
1507    }
1508
1509    for (; Start < End; ++Start)
1510    {
1511       if ((*Start)->HasPackages() == false)
1512          continue;
1513       TotalSize += (*Start)->Size();
1514    }
1515    return TotalSize;
1516 }
1517                                                                         /*}}}*/
1518 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1519 static bool BuildCache(pkgCacheGenerator &Gen,
1520                        OpProgress * const Progress,
1521                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1522                        pkgSourceList const * const List,
1523                        FileIterator const Start, FileIterator const End)
1524 {
1525    bool mergeFailure = false;
1526
1527    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1528       if (I->HasPackages() == false || mergeFailure)
1529          return;
1530
1531       if (I->Exists() == false)
1532          return;
1533
1534       if (I->FindInCache(Gen.GetCache()).end() == false)
1535       {
1536          _error->Warning("Duplicate sources.list entry %s",
1537                I->Describe().c_str());
1538          return;
1539       }
1540
1541       map_filesize_t const Size = I->Size();
1542       if (Progress != NULL)
1543          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1544       CurrentSize += Size;
1545
1546       if (I->Merge(Gen,Progress) == false)
1547          mergeFailure = true;
1548    };
1549
1550    if (List !=  NULL)
1551    {
1552       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1553       {
1554          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1555          {
1556             _error->Warning("Duplicate sources.list entry %s",
1557                   (*i)->Describe().c_str());
1558             continue;
1559          }
1560
1561          if ((*i)->Merge(Gen, Progress) == false)
1562             return false;
1563
1564          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1565          if (Indexes != NULL)
1566             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1567          if (mergeFailure)
1568             return false;
1569       }
1570    }
1571
1572    if (Start != End)
1573    {
1574       Gen.SelectReleaseFile("", "");
1575       std::for_each(Start, End, indexFileMerge);
1576       if (mergeFailure)
1577          return false;
1578    }
1579    return true;
1580 }
1581                                                                         /*}}}*/
1582 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1583 // ---------------------------------------------------------------------
1584 /* This makes sure that the status cache (the cache that has all
1585    index files from the sources list and all local ones) is ready
1586    to be mmaped. If OutMap is not zero then a MMap object representing
1587    the cache will be stored there. This is pretty much mandetory if you
1588    are using AllowMem. AllowMem lets the function be run as non-root
1589    where it builds the cache 'fast' into a memory buffer. */
1590 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1591 {
1592    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1593    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1594    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1595    Flags |= MMap::Moveable;
1596    if (_config->FindB("APT::Cache-Fallback", false) == true)
1597       Flags |= MMap::Fallback;
1598    if (CacheF != NULL)
1599       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1600    else
1601       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1602 }
1603 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1604       std::string const &FileName)
1605 {
1606    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1607    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1608       return false;
1609
1610    fchmod(SCacheF.Fd(),0644);
1611
1612    // Write out the main data
1613    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1614       return _error->Error(_("IO Error saving source cache"));
1615
1616    // Write out the proper header
1617    Gen->GetCache().HeaderP->Dirty = false;
1618    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1619    if (SCacheF.Seek(0) == false ||
1620          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1621       return _error->Error(_("IO Error saving source cache"));
1622    Gen->GetCache().HeaderP->Dirty = true;
1623    return true;
1624 }
1625 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1626       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1627 {
1628    Map.reset(CreateDynamicMMap(NULL, 0));
1629    if (unlikely(Map->validData()) == false)
1630       return false;
1631    FileFd CacheF(FileName, FileFd::ReadOnly);
1632    if (CacheF.IsOpen() == false || CacheF.Failed())
1633       return false;
1634    _error->PushToStack();
1635    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1636    bool const newError = _error->PendingError();
1637    _error->MergeWithStack();
1638    if (alloc == 0 && newError)
1639       return false;
1640    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1641       return false;
1642    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1643    return Gen->Start();
1644 }
1645 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1646                         MMap **OutMap, bool AllowMem)
1647    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1648 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1649                         MMap **OutMap,bool)
1650 {
1651    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1652 }
1653 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1654                         MMap **OutMap,pkgCache **OutCache, bool)
1655 {
1656    // FIXME: deprecate the ignored AllowMem parameter
1657    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1658
1659    std::vector<pkgIndexFile *> Files;
1660    if (_system->AddStatusFiles(Files) == false)
1661       return false;
1662
1663    // Decide if we can write to the files..
1664    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1665    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1666
1667    // ensure the cache directory exists
1668    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1669    {
1670       string dir = _config->FindDir("Dir::Cache");
1671       size_t const len = dir.size();
1672       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1673          dir = dir.substr(0, len - 5);
1674       if (CacheFile.empty() == false)
1675          CreateDirectory(dir, flNotFile(CacheFile));
1676       if (SrcCacheFile.empty() == false)
1677          CreateDirectory(dir, flNotFile(SrcCacheFile));
1678    }
1679
1680    if (Progress != NULL)
1681       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1682
1683    bool pkgcache_fine = false;
1684    bool srcpkgcache_fine = false;
1685    bool volatile_fine = List.GetVolatileFiles().empty();
1686
1687    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1688                      volatile_fine ? OutCache : NULL) == true)
1689    {
1690       if (Debug == true)
1691          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1692       pkgcache_fine = true;
1693       srcpkgcache_fine = true;
1694    }
1695    if (pkgcache_fine == false)
1696    {
1697       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1698       {
1699          if (Debug == true)
1700             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1701          srcpkgcache_fine = true;
1702       }
1703    }
1704
1705    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1706    {
1707       if (Progress != NULL)
1708          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1709       return true;
1710    }
1711
1712    bool Writeable = false;
1713    if (srcpkgcache_fine == false || pkgcache_fine == false)
1714    {
1715       if (CacheFile.empty() == false)
1716          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1717       else if (SrcCacheFile.empty() == false)
1718          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1719
1720       if (Debug == true)
1721          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1722    }
1723
1724    // At this point we know we need to construct something, so get storage ready
1725    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1726    if (unlikely(Map->validData()) == false)
1727       return false;
1728    if (Debug == true)
1729       std::clog << "Open memory Map (not filebased)" << std::endl;
1730
1731    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1732    map_filesize_t CurrentSize = 0;
1733    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1734    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1735    if (srcpkgcache_fine == true && pkgcache_fine == false)
1736    {
1737       if (Debug == true)
1738          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1739       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1740          return false;
1741       srcpkgcache_fine = true;
1742       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1743    }
1744    else if (srcpkgcache_fine == false)
1745    {
1746       if (Debug == true)
1747          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1748       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1749       if (Gen->Start() == false)
1750          return false;
1751
1752       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1753       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1754                Files.end(),Files.end()) == false)
1755          return false;
1756
1757       if (Writeable == true && SrcCacheFile.empty() == false)
1758          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1759             return false;
1760    }
1761
1762    if (pkgcache_fine == false)
1763    {
1764       if (Debug == true)
1765          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1766       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1767                Files.begin(), Files.end()) == false)
1768          return false;
1769
1770       if (Writeable == true && CacheFile.empty() == false)
1771          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1772             return false;
1773    }
1774
1775    if (Debug == true)
1776       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1777
1778    if (volatile_fine == false)
1779    {
1780       if (Gen == nullptr)
1781       {
1782          if (Debug == true)
1783             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1784          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1785             return false;
1786       }
1787
1788       Files = List.GetVolatileFiles();
1789       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1790                Files.begin(), Files.end()) == false)
1791          return false;
1792    }
1793
1794    if (OutMap != nullptr)
1795       *OutMap = Map.release();
1796
1797    if (Debug == true)
1798       std::clog << "Everything is ready for shipping" << std::endl;
1799    return true;
1800 }
1801                                                                         /*}}}*/
1802 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1803 class APT_HIDDEN ScopedErrorMerge {
1804 public:
1805    ScopedErrorMerge() { _error->PushToStack(); }
1806    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1807 };
1808 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1809    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1810 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1811 {
1812    std::vector<pkgIndexFile *> Files;
1813    if (_system->AddStatusFiles(Files) == false)
1814       return false;
1815
1816    ScopedErrorMerge sem;
1817    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1818    if (unlikely(Map->validData()) == false)
1819       return false;
1820    map_filesize_t CurrentSize = 0;
1821    map_filesize_t TotalSize = 0;
1822    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1823
1824    // Build the status cache
1825    if (Progress != NULL)
1826       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1827    pkgCacheGenerator Gen(Map.get(),Progress);
1828    if (Gen.Start() == false || _error->PendingError() == true)
1829       return false;
1830    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1831                   Files.begin(), Files.end()) == false)
1832       return false;
1833
1834    if (_error->PendingError() == true)
1835       return false;
1836    *OutMap = Map.release();
1837
1838    return true;
1839 }
1840                                                                         /*}}}*/
1841 // IsDuplicateDescription                                               /*{{{*/
1842 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
1843                             APT::StringView CurMd5, std::string const &CurLang)
1844 {
1845    // Descriptions in the same link-list have all the same md5
1846    if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1847       return false;
1848    for (; Desc.end() == false; ++Desc)
1849       if (Desc.LanguageCode() == CurLang)
1850          return true;
1851    return false;
1852 }
1853                                                                         /*}}}*/
1854
1855 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1856 pkgCacheListParser::~pkgCacheListParser() {}