apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
  49                             APT::StringView CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return false;
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (_error->PendingError() == true || Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 180         if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 184       if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 199       if (std::get<1>(seen.insert(ViewP)) == false)
 200          continue;
 201       // Ignore views outside of the cache.
 202       if (ViewP->data() < static_cast<const char*>(oldMap)
 203          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 204          continue;
 205       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 206       *ViewP = StringView(data , ViewP->size());
 207    }
 208 }                                                                       /*}}}*/
 209 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 210 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 211                                         const unsigned long &Len) {
 212    size_t oldSize = Map.Size();
 213    void const * const oldMap = Map.Data();
 214    map_stringitem_t const index = Map.WriteString(String, Len);
 215    if (index != 0)
 216       ReMap(oldMap, Map.Data(), oldSize);
 217    return index;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 221 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 222    size_t oldSize = Map.Size();
 223    void const * const oldMap = Map.Data();
 224    map_stringitem_t const index = Map.WriteString(String);
 225    if (index != 0)
 226       ReMap(oldMap, Map.Data(), oldSize);
 227    return index;
 228 }
 229                                                                         /*}}}*/
 230 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 231    size_t oldSize = Map.Size();
 232    void const * const oldMap = Map.Data();
 233    map_pointer_t const index = Map.Allocate(size);
 234    if (index != 0)
 235       ReMap(oldMap, Map.Data(), oldSize);
 236    return index;
 237 }
 238                                                                         /*}}}*/
 239 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This provides the generation of the entries in the cache. Each loop
 242    goes through a single package record from the underlying parse engine. */
 243 bool pkgCacheGenerator::MergeList(ListParser &List,
 244                                   pkgCache::VerIterator *OutVer)
 245 {
 246    List.Owner = this;
 247
 248    unsigned int Counter = 0;
 249    while (List.Step() == true)
 250    {
 251       string const PackageName = List.Package();
 252       if (PackageName.empty() == true) {
 253           _error->Warning("Encountered a section with no Package: header");
 254           continue;
 255       }
 256
 257       Counter++;
 258       if (Counter % 100 == 0 && Progress != 0)
 259          Progress->Progress(List.Offset());
 260
 261       APT::StringView Arch = List.Architecture();
 262       Dynamic<APT::StringView> DynArch(Arch);
 263       APT::StringView Version = List.Version();
 264       Dynamic<APT::StringView> DynVersion(Version);
 265       if (Version.empty() == true && Arch.empty() == true)
 266       {
 267          // package descriptions
 268          if (MergeListGroup(List, PackageName) == false)
 269             continue;
 270          continue;
 271       }
 272
 273       // Get a pointer to the package structure
 274       pkgCache::PkgIterator Pkg;
 275       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 276       if (NewPackage(Pkg, PackageName, Arch) == false) {
 277          // TRANSLATOR: The first placeholder is a package name,
 278          // the other two should be copied verbatim as they include debug info
 279          _error->Warning(_("Error occurred while processing %s (%s%d)"),
 280                               PackageName.c_str(), "NewPackage", 1);
 281          continue;
 282       }
 283
 284
 285       if (Version.empty() == true)
 286       {
 287          if (MergeListPackage(List, Pkg) == false)
 288             continue;
 289       }
 290       else
 291       {
 292          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 293             return false;
 294       }
 295
 296       if (OutVer != 0)
 297          return true;
 298    }
 299
 300    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 301       return _error->Error(_("Wow, you exceeded the number of package "
 302                              "names this APT is capable of."));
 303    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 304       return _error->Error(_("Wow, you exceeded the number of versions "
 305                              "this APT is capable of."));
 306    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 307       return _error->Error(_("Wow, you exceeded the number of descriptions "
 308                              "this APT is capable of."));
 309    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 310       return _error->Error(_("Wow, you exceeded the number of dependencies "
 311                              "this APT is capable of."));
 312
 313    return true;
 314 }
 315 // CacheGenerator::MergeListGroup                                       /*{{{*/
 316 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 317 {
 318    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 319    // a group has no data on it's own, only packages have it but these
 320    // stanzas like this come from Translation- files to add descriptions,
 321    // but without a version we don't need a description for it…
 322    if (Grp.end() == true)
 323       return true;
 324    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 325
 326    pkgCache::PkgIterator Pkg;
 327    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 328    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 329       if (MergeListPackage(List, Pkg) == false)
 330          return false;
 331
 332    return true;
 333 }
 334                                                                         /*}}}*/
 335 // CacheGenerator::MergeListPackage                                     /*{{{*/
 336 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 337 {
 338    // we first process the package, then the descriptions
 339    // (for deb this package processing is in fact a no-op)
 340    pkgCache::VerIterator Ver(Cache);
 341    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 342    if (List.UsePackage(Pkg, Ver) == false)
 343       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 344                            Pkg.Name(), "UsePackage", 1);
 345
 346    // Find the right version to write the description
 347    StringView CurMd5 = List.Description_md5();
 348    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 349    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 350    {
 351       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 352
 353       // a version can only have one md5 describing it
 354       if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
 355          continue;
 356
 357       map_stringitem_t md5idx = VerDesc->md5sum;
 358       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 359       {
 360          // don't add a new description if we have one for the given
 361          // md5 && language
 362          if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
 363             continue;
 364
 365          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 366       }
 367
 368       // we can stop here as all "same" versions will share the description
 369       break;
 370    }
 371
 372    return true;
 373 }
 374                                                                         /*}}}*/
 375 // CacheGenerator::MergeListVersion                                     /*{{{*/
 376 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 377                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 378 {
 379    pkgCache::VerIterator Ver = Pkg.VersionList();
 380    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 381    map_pointer_t *LastVer = &Pkg->VersionList;
 382    void const * oldMap = Map.Data();
 383
 384    unsigned short const Hash = List.VersionHash();
 385    if (Ver.end() == false)
 386    {
 387       /* We know the list is sorted so we use that fact in the search.
 388          Insertion of new versions is done with correct sorting */
 389       int Res = 1;
 390       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 391       {
 392          char const * const VerStr = Ver.VerStr();
 393          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 394                VerStr, VerStr + strlen(VerStr));
 395          // Version is higher as current version - insert here
 396          if (Res > 0)
 397             break;
 398          // Versionstrings are equal - is hash also equal?
 399          if (Res == 0)
 400          {
 401             if (List.SameVersion(Hash, Ver) == true)
 402                break;
 403             // sort (volatile) sources above not-sources like the status file
 404             if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 405             {
 406                auto VF = Ver.FileList();
 407                for (; VF.end() == false; ++VF)
 408                   if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
 409                      break;
 410                if (VF.end() == true)
 411                   break;
 412             }
 413          }
 414          // proceed with the next till we have either the right
 415          // or we found another version (which will be lower)
 416       }
 417
 418       /* We already have a version for this item, record that we saw it */
 419       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 420       {
 421          if (List.UsePackage(Pkg,Ver) == false)
 422             return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 423                                  Pkg.Name(), "UsePackage", 2);
 424
 425          if (NewFileVer(Ver,List) == false)
 426             return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 427                                  Pkg.Name(), "NewFileVer", 1);
 428
 429          // Read only a single record and return
 430          if (OutVer != 0)
 431          {
 432             *OutVer = Ver;
 433             return true;
 434          }
 435
 436          return true;
 437       }
 438    }
 439
 440    // Add a new version
 441    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 442    if (unlikely(verindex == 0))
 443       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 444                            Pkg.Name(), "NewVersion", 1);
 445
 446    if (oldMap != Map.Data())
 447          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 448    *LastVer = verindex;
 449
 450    if (unlikely(List.NewVersion(Ver) == false))
 451       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 452                            Pkg.Name(), "NewVersion", 2);
 453
 454    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 455       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 456                            Pkg.Name(), "UsePackage", 3);
 457
 458    if (unlikely(NewFileVer(Ver,List) == false))
 459       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 460                            Pkg.Name(), "NewFileVer", 2);
 461
 462    pkgCache::GrpIterator Grp = Pkg.Group();
 463    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 464
 465    /* If it is the first version of this package we need to add implicit
 466       Multi-Arch dependencies to all other package versions in the group now -
 467       otherwise we just add them for this new version */
 468    if (Pkg.VersionList()->NextVer == 0)
 469    {
 470       pkgCache::PkgIterator P = Grp.PackageList();
 471       Dynamic<pkgCache::PkgIterator> DynP(P);
 472       for (; P.end() != true; P = Grp.NextPkg(P))
 473       {
 474          if (P->ID == Pkg->ID)
 475             continue;
 476          pkgCache::VerIterator V = P.VersionList();
 477          Dynamic<pkgCache::VerIterator> DynV(V);
 478          for (; V.end() != true; ++V)
 479             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 480                return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 481                                     Pkg.Name(), "AddImplicitDepends", 1);
 482       }
 483    }
 484    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 485       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 486                            Pkg.Name(), "AddImplicitDepends", 2);
 487
 488    // Read only a single record and return
 489    if (OutVer != 0)
 490    {
 491       *OutVer = Ver;
 492       return true;
 493    }
 494
 495    /* Record the Description(s) based on their master md5sum */
 496    StringView CurMd5 = List.Description_md5();
 497
 498    /* Before we add a new description we first search in the group for
 499       a version with a description of the same MD5 - if so we reuse this
 500       description group instead of creating our own for this version */
 501    for (pkgCache::PkgIterator P = Grp.PackageList();
 502         P.end() == false; P = Grp.NextPkg(P))
 503    {
 504       for (pkgCache::VerIterator V = P.VersionList();
 505            V.end() == false; ++V)
 506       {
 507          if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
 508             continue;
 509          Ver->DescriptionList = V->DescriptionList;
 510       }
 511    }
 512
 513    // We haven't found reusable descriptions, so add the first description(s)
 514    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 515    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 516    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 517       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 518          return false;
 519    return true;
 520 }
 521                                                                         /*}}}*/
 522 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 523 {
 524    pkgCache::DescIterator Desc;
 525    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 526
 527    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 528    if (unlikely(descindex == 0))
 529       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 530             Ver.ParentPkg().Name(), "NewDescription", 1);
 531
 532    md5idx = Desc->md5sum;
 533    Desc->ParentPkg = Ver.ParentPkg().Index();
 534
 535    // we add at the end, so that the start is constant as we need
 536    // that to be able to efficiently share these lists
 537    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 538    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 539    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 540    *LastNextDesc = descindex;
 541
 542    if (NewFileDesc(Desc,List) == false)
 543       return _error->Warning(_("Error occurred while processing %s (%s%d)"),
 544             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 545
 546    return true;
 547 }
 548                                                                         /*}}}*/
 549                                                                         /*}}}*/
 550 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* This creates a new group structure and adds it to the hash table */
 553 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 554 {
 555    Dynamic<StringView> DName(Name);
 556    Grp = Cache.FindGrp(Name);
 557    if (Grp.end() == false)
 558       return true;
 559
 560    // Get a structure
 561    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 562    if (unlikely(Group == 0))
 563       return false;
 564
 565    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 566    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 567    if (unlikely(idxName == 0))
 568       return false;
 569    Grp->Name = idxName;
 570
 571    // Insert it into the hash table
 572    unsigned long const Hash = Cache.Hash(Name);
 573    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 574
 575    while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 576       insertAt = &(Cache.GrpP + *insertAt)->Next;
 577    Grp->Next = *insertAt;
 578    *insertAt = Group;
 579
 580    Grp->ID = Cache.HeaderP->GroupCount++;
 581    return true;
 582 }
 583                                                                         /*}}}*/
 584 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 585 // ---------------------------------------------------------------------
 586 /* This creates a new package structure and adds it to the hash table */
 587 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 588                                         StringView Arch) {
 589    pkgCache::GrpIterator Grp;
 590    Dynamic<StringView> DName(Name);
 591    Dynamic<StringView> DArch(Arch);
 592    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 593    if (unlikely(NewGroup(Grp, Name) == false))
 594       return false;
 595
 596    Pkg = Grp.FindPkg(Arch);
 597       if (Pkg.end() == false)
 598          return true;
 599
 600    // Get a structure
 601    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 602    if (unlikely(Package == 0))
 603       return false;
 604    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 605
 606    // Set the name, arch and the ID
 607    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 608    Pkg->Group = Grp.Index();
 609    // all is mapped to the native architecture
 610    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 611    if (unlikely(idxArch == 0))
 612       return false;
 613    Pkg->Arch = idxArch;
 614    Pkg->ID = Cache.HeaderP->PackageCount++;
 615
 616    // Insert the package into our package list
 617    if (Grp->FirstPackage == 0) // the group is new
 618    {
 619       Grp->FirstPackage = Package;
 620       // Insert it into the hash table
 621       map_id_t const Hash = Cache.Hash(Name);
 622       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 623       while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
 624          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 625       Pkg->NextPackage = *insertAt;
 626       *insertAt = Package;
 627    }
 628    else // Group the Packages together
 629    {
 630       // if sibling is provided by another package, this one is too
 631       {
 632          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 633          if (M.end() == false) {
 634             pkgCache::PrvIterator Prv;
 635             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 636             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 637             {
 638                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 639                   continue;
 640                pkgCache::VerIterator Ver = Prv.OwnerVer();
 641                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 642                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 643                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 644                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 645                {
 646                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 647                      continue;
 648                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 649                      return false;
 650                }
 651             }
 652          }
 653       }
 654       // let M-A:foreign package siblings provide this package
 655       {
 656          pkgCache::PkgIterator P;
 657          pkgCache::VerIterator Ver;
 658          Dynamic<pkgCache::PkgIterator> DynP(P);
 659          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 660
 661          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 662          {
 663             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 664                continue;
 665             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 666                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 667                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 668                      return false;
 669          }
 670       }
 671       // and negative dependencies, don't forget negative dependencies
 672       {
 673          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 674          if (M.end() == false) {
 675             pkgCache::DepIterator Dep;
 676             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 677             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 678             {
 679                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 680                   continue;
 681                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 682                      Dep->Type != pkgCache::Dep::Replaces)
 683                   continue;
 684                pkgCache::VerIterator Ver = Dep.ParentVer();
 685                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 686                map_pointer_t * unused = NULL;
 687                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 688                   return false;
 689             }
 690          }
 691       }
 692
 693       // this package is the new last package
 694       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 695       Pkg->NextPackage = LastPkg->NextPackage;
 696       LastPkg->NextPackage = Package;
 697    }
 698    Grp->LastPackage = Package;
 699
 700    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 701    if (Arch == "any")
 702    {
 703       size_t const found = Name.rfind(':');
 704       StringView ArchA = Name.substr(found + 1);
 705       if (ArchA != "any")
 706       {
 707          // ArchA is used inside the loop which might remap (NameA is not used)
 708          Dynamic<StringView> DynArchA(ArchA);
 709          StringView NameA = Name.substr(0, found);
 710          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 711          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 712          if (PkgA.end())
 713          {
 714             Dynamic<StringView> DynNameA(NameA);
 715             if (NewPackage(PkgA, NameA, ArchA) == false)
 716                return false;
 717          }
 718          if (unlikely(PkgA.end()))
 719             return _error->Fatal("NewPackage was successful for %s:%s,"
 720                   "but the package doesn't exist anyhow!",
 721                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 722          else
 723          {
 724             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 725             for (; Prv.end() == false; ++Prv)
 726             {
 727                if (Prv.IsMultiArchImplicit())
 728                   continue;
 729                pkgCache::VerIterator V = Prv.OwnerVer();
 730                if (ArchA != V.ParentPkg().Arch())
 731                   continue;
 732                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 733                   return false;
 734             }
 735             pkgCache::VerIterator V = PkgA.VersionList();
 736             Dynamic<pkgCache::VerIterator> DynV(V);
 737             for (; V.end() == false; ++V)
 738             {
 739                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 740                   return false;
 741             }
 742          }
 743       }
 744    }
 745    return true;
 746 }
 747                                                                         /*}}}*/
 748 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 749 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 750                                            pkgCache::PkgIterator &P,
 751                                            pkgCache::VerIterator &V)
 752 {
 753    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 754    Dynamic<APT::StringView> DynArch(Arch);
 755    map_pointer_t *OldDepLast = NULL;
 756    /* MultiArch handling introduces a lot of implicit Dependencies:
 757       - MultiArch: same → Co-Installable if they have the same version
 758       - All others conflict with all other group members */
 759    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 760    pkgCache::PkgIterator D = G.PackageList();
 761    Dynamic<pkgCache::PkgIterator> DynD(D);
 762    map_stringitem_t const VerStrIdx = V->VerStr;
 763    for (; D.end() != true; D = G.NextPkg(D))
 764    {
 765       if (Arch == D.Arch() || D->VersionList == 0)
 766          continue;
 767       /* We allow only one installed arch at the time
 768          per group, therefore each group member conflicts
 769          with all other group members */
 770       if (coInstall == true)
 771       {
 772          // Replaces: ${self}:other ( << ${binary:Version})
 773          NewDepends(D, V, VerStrIdx,
 774                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 775                     OldDepLast);
 776          // Breaks: ${self}:other (!= ${binary:Version})
 777          NewDepends(D, V, VerStrIdx,
 778                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 779                     OldDepLast);
 780       } else {
 781          // Conflicts: ${self}:other
 782          NewDepends(D, V, 0,
 783                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 784                     OldDepLast);
 785       }
 786    }
 787    return true;
 788 }
 789 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 790                                            pkgCache::PkgIterator &D)
 791 {
 792    /* MultiArch handling introduces a lot of implicit Dependencies:
 793       - MultiArch: same → Co-Installable if they have the same version
 794       - All others conflict with all other group members */
 795    map_pointer_t *OldDepLast = NULL;
 796    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 797    if (coInstall == true)
 798    {
 799       map_stringitem_t const VerStrIdx = V->VerStr;
 800       // Replaces: ${self}:other ( << ${binary:Version})
 801       NewDepends(D, V, VerStrIdx,
 802                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 803                  OldDepLast);
 804       // Breaks: ${self}:other (!= ${binary:Version})
 805       NewDepends(D, V, VerStrIdx,
 806                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 807                  OldDepLast);
 808    } else {
 809       // Conflicts: ${self}:other
 810       NewDepends(D, V, 0,
 811                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 812                  OldDepLast);
 813    }
 814    return true;
 815 }
 816
 817                                                                         /*}}}*/
 818 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 819 // ---------------------------------------------------------------------
 820 /* */
 821 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 822                                    ListParser &List)
 823 {
 824    if (CurrentFile == 0)
 825       return true;
 826
 827    // Get a structure
 828    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 829    if (VerFile == 0)
 830       return false;
 831
 832    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 833    VF->File = CurrentFile - Cache.PkgFileP;
 834
 835    // Link it to the end of the list
 836    map_pointer_t *Last = &Ver->FileList;
 837    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 838       Last = &V->NextFile;
 839    VF->NextFile = *Last;
 840    *Last = VF.Index();
 841
 842    VF->Offset = List.Offset();
 843    VF->Size = List.Size();
 844    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 845       Cache.HeaderP->MaxVerFileSize = VF->Size;
 846    Cache.HeaderP->VerFileCount++;
 847
 848    return true;
 849 }
 850                                                                         /*}}}*/
 851 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 852 // ---------------------------------------------------------------------
 853 /* This puts a version structure in the linked list */
 854 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 855                                             APT::StringView const &VerStr,
 856                                             map_pointer_t const ParentPkg,
 857                                             unsigned short const Hash,
 858                                             map_pointer_t const Next)
 859 {
 860    // Get a structure
 861    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 862    if (Version == 0)
 863       return 0;
 864
 865    // Fill it in
 866    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 867    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 868    Ver->NextVer = Next;
 869    Ver->ParentPkg = ParentPkg;
 870    Ver->Hash = Hash;
 871    Ver->ID = Cache.HeaderP->VersionCount++;
 872
 873    // try to find the version string in the group for reuse
 874    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 875    pkgCache::GrpIterator Grp = Pkg.Group();
 876    if (Pkg.end() == false && Grp.end() == false)
 877    {
 878       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 879       {
 880          if (Pkg == P)
 881             continue;
 882          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 883          {
 884             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 885             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 886             {
 887                Ver->VerStr = V->VerStr;
 888                return Version;
 889             }
 890             else if (cmp < 0)
 891                break;
 892          }
 893       }
 894    }
 895    // haven't found the version string, so create
 896    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 897    if (unlikely(idxVerStr == 0))
 898       return 0;
 899    Ver->VerStr = idxVerStr;
 900    return Version;
 901 }
 902                                                                         /*}}}*/
 903 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 904 // ---------------------------------------------------------------------
 905 /* */
 906 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 907                                    ListParser &List)
 908 {
 909    if (CurrentFile == 0)
 910       return true;
 911
 912    // Get a structure
 913    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 914    if (DescFile == 0)
 915       return false;
 916
 917    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 918    DF->File = CurrentFile - Cache.PkgFileP;
 919
 920    // Link it to the end of the list
 921    map_pointer_t *Last = &Desc->FileList;
 922    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 923       Last = &D->NextFile;
 924
 925    DF->NextFile = *Last;
 926    *Last = DF.Index();
 927
 928    DF->Offset = List.Offset();
 929    DF->Size = List.Size();
 930    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 931       Cache.HeaderP->MaxDescFileSize = DF->Size;
 932    Cache.HeaderP->DescFileCount++;
 933
 934    return true;
 935 }
 936                                                                         /*}}}*/
 937 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 938 // ---------------------------------------------------------------------
 939 /* This puts a description structure in the linked list */
 940 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 941                                             const string &Lang,
 942                                             APT::StringView md5sum,
 943                                             map_stringitem_t const idxmd5str)
 944 {
 945    // Get a structure
 946    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 947    if (Description == 0)
 948       return 0;
 949
 950    // Fill it in
 951    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 952    Desc->ID = Cache.HeaderP->DescriptionCount++;
 953    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 954    if (unlikely(idxlanguage_code == 0))
 955       return 0;
 956    Desc->language_code = idxlanguage_code;
 957
 958    if (idxmd5str != 0)
 959       Desc->md5sum = idxmd5str;
 960    else
 961    {
 962       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
 963       if (unlikely(idxmd5sum == 0))
 964          return 0;
 965       Desc->md5sum = idxmd5sum;
 966    }
 967
 968    return Description;
 969 }
 970                                                                         /*}}}*/
 971 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 972 // ---------------------------------------------------------------------
 973 /* This creates a dependency element in the tree. It is linked to the
 974    version and to the package that it is pointing to. */
 975 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 976                                    pkgCache::VerIterator &Ver,
 977                                    map_pointer_t const Version,
 978                                    uint8_t const Op,
 979                                    uint8_t const Type,
 980                                    map_pointer_t* &OldDepLast)
 981 {
 982    void const * const oldMap = Map.Data();
 983    // Get a structure
 984    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 985    if (unlikely(Dependency == 0))
 986       return false;
 987
 988    bool isDuplicate = false;
 989    map_pointer_t DependencyData = 0;
 990    map_pointer_t PreviousData = 0;
 991    if (Pkg->RevDepends != 0)
 992    {
 993       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 994       DependencyData = L->DependencyData;
 995       do {
 996          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 997          if (Version > D->Version)
 998             break;
 999          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
1000          {
1001             isDuplicate = true;
1002             break;
1003          }
1004          PreviousData = DependencyData;
1005          DependencyData = D->NextData;
1006       } while (DependencyData != 0);
1007    }
1008
1009    if (isDuplicate == false)
1010    {
1011       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1012       if (unlikely(DependencyData == 0))
1013         return false;
1014    }
1015
1016    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1017    Link->ParentVer = Ver.Index();
1018    Link->DependencyData = DependencyData;
1019    Link->ID = Cache.HeaderP->DependsCount++;
1020
1021    pkgCache::DepIterator Dep(Cache, Link);
1022    if (isDuplicate == false)
1023    {
1024       Dep->Type = Type;
1025       Dep->CompareOp = Op;
1026       Dep->Version = Version;
1027       Dep->Package = Pkg.Index();
1028       ++Cache.HeaderP->DependsDataCount;
1029       if (PreviousData != 0)
1030       {
1031          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1032          Dep->NextData = D->NextData;
1033          D->NextData = DependencyData;
1034       }
1035       else if (Pkg->RevDepends != 0)
1036       {
1037          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1038          Dep->NextData = D->DependencyData;
1039       }
1040    }
1041
1042    if (isDuplicate == true || PreviousData != 0)
1043    {
1044       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1045       Link->NextRevDepends = L->NextRevDepends;
1046       L->NextRevDepends = Dependency;
1047    }
1048    else
1049    {
1050       Link->NextRevDepends = Pkg->RevDepends;
1051       Pkg->RevDepends = Dependency;
1052    }
1053
1054
1055    // Do we know where to link the Dependency to?
1056    if (OldDepLast == NULL)
1057    {
1058       OldDepLast = &Ver->DependsList;
1059       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1060          OldDepLast = &D->NextDepends;
1061    } else if (oldMap != Map.Data())
1062       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1063
1064    Dep->NextDepends = *OldDepLast;
1065    *OldDepLast = Dependency;
1066    OldDepLast = &Dep->NextDepends;
1067    return true;
1068 }
1069                                                                         /*}}}*/
1070 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1071 // ---------------------------------------------------------------------
1072 /* This creates a Group and the Package to link this dependency to if
1073    needed and handles also the caching of the old endpoint */
1074 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1075                                                StringView PackageName,
1076                                                StringView Arch,
1077                                                StringView Version,
1078                                                uint8_t const Op,
1079                                                uint8_t const Type)
1080 {
1081    pkgCache::GrpIterator Grp;
1082    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1083    Dynamic<StringView> DynPackageName(PackageName);
1084    Dynamic<StringView> DynArch(Arch);
1085    Dynamic<StringView> DynVersion(Version);
1086    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1087       return false;
1088
1089    map_stringitem_t idxVersion = 0;
1090    if (Version.empty() == false)
1091    {
1092       int const CmpOp = Op & 0x0F;
1093       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1094       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1095          idxVersion = Ver->VerStr;
1096
1097       if (idxVersion == 0)
1098       {
1099          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1100          if (unlikely(idxVersion == 0))
1101             return false;
1102       }
1103    }
1104
1105    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1106          Type == pkgCache::Dep::Conflicts ||
1107          Type == pkgCache::Dep::Replaces);
1108
1109    pkgCache::PkgIterator Pkg;
1110    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1111    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1112    {
1113       // Locate the target package
1114       Pkg = Grp.FindPkg(Arch);
1115       if (Pkg.end() == true) {
1116          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1117             return false;
1118       }
1119
1120       /* Caching the old end point speeds up generation substantially */
1121       if (OldDepVer != Ver) {
1122          OldDepLast = NULL;
1123          OldDepVer = Ver;
1124       }
1125
1126       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1127    }
1128    else
1129    {
1130       /* Caching the old end point speeds up generation substantially */
1131       if (OldDepVer != Ver) {
1132          OldDepLast = NULL;
1133          OldDepVer = Ver;
1134       }
1135
1136       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1137       {
1138          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1139                return false;
1140       }
1141    }
1142    return true;
1143 }
1144                                                                         /*}}}*/
1145 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1146 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1147                                                 StringView PkgName,
1148                                                 StringView PkgArch,
1149                                                 StringView Version,
1150                                                 uint8_t const Flags)
1151 {
1152    pkgCache const &Cache = Owner->Cache;
1153    Dynamic<StringView> DynPkgName(PkgName);
1154    Dynamic<StringView> DynArch(PkgArch);
1155    Dynamic<StringView> DynVersion(Version);
1156
1157    // We do not add self referencing provides
1158    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1159         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1160          (Version.empty() || Version == Ver.VerStr()))
1161       return true;
1162
1163    // Locate the target package
1164    pkgCache::PkgIterator Pkg;
1165    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1166    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1167       return false;
1168
1169    map_stringitem_t idxProvideVersion = 0;
1170    if (Version.empty() == false) {
1171       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1172       if (unlikely(idxProvideVersion == 0))
1173          return false;
1174    }
1175    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1176 }
1177 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1178                                     pkgCache::PkgIterator &Pkg,
1179                                     map_pointer_t const ProvideVersion,
1180                                     uint8_t const Flags)
1181 {
1182    // Get a structure
1183    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1184    if (unlikely(Provides == 0))
1185       return false;
1186    ++Cache.HeaderP->ProvidesCount;
1187
1188    // Fill it in
1189    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1190    Prv->Version = Ver.Index();
1191    Prv->ProvideVersion = ProvideVersion;
1192    Prv->Flags = Flags;
1193    Prv->NextPkgProv = Ver->ProvidesList;
1194    Ver->ProvidesList = Prv.Index();
1195
1196    // Link it to the package
1197    Prv->ParentPkg = Pkg.Index();
1198    Prv->NextProvides = Pkg->ProvidesList;
1199    Pkg->ProvidesList = Prv.Index();
1200    return true;
1201 }
1202                                                                         /*}}}*/
1203 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1204 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1205                                 StringView Version, uint8_t const Flags) {
1206    pkgCache &Cache = Owner->Cache;
1207    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1208    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1209    Dynamic<StringView> DynPackage(Package);
1210    Dynamic<StringView> DynVersion(Version);
1211
1212    if (Grp.end() == true)
1213       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1214    else
1215    {
1216       map_stringitem_t idxProvideVersion = 0;
1217       if (Version.empty() == false) {
1218          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1219          if (unlikely(idxProvideVersion == 0))
1220             return false;
1221       }
1222
1223       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1224       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1225       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1226       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1227       pkgCache::PkgIterator Pkg;
1228       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1229       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1230       {
1231          if (isImplicit && OwnerPkg == Pkg)
1232             continue;
1233          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1234             continue;
1235          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1236             return false;
1237       }
1238    }
1239    return true;
1240 }
1241                                                                         /*}}}*/
1242 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1243       pkgCache::VerIterator const &Ver)
1244 {
1245    return Hash == Ver->Hash;
1246 }
1247                                                                         /*}}}*/
1248 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1249 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1250                                    unsigned long Flags)
1251 {
1252    if (File.empty() && Site.empty())
1253    {
1254       CurrentRlsFile = NULL;
1255       return true;
1256    }
1257
1258    // Get some space for the structure
1259    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1260    if (unlikely(idxFile == 0))
1261       return false;
1262    CurrentRlsFile = Cache.RlsFileP + idxFile;
1263
1264    // Fill it in
1265    map_stringitem_t const idxFileName = WriteStringInMap(File);
1266    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1267    if (unlikely(idxFileName == 0 || idxSite == 0))
1268       return false;
1269    CurrentRlsFile->FileName = idxFileName;
1270    CurrentRlsFile->Site = idxSite;
1271    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1272    CurrentRlsFile->Flags = Flags;
1273    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1274    RlsFileName = File;
1275    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1276    Cache.HeaderP->ReleaseFileCount++;
1277
1278    return true;
1279 }
1280                                                                         /*}}}*/
1281 // ListParser::NewTag - Create a Tag element                            /*{{{*/
1282 // ---------------------------------------------------------------------
1283 /* */
1284 bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
1285                                            const char *NameStart,
1286                                            unsigned int NameSize)
1287 {
1288    return Owner->NewTag(Ver, NameStart, NameSize);
1289 }
1290 bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
1291                                            const char *NameStart,
1292                                            unsigned int NameSize)
1293 {
1294    // Get a structure
1295    unsigned long Tagg = AllocateInMap(sizeof(pkgCache::Tag));
1296    if (Tagg == 0)
1297       return false;
1298    Cache.HeaderP->TagCount++;
1299
1300    // Fill it in
1301    pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
1302    Tg->Name = WriteStringInMap(NameStart,NameSize);
1303    if (Tg->Name == 0)
1304       return false;
1305    Tg->NextTag = Ver->TagList;
1306    Ver->TagList = Tg.Index();
1307
1308    return true;
1309 }
1310                                                                         /*}}}*/
1311 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1312 // ---------------------------------------------------------------------
1313 /* This is used to select which file is to be associated with all newly
1314    added versions. The caller is responsible for setting the IMS fields. */
1315 bool pkgCacheGenerator::SelectFile(std::string const &File,
1316                                    pkgIndexFile const &Index,
1317                                    std::string const &Architecture,
1318                                    std::string const &Component,
1319                                    unsigned long const Flags)
1320 {
1321    // Get some space for the structure
1322    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1323    if (unlikely(idxFile == 0))
1324       return false;
1325    CurrentFile = Cache.PkgFileP + idxFile;
1326
1327    // Fill it in
1328    map_stringitem_t const idxFileName = WriteStringInMap(File);
1329    if (unlikely(idxFileName == 0))
1330       return false;
1331    CurrentFile->FileName = idxFileName;
1332    CurrentFile->NextFile = Cache.HeaderP->FileList;
1333    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1334    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1335    if (unlikely(idxIndexType == 0))
1336       return false;
1337    CurrentFile->IndexType = idxIndexType;
1338    if (Architecture.empty())
1339       CurrentFile->Architecture = 0;
1340    else
1341    {
1342       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1343       if (unlikely(arch == 0))
1344          return false;
1345       CurrentFile->Architecture = arch;
1346    }
1347    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1348    if (unlikely(component == 0))
1349       return false;
1350    CurrentFile->Component = component;
1351    CurrentFile->Flags = Flags;
1352    if (CurrentRlsFile != NULL)
1353       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1354    else
1355       CurrentFile->Release = 0;
1356    PkgFileName = File;
1357    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1358    Cache.HeaderP->PackageFileCount++;
1359
1360    if (Progress != 0)
1361       Progress->SubProgress(Index.Size());
1362    return true;
1363 }
1364                                                                         /*}}}*/
1365 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1366 // ---------------------------------------------------------------------
1367 /* This is used to create handles to strings. Given the same text it
1368    always returns the same number */
1369 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1370                                                  unsigned int Size)
1371 {
1372    auto strings = &strMixed;
1373    switch(type) {
1374       case MIXED: strings = &strMixed; break;
1375       case PKGNAME: strings = &strPkgNames; break;
1376       case VERSIONNUMBER: strings = &strVersions; break;
1377       case SECTION: strings = &strSections; break;
1378       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1379    }
1380
1381    auto const item = strings->find({S, Size, nullptr, 0});
1382    if (item != strings->end())
1383       return item->item;
1384
1385    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1386    strings->insert({nullptr, Size, this, idxString});
1387    return idxString;
1388 }
1389                                                                         /*}}}*/
1390 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1391 // ---------------------------------------------------------------------
1392 /* This just verifies that each file in the list of index files exists,
1393    has matching attributes with the cache and the cache does not have
1394    any extra files. */
1395 class APT_HIDDEN ScopedErrorRevert {
1396 public:
1397    ScopedErrorRevert() { _error->PushToStack(); }
1398    ~ScopedErrorRevert() { _error->RevertToStack(); }
1399 };
1400 static bool CheckValidity(const string &CacheFile,
1401                           pkgSourceList &List,
1402                           FileIterator const Start,
1403                           FileIterator const End,
1404                           MMap **OutMap = 0,
1405                           pkgCache **OutCache = 0)
1406 {
1407    ScopedErrorRevert ser;
1408    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1409    // No file, certainly invalid
1410    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1411    {
1412       if (Debug == true)
1413          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1414       return false;
1415    }
1416
1417    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1418    {
1419       if (Debug == true)
1420          std::clog << "sources.list is newer than the cache" << std::endl;
1421       return false;
1422    }
1423
1424    // Map it
1425    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1426    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1427    if (unlikely(Map->validData()) == false)
1428       return false;
1429    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1430    pkgCache &Cache = *CacheP.get();
1431    if (_error->PendingError() || Map->Size() == 0)
1432    {
1433       if (Debug == true)
1434          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1435       return false;
1436    }
1437
1438    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1439    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1440    std::vector<pkgIndexFile *> Files;
1441    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1442    {
1443       if (Debug == true)
1444          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1445       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1446       if (RlsFile.end() == true)
1447       {
1448          if (Debug == true)
1449             std::clog << "FindInCache returned end-Pointer" << std::endl;
1450          return false;
1451       }
1452
1453       RlsVisited[RlsFile->ID] = true;
1454       if (Debug == true)
1455          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1456
1457       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1458       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1459             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1460    }
1461    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1462       if (RlsVisited[I] == false)
1463       {
1464          if (Debug == true)
1465             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1466          return false;
1467       }
1468
1469    std::copy(Start, End, std::back_inserter(Files));
1470
1471    /* Now we check every index file, see if it is in the cache,
1472       verify the IMS data and check that it is on the disk too.. */
1473    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1474    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1475    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1476    {
1477       if (Debug == true)
1478          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1479       if ((*PkgFile)->Exists() == false)
1480       {
1481          if (Debug == true)
1482             std::clog << "file doesn't exist" << std::endl;
1483          continue;
1484       }
1485
1486       // FindInCache is also expected to do an IMS check.
1487       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1488       if (File.end() == true)
1489       {
1490          if (Debug == true)
1491             std::clog << "FindInCache returned end-Pointer" << std::endl;
1492          return false;
1493       }
1494
1495       Visited[File->ID] = true;
1496       if (Debug == true)
1497          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1498    }
1499
1500    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1501       if (Visited[I] == false)
1502       {
1503          if (Debug == true)
1504             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1505          return false;
1506       }
1507
1508    if (_error->PendingError() == true)
1509    {
1510       if (Debug == true)
1511       {
1512          std::clog << "Validity failed because of pending errors:" << std::endl;
1513          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1514       }
1515       return false;
1516    }
1517
1518    if (OutMap != 0)
1519       *OutMap = Map.release();
1520    if (OutCache != 0)
1521       *OutCache = CacheP.release();
1522    return true;
1523 }
1524                                                                         /*}}}*/
1525 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1526 // ---------------------------------------------------------------------
1527 /* Size is kind of an abstract notion that is only used for the progress
1528    meter */
1529 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1530 {
1531    map_filesize_t TotalSize = 0;
1532    if (List !=  NULL)
1533    {
1534       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1535       {
1536          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1537          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1538             if ((*j)->HasPackages() == true)
1539                TotalSize += (*j)->Size();
1540       }
1541    }
1542
1543    for (; Start < End; ++Start)
1544    {
1545       if ((*Start)->HasPackages() == false)
1546          continue;
1547       TotalSize += (*Start)->Size();
1548    }
1549    return TotalSize;
1550 }
1551                                                                         /*}}}*/
1552 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1553 static void BuildCache(pkgCacheGenerator &Gen,
1554                        OpProgress * const Progress,
1555                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1556                        pkgSourceList const * const List,
1557                        FileIterator const Start, FileIterator const End)
1558 {
1559    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1560       if (I->HasPackages() == false)
1561          return;
1562
1563       if (I->Exists() == false)
1564          return;
1565
1566       if (I->FindInCache(Gen.GetCache()).end() == false)
1567       {
1568          _error->Warning("Duplicate sources.list entry %s",
1569                I->Describe().c_str());
1570          return;
1571       }
1572
1573       map_filesize_t const Size = I->Size();
1574       if (Progress != NULL)
1575          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1576       CurrentSize += Size;
1577
1578       if (I->Merge(Gen,Progress) == false)
1579          return;
1580    };
1581
1582    if (List !=  NULL)
1583    {
1584       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1585       {
1586          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1587          {
1588             _error->Warning("Duplicate sources.list entry %s",
1589                   (*i)->Describe().c_str());
1590             continue;
1591          }
1592
1593          if ((*i)->Merge(Gen, Progress) == false)
1594             continue;
1595
1596          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1597          if (Indexes != NULL)
1598             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1599       }
1600    }
1601
1602    if (Start != End)
1603    {
1604       Gen.SelectReleaseFile("", "");
1605       std::for_each(Start, End, indexFileMerge);
1606    }
1607 }
1608                                                                         /*}}}*/
1609 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1610 // ---------------------------------------------------------------------
1611 /* This makes sure that the status cache (the cache that has all
1612    index files from the sources list and all local ones) is ready
1613    to be mmaped. If OutMap is not zero then a MMap object representing
1614    the cache will be stored there. This is pretty much mandetory if you
1615    are using AllowMem. AllowMem lets the function be run as non-root
1616    where it builds the cache 'fast' into a memory buffer. */
1617 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1618 {
1619    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1620    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1621    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1622    Flags |= MMap::Moveable;
1623    if (_config->FindB("APT::Cache-Fallback", false) == true)
1624       Flags |= MMap::Fallback;
1625    if (CacheF != NULL)
1626       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1627    else
1628       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1629 }
1630 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1631       std::string const &FileName)
1632 {
1633    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1634    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1635       return false;
1636
1637    fchmod(SCacheF.Fd(),0644);
1638
1639    // Write out the main data
1640    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1641       return _error->Error(_("IO Error saving source cache"));
1642
1643    // Write out the proper header
1644    Gen->GetCache().HeaderP->Dirty = false;
1645    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1646    if (SCacheF.Seek(0) == false ||
1647          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1648       return _error->Error(_("IO Error saving source cache"));
1649    Gen->GetCache().HeaderP->Dirty = true;
1650    return true;
1651 }
1652 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1653       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1654 {
1655    Map.reset(CreateDynamicMMap(NULL, 0));
1656    if (unlikely(Map->validData()) == false)
1657       return false;
1658    FileFd CacheF(FileName, FileFd::ReadOnly);
1659    if (CacheF.IsOpen() == false || CacheF.Failed())
1660       return false;
1661    _error->PushToStack();
1662    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1663    bool const newError = _error->PendingError();
1664    _error->MergeWithStack();
1665    if (alloc == 0 && newError)
1666       return false;
1667    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1668       return false;
1669    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1670    return Gen->Start();
1671 }
1672 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1673                         MMap **OutMap, bool AllowMem)
1674    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1675 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1676                         MMap **OutMap,bool)
1677 {
1678    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1679 }
1680 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1681                         MMap **OutMap,pkgCache **OutCache, bool)
1682 {
1683    // FIXME: deprecate the ignored AllowMem parameter
1684    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1685
1686    std::vector<pkgIndexFile *> Files;
1687    if (_system->AddStatusFiles(Files) == false)
1688       return false;
1689
1690    // Decide if we can write to the files..
1691    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1692    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1693
1694    // ensure the cache directory exists
1695    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1696    {
1697       string dir = _config->FindDir("Dir::Cache");
1698       size_t const len = dir.size();
1699       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1700          dir = dir.substr(0, len - 5);
1701       if (CacheFile.empty() == false)
1702          CreateDirectory(dir, flNotFile(CacheFile));
1703       if (SrcCacheFile.empty() == false)
1704          CreateDirectory(dir, flNotFile(SrcCacheFile));
1705    }
1706
1707    if (Progress != NULL)
1708       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1709
1710    bool pkgcache_fine = false;
1711    bool srcpkgcache_fine = false;
1712    bool volatile_fine = List.GetVolatileFiles().empty();
1713
1714    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1715                      volatile_fine ? OutCache : NULL) == true)
1716    {
1717       if (Debug == true)
1718          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1719       pkgcache_fine = true;
1720       srcpkgcache_fine = true;
1721    }
1722    if (pkgcache_fine == false)
1723    {
1724       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1725       {
1726          if (Debug == true)
1727             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1728          srcpkgcache_fine = true;
1729       }
1730    }
1731
1732    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1733    {
1734       if (Progress != NULL)
1735          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1736       return true;
1737    }
1738
1739    bool Writeable = false;
1740    if (srcpkgcache_fine == false || pkgcache_fine == false)
1741    {
1742       if (CacheFile.empty() == false)
1743          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1744       else if (SrcCacheFile.empty() == false)
1745          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1746
1747       if (Debug == true)
1748          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1749    }
1750
1751    // At this point we know we need to construct something, so get storage ready
1752    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1753    if (unlikely(Map->validData()) == false)
1754       return false;
1755    if (Debug == true)
1756       std::clog << "Open memory Map (not filebased)" << std::endl;
1757
1758    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1759    map_filesize_t CurrentSize = 0;
1760    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1761    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1762    if (srcpkgcache_fine == true && pkgcache_fine == false)
1763    {
1764       if (Debug == true)
1765          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1766       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1767          return false;
1768       srcpkgcache_fine = true;
1769       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1770    }
1771    else if (srcpkgcache_fine == false)
1772    {
1773       if (Debug == true)
1774          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1775       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1776       if (Gen->Start() == false)
1777          return false;
1778
1779       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1780       BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1781                Files.end(),Files.end());
1782
1783       if (Writeable == true && SrcCacheFile.empty() == false)
1784          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1785             return false;
1786    }
1787
1788    if (pkgcache_fine == false)
1789    {
1790       if (Debug == true)
1791          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1792       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1793                Files.begin(), Files.end());
1794
1795       if (Writeable == true && CacheFile.empty() == false)
1796          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1797             return false;
1798    }
1799
1800    if (Debug == true)
1801       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1802
1803    if (volatile_fine == false)
1804    {
1805       if (Gen == nullptr)
1806       {
1807          if (Debug == true)
1808             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1809          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1810             return false;
1811       }
1812
1813       Files = List.GetVolatileFiles();
1814       BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1815                Files.begin(), Files.end());
1816    }
1817
1818    if (OutMap != nullptr)
1819       *OutMap = Map.release();
1820
1821    if (Debug == true)
1822       std::clog << "Everything is ready for shipping" << std::endl;
1823    return true;
1824 }
1825                                                                         /*}}}*/
1826 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1827 class APT_HIDDEN ScopedErrorMerge {
1828 public:
1829    ScopedErrorMerge() { _error->PushToStack(); }
1830    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1831 };
1832 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1833    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1834 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1835 {
1836    std::vector<pkgIndexFile *> Files;
1837    if (_system->AddStatusFiles(Files) == false)
1838       return false;
1839
1840    ScopedErrorMerge sem;
1841    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1842    if (unlikely(Map->validData()) == false)
1843       return false;
1844    map_filesize_t CurrentSize = 0;
1845    map_filesize_t TotalSize = 0;
1846    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1847
1848    // Build the status cache
1849    if (Progress != NULL)
1850       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1851    pkgCacheGenerator Gen(Map.get(),Progress);
1852    if (Gen.Start() == false || _error->PendingError() == true)
1853       return false;
1854    BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1855                   Files.begin(), Files.end());
1856
1857    if (_error->PendingError() == true)
1858       return false;
1859    *OutMap = Map.release();
1860
1861    return true;
1862 }
1863                                                                         /*}}}*/
1864 // IsDuplicateDescription                                               /*{{{*/
1865 static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
1866                             APT::StringView CurMd5, std::string const &CurLang)
1867 {
1868    // Descriptions in the same link-list have all the same md5
1869    if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
1870       return false;
1871    for (; Desc.end() == false; ++Desc)
1872       if (Desc.LanguageCode() == CurLang)
1873          return true;
1874    return false;
1875 }
1876                                                                         /*}}}*/
1877
1878 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1879 pkgCacheListParser::~pkgCacheListParser() {}