apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180 }                                                                       /*}}}*/
 181 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 182 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 183                                         const unsigned long &Len) {
 184    void const * const oldMap = Map.Data();
 185    map_stringitem_t const index = Map.WriteString(String, Len);
 186    if (index != 0)
 187       ReMap(oldMap, Map.Data());
 188    return index;
 189 }
 190                                                                         /*}}}*/
 191 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 192 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 201    void const * const oldMap = Map.Data();
 202    map_pointer_t const index = Map.Allocate(size);
 203    if (index != 0)
 204       ReMap(oldMap, Map.Data());
 205    return index;
 206 }
 207                                                                         /*}}}*/
 208 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 209 // ---------------------------------------------------------------------
 210 /* This provides the generation of the entries in the cache. Each loop
 211    goes through a single package record from the underlying parse engine. */
 212 bool pkgCacheGenerator::MergeList(ListParser &List,
 213                                   pkgCache::VerIterator *OutVer)
 214 {
 215    List.Owner = this;
 216
 217    unsigned int Counter = 0;
 218    while (List.Step() == true)
 219    {
 220       string const PackageName = List.Package();
 221       if (PackageName.empty() == true)
 222          return false;
 223
 224       Counter++;
 225       if (Counter % 100 == 0 && Progress != 0)
 226          Progress->Progress(List.Offset());
 227
 228       string Arch = List.Architecture();
 229       string const Version = List.Version();
 230       if (Version.empty() == true && Arch.empty() == true)
 231       {
 232          // package descriptions
 233          if (MergeListGroup(List, PackageName) == false)
 234             return false;
 235          continue;
 236       }
 237
 238       // Get a pointer to the package structure
 239       pkgCache::PkgIterator Pkg;
 240       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 241       if (NewPackage(Pkg, PackageName, Arch) == false)
 242          // TRANSLATOR: The first placeholder is a package name,
 243          // the other two should be copied verbatim as they include debug info
 244          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 245                               PackageName.c_str(), "NewPackage", 1);
 246
 247
 248       if (Version.empty() == true)
 249       {
 250          if (MergeListPackage(List, Pkg) == false)
 251             return false;
 252       }
 253       else
 254       {
 255          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 256             return false;
 257       }
 258
 259       if (OutVer != 0)
 260          return true;
 261    }
 262
 263    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 264       return _error->Error(_("Wow, you exceeded the number of package "
 265                              "names this APT is capable of."));
 266    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of versions "
 268                              "this APT is capable of."));
 269    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 270       return _error->Error(_("Wow, you exceeded the number of descriptions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 273       return _error->Error(_("Wow, you exceeded the number of dependencies "
 274                              "this APT is capable of."));
 275
 276    return true;
 277 }
 278 // CacheGenerator::MergeListGroup                                       /*{{{*/
 279 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 280 {
 281    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 282    // a group has no data on it's own, only packages have it but these
 283    // stanzas like this come from Translation- files to add descriptions,
 284    // but without a version we don't need a description for it…
 285    if (Grp.end() == true)
 286       return true;
 287    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 288
 289    pkgCache::PkgIterator Pkg;
 290    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 291    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 292       if (MergeListPackage(List, Pkg) == false)
 293          return false;
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298 // CacheGenerator::MergeListPackage                                     /*{{{*/
 299 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 300 {
 301    // we first process the package, then the descriptions
 302    // (for deb this package processing is in fact a no-op)
 303    pkgCache::VerIterator Ver(Cache);
 304    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 305    if (List.UsePackage(Pkg, Ver) == false)
 306       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 307                            Pkg.Name(), "UsePackage", 1);
 308
 309    // Find the right version to write the description
 310    MD5SumValue CurMd5 = List.Description_md5();
 311    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 312    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 313    {
 314       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 315
 316       // a version can only have one md5 describing it
 317       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 318          continue;
 319
 320       map_stringitem_t md5idx = VerDesc->md5sum;
 321       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 322       {
 323          // don't add a new description if we have one for the given
 324          // md5 && language
 325          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 326             continue;
 327
 328          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 329       }
 330
 331       // we can stop here as all "same" versions will share the description
 332       break;
 333    }
 334
 335    return true;
 336 }
 337                                                                         /*}}}*/
 338 // CacheGenerator::MergeListVersion                                     /*{{{*/
 339 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 340                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 341 {
 342    pkgCache::VerIterator Ver = Pkg.VersionList();
 343    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 344    map_pointer_t *LastVer = &Pkg->VersionList;
 345    void const * oldMap = Map.Data();
 346
 347    unsigned short const Hash = List.VersionHash();
 348    if (Ver.end() == false)
 349    {
 350       /* We know the list is sorted so we use that fact in the search.
 351          Insertion of new versions is done with correct sorting */
 352       int Res = 1;
 353       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 354       {
 355          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 356          // Version is higher as current version - insert here
 357          if (Res > 0)
 358             break;
 359          // Versionstrings are equal - is hash also equal?
 360          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 361             break;
 362          // proceed with the next till we have either the right
 363          // or we found another version (which will be lower)
 364       }
 365
 366       /* We already have a version for this item, record that we saw it */
 367       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 368       {
 369          if (List.UsePackage(Pkg,Ver) == false)
 370             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 371                                  Pkg.Name(), "UsePackage", 2);
 372
 373          if (NewFileVer(Ver,List) == false)
 374             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 375                                  Pkg.Name(), "NewFileVer", 1);
 376
 377          // Read only a single record and return
 378          if (OutVer != 0)
 379          {
 380             *OutVer = Ver;
 381             return true;
 382          }
 383
 384          return true;
 385       }
 386    }
 387
 388    // Add a new version
 389    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 390    if (unlikely(verindex == 0))
 391       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 392                            Pkg.Name(), "NewVersion", 1);
 393
 394    if (oldMap != Map.Data())
 395          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 396    *LastVer = verindex;
 397
 398    if (unlikely(List.NewVersion(Ver) == false))
 399       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 400                            Pkg.Name(), "NewVersion", 2);
 401
 402    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "UsePackage", 3);
 405
 406    if (unlikely(NewFileVer(Ver,List) == false))
 407       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 408                            Pkg.Name(), "NewFileVer", 2);
 409
 410    pkgCache::GrpIterator Grp = Pkg.Group();
 411    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 412
 413    /* If it is the first version of this package we need to add implicit
 414       Multi-Arch dependencies to all other package versions in the group now -
 415       otherwise we just add them for this new version */
 416    if (Pkg.VersionList()->NextVer == 0)
 417    {
 418       pkgCache::PkgIterator P = Grp.PackageList();
 419       Dynamic<pkgCache::PkgIterator> DynP(P);
 420       for (; P.end() != true; P = Grp.NextPkg(P))
 421       {
 422          if (P->ID == Pkg->ID)
 423             continue;
 424          pkgCache::VerIterator V = P.VersionList();
 425          Dynamic<pkgCache::VerIterator> DynV(V);
 426          for (; V.end() != true; ++V)
 427             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 428                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 429                                     Pkg.Name(), "AddImplicitDepends", 1);
 430       }
 431    }
 432    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 433       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 434                            Pkg.Name(), "AddImplicitDepends", 2);
 435
 436    // Read only a single record and return
 437    if (OutVer != 0)
 438    {
 439       *OutVer = Ver;
 440       return true;
 441    }
 442
 443    /* Record the Description(s) based on their master md5sum */
 444    MD5SumValue CurMd5 = List.Description_md5();
 445
 446    /* Before we add a new description we first search in the group for
 447       a version with a description of the same MD5 - if so we reuse this
 448       description group instead of creating our own for this version */
 449    for (pkgCache::PkgIterator P = Grp.PackageList();
 450         P.end() == false; P = Grp.NextPkg(P))
 451    {
 452       for (pkgCache::VerIterator V = P.VersionList();
 453            V.end() == false; ++V)
 454       {
 455          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 456             continue;
 457          Ver->DescriptionList = V->DescriptionList;
 458       }
 459    }
 460
 461    // We haven't found reusable descriptions, so add the first description(s)
 462    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 463    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 464    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 465       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 466          return false;
 467    return true;
 468 }
 469                                                                         /*}}}*/
 470 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 471 {
 472    pkgCache::DescIterator Desc;
 473    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 474
 475    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 476    if (unlikely(descindex == 0))
 477       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 478             Ver.ParentPkg().Name(), "NewDescription", 1);
 479
 480    md5idx = Desc->md5sum;
 481    Desc->ParentPkg = Ver.ParentPkg().Index();
 482
 483    // we add at the end, so that the start is constant as we need
 484    // that to be able to efficiently share these lists
 485    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 486    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 487    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 488    *LastNextDesc = descindex;
 489
 490    if (NewFileDesc(Desc,List) == false)
 491       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 492             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 493
 494    return true;
 495 }
 496                                                                         /*}}}*/
 497                                                                         /*}}}*/
 498 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 499 // ---------------------------------------------------------------------
 500 /* This creates a new group structure and adds it to the hash table */
 501 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 502 {
 503    Grp = Cache.FindGrp(Name);
 504    if (Grp.end() == false)
 505       return true;
 506
 507    // Get a structure
 508    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 509    if (unlikely(Group == 0))
 510       return false;
 511
 512    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 513    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 514    if (unlikely(idxName == 0))
 515       return false;
 516    Grp->Name = idxName;
 517
 518    // Insert it into the hash table
 519    unsigned long const Hash = Cache.Hash(Name);
 520    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 521
 522    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 523       insertAt = &(Cache.GrpP + *insertAt)->Next;
 524    Grp->Next = *insertAt;
 525    *insertAt = Group;
 526
 527    Grp->ID = Cache.HeaderP->GroupCount++;
 528    return true;
 529 }
 530                                                                         /*}}}*/
 531 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 532 // ---------------------------------------------------------------------
 533 /* This creates a new package structure and adds it to the hash table */
 534 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 535                                         StringView Arch) {
 536    pkgCache::GrpIterator Grp;
 537    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 538    if (unlikely(NewGroup(Grp, Name) == false))
 539       return false;
 540
 541    Pkg = Grp.FindPkg(Arch);
 542       if (Pkg.end() == false)
 543          return true;
 544
 545    // Get a structure
 546    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 547    if (unlikely(Package == 0))
 548       return false;
 549    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 550
 551    // Set the name, arch and the ID
 552    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 553    Pkg->Group = Grp.Index();
 554    // all is mapped to the native architecture
 555    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 556    if (unlikely(idxArch == 0))
 557       return false;
 558    Pkg->Arch = idxArch;
 559    Pkg->ID = Cache.HeaderP->PackageCount++;
 560
 561    // Insert the package into our package list
 562    if (Grp->FirstPackage == 0) // the group is new
 563    {
 564       Grp->FirstPackage = Package;
 565       // Insert it into the hash table
 566       map_id_t const Hash = Cache.Hash(Name);
 567       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 568       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 569          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 570       Pkg->NextPackage = *insertAt;
 571       *insertAt = Package;
 572    }
 573    else // Group the Packages together
 574    {
 575       // but first get implicit provides done
 576       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 577       {
 578          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 579          if (M.end() == false) {
 580             pkgCache::PrvIterator Prv;
 581             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 582             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 583             {
 584                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 585                   continue;
 586                pkgCache::VerIterator Ver = Prv.OwnerVer();
 587                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 588                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 589                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 590                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 591                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 592                      return false;
 593             }
 594          }
 595
 596
 597          pkgCache::PkgIterator P;
 598          pkgCache::VerIterator Ver;
 599          Dynamic<pkgCache::PkgIterator> DynP(P);
 600          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 601
 602          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 603             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 604                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 605                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 606                      return false;
 607       }
 608       // and negative dependencies, don't forget negative dependencies
 609       {
 610          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 611          if (M.end() == false) {
 612             pkgCache::DepIterator Dep;
 613             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 614             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 615             {
 616                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 617                   continue;
 618                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 619                      Dep->Type != pkgCache::Dep::Replaces)
 620                   continue;
 621                pkgCache::VerIterator Ver = Dep.ParentVer();
 622                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 623                map_pointer_t * unused = NULL;
 624                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 625                   return false;
 626             }
 627          }
 628       }
 629
 630       // this package is the new last package
 631       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 632       Pkg->NextPackage = LastPkg->NextPackage;
 633       LastPkg->NextPackage = Package;
 634    }
 635    Grp->LastPackage = Package;
 636
 637    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 638    if (Arch == "any")
 639    {
 640       size_t const found = Name.find(':');
 641       StringView const NameA = Name.substr(0, found);
 642       StringView const ArchA = Name.substr(found + 1);
 643       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 644       if (PkgA.end() == false)
 645       {
 646          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 647          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 648          for (; Prv.end() == false; ++Prv)
 649          {
 650             if (Prv.IsMultiArchImplicit())
 651                continue;
 652             pkgCache::VerIterator V = Prv.OwnerVer();
 653             if (ArchA != V.ParentPkg().Arch())
 654                continue;
 655             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 656                return false;
 657          }
 658          pkgCache::VerIterator V = PkgA.VersionList();
 659          Dynamic<pkgCache::VerIterator> DynV(V);
 660          for (; V.end() == false; ++V)
 661          {
 662             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 663                return false;
 664          }
 665       }
 666    }
 667    return true;
 668 }
 669                                                                         /*}}}*/
 670 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 671 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 672                                            pkgCache::PkgIterator &P,
 673                                            pkgCache::VerIterator &V)
 674 {
 675    // copy P.Arch() into a string here as a cache remap
 676    // in NewDepends() later may alter the pointer location
 677    string Arch = P.Arch() == NULL ? "" : P.Arch();
 678    map_pointer_t *OldDepLast = NULL;
 679    /* MultiArch handling introduces a lot of implicit Dependencies:
 680       - MultiArch: same → Co-Installable if they have the same version
 681       - All others conflict with all other group members */
 682    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 683    pkgCache::PkgIterator D = G.PackageList();
 684    Dynamic<pkgCache::PkgIterator> DynD(D);
 685    map_stringitem_t const VerStrIdx = V->VerStr;
 686    for (; D.end() != true; D = G.NextPkg(D))
 687    {
 688       if (Arch == D.Arch() || D->VersionList == 0)
 689          continue;
 690       /* We allow only one installed arch at the time
 691          per group, therefore each group member conflicts
 692          with all other group members */
 693       if (coInstall == true)
 694       {
 695          // Replaces: ${self}:other ( << ${binary:Version})
 696          NewDepends(D, V, VerStrIdx,
 697                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 698                     OldDepLast);
 699          // Breaks: ${self}:other (!= ${binary:Version})
 700          NewDepends(D, V, VerStrIdx,
 701                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 702                     OldDepLast);
 703       } else {
 704          // Conflicts: ${self}:other
 705          NewDepends(D, V, 0,
 706                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 707                     OldDepLast);
 708       }
 709    }
 710    return true;
 711 }
 712 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 713                                            pkgCache::PkgIterator &D)
 714 {
 715    /* MultiArch handling introduces a lot of implicit Dependencies:
 716       - MultiArch: same → Co-Installable if they have the same version
 717       - All others conflict with all other group members */
 718    map_pointer_t *OldDepLast = NULL;
 719    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 720    if (coInstall == true)
 721    {
 722       map_stringitem_t const VerStrIdx = V->VerStr;
 723       // Replaces: ${self}:other ( << ${binary:Version})
 724       NewDepends(D, V, VerStrIdx,
 725                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 726                  OldDepLast);
 727       // Breaks: ${self}:other (!= ${binary:Version})
 728       NewDepends(D, V, VerStrIdx,
 729                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 730                  OldDepLast);
 731    } else {
 732       // Conflicts: ${self}:other
 733       NewDepends(D, V, 0,
 734                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 735                  OldDepLast);
 736    }
 737    return true;
 738 }
 739
 740                                                                         /*}}}*/
 741 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 742 // ---------------------------------------------------------------------
 743 /* */
 744 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 745                                    ListParser &List)
 746 {
 747    if (CurrentFile == 0)
 748       return true;
 749
 750    // Get a structure
 751    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 752    if (VerFile == 0)
 753       return false;
 754
 755    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 756    VF->File = CurrentFile - Cache.PkgFileP;
 757
 758    // Link it to the end of the list
 759    map_pointer_t *Last = &Ver->FileList;
 760    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 761       Last = &V->NextFile;
 762    VF->NextFile = *Last;
 763    *Last = VF.Index();
 764
 765    VF->Offset = List.Offset();
 766    VF->Size = List.Size();
 767    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 768       Cache.HeaderP->MaxVerFileSize = VF->Size;
 769    Cache.HeaderP->VerFileCount++;
 770
 771    return true;
 772 }
 773                                                                         /*}}}*/
 774 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* This puts a version structure in the linked list */
 777 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 778                                             const string &VerStr,
 779                                             map_pointer_t const ParentPkg,
 780                                             unsigned short const Hash,
 781                                             map_pointer_t const Next)
 782 {
 783    // Get a structure
 784    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 785    if (Version == 0)
 786       return 0;
 787
 788    // Fill it in
 789    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 790    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 791    Ver->NextVer = Next;
 792    Ver->ParentPkg = ParentPkg;
 793    Ver->Hash = Hash;
 794    Ver->ID = Cache.HeaderP->VersionCount++;
 795
 796    // try to find the version string in the group for reuse
 797    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 798    pkgCache::GrpIterator Grp = Pkg.Group();
 799    if (Pkg.end() == false && Grp.end() == false)
 800    {
 801       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 802       {
 803          if (Pkg == P)
 804             continue;
 805          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 806          {
 807             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 808             if (cmp == 0)
 809             {
 810                Ver->VerStr = V->VerStr;
 811                return Version;
 812             }
 813             else if (cmp < 0)
 814                break;
 815          }
 816       }
 817    }
 818    // haven't found the version string, so create
 819    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 820    if (unlikely(idxVerStr == 0))
 821       return 0;
 822    Ver->VerStr = idxVerStr;
 823    return Version;
 824 }
 825                                                                         /*}}}*/
 826 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 827 // ---------------------------------------------------------------------
 828 /* */
 829 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 830                                    ListParser &List)
 831 {
 832    if (CurrentFile == 0)
 833       return true;
 834
 835    // Get a structure
 836    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 837    if (DescFile == 0)
 838       return false;
 839
 840    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 841    DF->File = CurrentFile - Cache.PkgFileP;
 842
 843    // Link it to the end of the list
 844    map_pointer_t *Last = &Desc->FileList;
 845    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 846       Last = &D->NextFile;
 847
 848    DF->NextFile = *Last;
 849    *Last = DF.Index();
 850
 851    DF->Offset = List.Offset();
 852    DF->Size = List.Size();
 853    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 854       Cache.HeaderP->MaxDescFileSize = DF->Size;
 855    Cache.HeaderP->DescFileCount++;
 856
 857    return true;
 858 }
 859                                                                         /*}}}*/
 860 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 861 // ---------------------------------------------------------------------
 862 /* This puts a description structure in the linked list */
 863 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 864                                             const string &Lang,
 865                                             const MD5SumValue &md5sum,
 866                                             map_stringitem_t const idxmd5str)
 867 {
 868    // Get a structure
 869    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 870    if (Description == 0)
 871       return 0;
 872
 873    // Fill it in
 874    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 875    Desc->ID = Cache.HeaderP->DescriptionCount++;
 876    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 877    if (unlikely(idxlanguage_code == 0))
 878       return 0;
 879    Desc->language_code = idxlanguage_code;
 880
 881    if (idxmd5str != 0)
 882       Desc->md5sum = idxmd5str;
 883    else
 884    {
 885       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 886       if (unlikely(idxmd5sum == 0))
 887          return 0;
 888       Desc->md5sum = idxmd5sum;
 889    }
 890
 891    return Description;
 892 }
 893                                                                         /*}}}*/
 894 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 895 // ---------------------------------------------------------------------
 896 /* This creates a dependency element in the tree. It is linked to the
 897    version and to the package that it is pointing to. */
 898 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 899                                    pkgCache::VerIterator &Ver,
 900                                    map_pointer_t const Version,
 901                                    uint8_t const Op,
 902                                    uint8_t const Type,
 903                                    map_pointer_t* &OldDepLast)
 904 {
 905    void const * const oldMap = Map.Data();
 906    // Get a structure
 907    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 908    if (unlikely(Dependency == 0))
 909       return false;
 910
 911    bool isDuplicate = false;
 912    map_pointer_t DependencyData = 0;
 913    map_pointer_t PreviousData = 0;
 914    if (Pkg->RevDepends != 0)
 915    {
 916       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 917       DependencyData = L->DependencyData;
 918       do {
 919          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 920          if (Version > D->Version)
 921             break;
 922          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 923          {
 924             isDuplicate = true;
 925             break;
 926          }
 927          PreviousData = DependencyData;
 928          DependencyData = D->NextData;
 929       } while (DependencyData != 0);
 930    }
 931
 932    if (isDuplicate == false)
 933    {
 934       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 935       if (unlikely(DependencyData == 0))
 936         return false;
 937    }
 938
 939    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 940    Link->ParentVer = Ver.Index();
 941    Link->DependencyData = DependencyData;
 942    Link->ID = Cache.HeaderP->DependsCount++;
 943
 944    pkgCache::DepIterator Dep(Cache, Link);
 945    if (isDuplicate == false)
 946    {
 947       Dep->Type = Type;
 948       Dep->CompareOp = Op;
 949       Dep->Version = Version;
 950       Dep->Package = Pkg.Index();
 951       ++Cache.HeaderP->DependsDataCount;
 952       if (PreviousData != 0)
 953       {
 954          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 955          Dep->NextData = D->NextData;
 956          D->NextData = DependencyData;
 957       }
 958       else if (Pkg->RevDepends != 0)
 959       {
 960          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 961          Dep->NextData = D->DependencyData;
 962       }
 963    }
 964
 965    if (isDuplicate == true || PreviousData != 0)
 966    {
 967       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 968       Link->NextRevDepends = L->NextRevDepends;
 969       L->NextRevDepends = Dependency;
 970    }
 971    else
 972    {
 973       Link->NextRevDepends = Pkg->RevDepends;
 974       Pkg->RevDepends = Dependency;
 975    }
 976
 977
 978    // Do we know where to link the Dependency to?
 979    if (OldDepLast == NULL)
 980    {
 981       OldDepLast = &Ver->DependsList;
 982       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 983          OldDepLast = &D->NextDepends;
 984    } else if (oldMap != Map.Data())
 985       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 986
 987    Dep->NextDepends = *OldDepLast;
 988    *OldDepLast = Dependency;
 989    OldDepLast = &Dep->NextDepends;
 990    return true;
 991 }
 992                                                                         /*}}}*/
 993 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 994 // ---------------------------------------------------------------------
 995 /* This creates a Group and the Package to link this dependency to if
 996    needed and handles also the caching of the old endpoint */
 997 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
 998                                                StringView PackageName,
 999                                                StringView Arch,
1000                                                StringView Version,
1001                                                uint8_t const Op,
1002                                                uint8_t const Type)
1003 {
1004    pkgCache::GrpIterator Grp;
1005    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1006    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1007       return false;
1008
1009    map_stringitem_t idxVersion = 0;
1010    if (Version.empty() == false)
1011    {
1012       int const CmpOp = Op & 0x0F;
1013       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1014       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1015          idxVersion = Ver->VerStr;
1016
1017       if (idxVersion == 0)
1018       {
1019          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1020          if (unlikely(idxVersion == 0))
1021             return false;
1022       }
1023    }
1024
1025    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1026          Type == pkgCache::Dep::Conflicts ||
1027          Type == pkgCache::Dep::Replaces);
1028
1029    pkgCache::PkgIterator Pkg;
1030    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1031    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1032    {
1033       // Locate the target package
1034       Pkg = Grp.FindPkg(Arch);
1035       if (Pkg.end() == true) {
1036          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1037             return false;
1038       }
1039
1040       /* Caching the old end point speeds up generation substantially */
1041       if (OldDepVer != Ver) {
1042          OldDepLast = NULL;
1043          OldDepVer = Ver;
1044       }
1045
1046       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1047    }
1048    else
1049    {
1050       /* Caching the old end point speeds up generation substantially */
1051       if (OldDepVer != Ver) {
1052          OldDepLast = NULL;
1053          OldDepVer = Ver;
1054       }
1055
1056       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1057       {
1058          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1059                return false;
1060       }
1061    }
1062    return true;
1063 }
1064                                                                         /*}}}*/
1065 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1066 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1067                                                 StringView PkgName,
1068                                                 StringView PkgArch,
1069                                                 StringView Version,
1070                                                 uint8_t const Flags)
1071 {
1072    pkgCache const &Cache = Owner->Cache;
1073
1074    // We do not add self referencing provides
1075    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1076         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1077          (Version.empty() || Version == Ver.VerStr()))
1078       return true;
1079
1080    // Locate the target package
1081    pkgCache::PkgIterator Pkg;
1082    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1083    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1084       return false;
1085
1086    map_stringitem_t idxProvideVersion = 0;
1087    if (Version.empty() == false) {
1088       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1089       if (unlikely(idxProvideVersion == 0))
1090          return false;
1091    }
1092    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1093 }
1094 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1095                                     pkgCache::PkgIterator &Pkg,
1096                                     map_pointer_t const ProvideVersion,
1097                                     uint8_t const Flags)
1098 {
1099    // Get a structure
1100    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1101    if (unlikely(Provides == 0))
1102       return false;
1103    ++Cache.HeaderP->ProvidesCount;
1104
1105    // Fill it in
1106    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1107    Prv->Version = Ver.Index();
1108    Prv->ProvideVersion = ProvideVersion;
1109    Prv->Flags = Flags;
1110    Prv->NextPkgProv = Ver->ProvidesList;
1111    Ver->ProvidesList = Prv.Index();
1112
1113    // Link it to the package
1114    Prv->ParentPkg = Pkg.Index();
1115    Prv->NextProvides = Pkg->ProvidesList;
1116    Pkg->ProvidesList = Prv.Index();
1117    return true;
1118 }
1119                                                                         /*}}}*/
1120 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1121 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1122                                 StringView Version, uint8_t const Flags) {
1123    pkgCache &Cache = Owner->Cache;
1124    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1125    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1126
1127    if (Grp.end() == true)
1128       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1129    else
1130    {
1131       map_stringitem_t idxProvideVersion = 0;
1132       if (Version.empty() == false) {
1133          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1134          if (unlikely(idxProvideVersion == 0))
1135             return false;
1136       }
1137
1138       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1139       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1140       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1141       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1142       pkgCache::PkgIterator Pkg;
1143       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1144       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1145       {
1146          if (isImplicit && OwnerPkg == Pkg)
1147             continue;
1148          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1149             continue;
1150          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1151             return false;
1152       }
1153    }
1154    return true;
1155 }
1156                                                                         /*}}}*/
1157 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1158       pkgCache::VerIterator const &Ver)
1159 {
1160    return Hash == Ver->Hash;
1161 }
1162                                                                         /*}}}*/
1163 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1164 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1165                                    unsigned long Flags)
1166 {
1167    if (File.empty() && Site.empty())
1168    {
1169       CurrentRlsFile = NULL;
1170       return true;
1171    }
1172
1173    // Get some space for the structure
1174    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1175    if (unlikely(idxFile == 0))
1176       return false;
1177    CurrentRlsFile = Cache.RlsFileP + idxFile;
1178
1179    // Fill it in
1180    map_stringitem_t const idxFileName = WriteStringInMap(File);
1181    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1182    if (unlikely(idxFileName == 0 || idxSite == 0))
1183       return false;
1184    CurrentRlsFile->FileName = idxFileName;
1185    CurrentRlsFile->Site = idxSite;
1186    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1187    CurrentRlsFile->Flags = Flags;
1188    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1189    RlsFileName = File;
1190    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1191    Cache.HeaderP->ReleaseFileCount++;
1192
1193    return true;
1194 }
1195                                                                         /*}}}*/
1196 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1197 // ---------------------------------------------------------------------
1198 /* This is used to select which file is to be associated with all newly
1199    added versions. The caller is responsible for setting the IMS fields. */
1200 bool pkgCacheGenerator::SelectFile(std::string const &File,
1201                                    pkgIndexFile const &Index,
1202                                    std::string const &Architecture,
1203                                    std::string const &Component,
1204                                    unsigned long const Flags)
1205 {
1206    // Get some space for the structure
1207    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1208    if (unlikely(idxFile == 0))
1209       return false;
1210    CurrentFile = Cache.PkgFileP + idxFile;
1211
1212    // Fill it in
1213    map_stringitem_t const idxFileName = WriteStringInMap(File);
1214    if (unlikely(idxFileName == 0))
1215       return false;
1216    CurrentFile->FileName = idxFileName;
1217    CurrentFile->NextFile = Cache.HeaderP->FileList;
1218    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1219    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1220    if (unlikely(idxIndexType == 0))
1221       return false;
1222    CurrentFile->IndexType = idxIndexType;
1223    if (Architecture.empty())
1224       CurrentFile->Architecture = 0;
1225    else
1226    {
1227       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1228       if (unlikely(arch == 0))
1229          return false;
1230       CurrentFile->Architecture = arch;
1231    }
1232    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1233    if (unlikely(component == 0))
1234       return false;
1235    CurrentFile->Component = component;
1236    CurrentFile->Flags = Flags;
1237    if (CurrentRlsFile != NULL)
1238       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1239    else
1240       CurrentFile->Release = 0;
1241    PkgFileName = File;
1242    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1243    Cache.HeaderP->PackageFileCount++;
1244
1245    if (Progress != 0)
1246       Progress->SubProgress(Index.Size());
1247    return true;
1248 }
1249                                                                         /*}}}*/
1250 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1251 // ---------------------------------------------------------------------
1252 /* This is used to create handles to strings. Given the same text it
1253    always returns the same number */
1254 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1255                                                  unsigned int Size)
1256 {
1257    auto strings = &strMixed;
1258    switch(type) {
1259       case MIXED: strings = &strMixed; break;
1260       case PKGNAME: strings = &strPkgNames; break;
1261       case VERSIONNUMBER: strings = &strVersions; break;
1262       case SECTION: strings = &strSections; break;
1263       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1264    }
1265
1266    auto const item = strings->find({S, Size, nullptr, 0});
1267    if (item != strings->end())
1268       return item->item;
1269
1270    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1271    strings->insert({nullptr, Size, this, idxString});
1272    return idxString;
1273 }
1274                                                                         /*}}}*/
1275 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1276 // ---------------------------------------------------------------------
1277 /* This just verifies that each file in the list of index files exists,
1278    has matching attributes with the cache and the cache does not have
1279    any extra files. */
1280 class APT_HIDDEN ScopedErrorRevert {
1281 public:
1282    ScopedErrorRevert() { _error->PushToStack(); }
1283    ~ScopedErrorRevert() { _error->RevertToStack(); }
1284 };
1285 static bool CheckValidity(const string &CacheFile,
1286                           pkgSourceList &List,
1287                           FileIterator const Start,
1288                           FileIterator const End,
1289                           MMap **OutMap = 0,
1290                           pkgCache **OutCache = 0)
1291 {
1292    ScopedErrorRevert ser;
1293    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1294    // No file, certainly invalid
1295    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1296    {
1297       if (Debug == true)
1298          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1299       return false;
1300    }
1301
1302    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1303    {
1304       if (Debug == true)
1305          std::clog << "sources.list is newer than the cache" << std::endl;
1306       return false;
1307    }
1308
1309    // Map it
1310    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1311    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1312    if (unlikely(Map->validData()) == false)
1313       return false;
1314    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1315    pkgCache &Cache = *CacheP.get();
1316    if (_error->PendingError() || Map->Size() == 0)
1317    {
1318       if (Debug == true)
1319          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1320       return false;
1321    }
1322
1323    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1324    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1325    std::vector<pkgIndexFile *> Files;
1326    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1327    {
1328       if (Debug == true)
1329          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1330       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1331       if (RlsFile.end() == true)
1332       {
1333          if (Debug == true)
1334             std::clog << "FindInCache returned end-Pointer" << std::endl;
1335          return false;
1336       }
1337
1338       RlsVisited[RlsFile->ID] = true;
1339       if (Debug == true)
1340          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1341
1342       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1343       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1344             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1345    }
1346    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1347       if (RlsVisited[I] == false)
1348       {
1349          if (Debug == true)
1350             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1351          return false;
1352       }
1353
1354    std::copy(Start, End, std::back_inserter(Files));
1355
1356    /* Now we check every index file, see if it is in the cache,
1357       verify the IMS data and check that it is on the disk too.. */
1358    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1359    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1360    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1361    {
1362       if (Debug == true)
1363          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1364       if ((*PkgFile)->Exists() == false)
1365       {
1366          if (Debug == true)
1367             std::clog << "file doesn't exist" << std::endl;
1368          continue;
1369       }
1370
1371       // FindInCache is also expected to do an IMS check.
1372       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1373       if (File.end() == true)
1374       {
1375          if (Debug == true)
1376             std::clog << "FindInCache returned end-Pointer" << std::endl;
1377          return false;
1378       }
1379
1380       Visited[File->ID] = true;
1381       if (Debug == true)
1382          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1383    }
1384
1385    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1386       if (Visited[I] == false)
1387       {
1388          if (Debug == true)
1389             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1390          return false;
1391       }
1392
1393    if (_error->PendingError() == true)
1394    {
1395       if (Debug == true)
1396       {
1397          std::clog << "Validity failed because of pending errors:" << std::endl;
1398          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1399       }
1400       return false;
1401    }
1402
1403    if (OutMap != 0)
1404       *OutMap = Map.release();
1405    if (OutCache != 0)
1406       *OutCache = CacheP.release();
1407    return true;
1408 }
1409                                                                         /*}}}*/
1410 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1411 // ---------------------------------------------------------------------
1412 /* Size is kind of an abstract notion that is only used for the progress
1413    meter */
1414 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1415 {
1416    map_filesize_t TotalSize = 0;
1417    if (List !=  NULL)
1418    {
1419       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1420       {
1421          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1422          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1423             if ((*j)->HasPackages() == true)
1424                TotalSize += (*j)->Size();
1425       }
1426    }
1427
1428    for (; Start < End; ++Start)
1429    {
1430       if ((*Start)->HasPackages() == false)
1431          continue;
1432       TotalSize += (*Start)->Size();
1433    }
1434    return TotalSize;
1435 }
1436                                                                         /*}}}*/
1437 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1438 static bool BuildCache(pkgCacheGenerator &Gen,
1439                        OpProgress * const Progress,
1440                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1441                        pkgSourceList const * const List,
1442                        FileIterator const Start, FileIterator const End)
1443 {
1444    bool mergeFailure = false;
1445
1446    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1447       if (I->HasPackages() == false || mergeFailure)
1448          return;
1449
1450       if (I->Exists() == false)
1451          return;
1452
1453       if (I->FindInCache(Gen.GetCache()).end() == false)
1454       {
1455          _error->Warning("Duplicate sources.list entry %s",
1456                I->Describe().c_str());
1457          return;
1458       }
1459
1460       map_filesize_t const Size = I->Size();
1461       if (Progress != NULL)
1462          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1463       CurrentSize += Size;
1464
1465       if (I->Merge(Gen,Progress) == false)
1466          mergeFailure = true;
1467    };
1468
1469    if (List !=  NULL)
1470    {
1471       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1472       {
1473          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1474          {
1475             _error->Warning("Duplicate sources.list entry %s",
1476                   (*i)->Describe().c_str());
1477             continue;
1478          }
1479
1480          if ((*i)->Merge(Gen, Progress) == false)
1481             return false;
1482
1483          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1484          if (Indexes != NULL)
1485             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1486          if (mergeFailure)
1487             return false;
1488       }
1489    }
1490
1491    if (Start != End)
1492    {
1493       Gen.SelectReleaseFile("", "");
1494       std::for_each(Start, End, indexFileMerge);
1495       if (mergeFailure)
1496          return false;
1497    }
1498    return true;
1499 }
1500                                                                         /*}}}*/
1501 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1502 // ---------------------------------------------------------------------
1503 /* This makes sure that the status cache (the cache that has all
1504    index files from the sources list and all local ones) is ready
1505    to be mmaped. If OutMap is not zero then a MMap object representing
1506    the cache will be stored there. This is pretty much mandetory if you
1507    are using AllowMem. AllowMem lets the function be run as non-root
1508    where it builds the cache 'fast' into a memory buffer. */
1509 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1510 {
1511    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1512    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1513    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1514    Flags |= MMap::Moveable;
1515    if (_config->FindB("APT::Cache-Fallback", false) == true)
1516       Flags |= MMap::Fallback;
1517    if (CacheF != NULL)
1518       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1519    else
1520       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1521 }
1522 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1523       std::string const &FileName)
1524 {
1525    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1526    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1527       return false;
1528
1529    fchmod(SCacheF.Fd(),0644);
1530
1531    // Write out the main data
1532    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1533       return _error->Error(_("IO Error saving source cache"));
1534
1535    // Write out the proper header
1536    Gen->GetCache().HeaderP->Dirty = false;
1537    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1538    if (SCacheF.Seek(0) == false ||
1539          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1540       return _error->Error(_("IO Error saving source cache"));
1541    Gen->GetCache().HeaderP->Dirty = true;
1542    return true;
1543 }
1544 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1545       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1546 {
1547    Map.reset(CreateDynamicMMap(NULL, 0));
1548    if (unlikely(Map->validData()) == false)
1549       return false;
1550    FileFd CacheF(FileName, FileFd::ReadOnly);
1551    if (CacheF.IsOpen() == false || CacheF.Failed())
1552       return false;
1553    _error->PushToStack();
1554    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1555    bool const newError = _error->PendingError();
1556    _error->MergeWithStack();
1557    if (alloc == 0 && newError)
1558       return false;
1559    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1560       return false;
1561    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1562    return true;
1563 }
1564 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1565                         MMap **OutMap, bool AllowMem)
1566    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1567 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1568                         MMap **OutMap,bool)
1569 {
1570    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1571 }
1572 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1573                         MMap **OutMap,pkgCache **OutCache, bool)
1574 {
1575    // FIXME: deprecate the ignored AllowMem parameter
1576    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1577
1578    std::vector<pkgIndexFile *> Files;
1579    if (_system->AddStatusFiles(Files) == false)
1580       return false;
1581
1582    // Decide if we can write to the files..
1583    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1584    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1585
1586    // ensure the cache directory exists
1587    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1588    {
1589       string dir = _config->FindDir("Dir::Cache");
1590       size_t const len = dir.size();
1591       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1592          dir = dir.substr(0, len - 5);
1593       if (CacheFile.empty() == false)
1594          CreateDirectory(dir, flNotFile(CacheFile));
1595       if (SrcCacheFile.empty() == false)
1596          CreateDirectory(dir, flNotFile(SrcCacheFile));
1597    }
1598
1599    if (Progress != NULL)
1600       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1601
1602    bool pkgcache_fine = false;
1603    bool srcpkgcache_fine = false;
1604    bool volatile_fine = List.GetVolatileFiles().empty();
1605
1606    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1607                      volatile_fine ? OutCache : NULL) == true)
1608    {
1609       if (Debug == true)
1610          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1611       pkgcache_fine = true;
1612       srcpkgcache_fine = true;
1613    }
1614    if (pkgcache_fine == false)
1615    {
1616       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1617       {
1618          if (Debug == true)
1619             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1620          srcpkgcache_fine = true;
1621       }
1622    }
1623
1624    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1625    {
1626       if (Progress != NULL)
1627          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1628       return true;
1629    }
1630
1631    bool Writeable = false;
1632    if (srcpkgcache_fine == false || pkgcache_fine == false)
1633    {
1634       if (CacheFile.empty() == false)
1635          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1636       else if (SrcCacheFile.empty() == false)
1637          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1638
1639       if (Debug == true)
1640          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1641    }
1642
1643    // At this point we know we need to construct something, so get storage ready
1644    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1645    if (unlikely(Map->validData()) == false)
1646       return false;
1647    if (Debug == true)
1648       std::clog << "Open memory Map (not filebased)" << std::endl;
1649
1650    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1651    map_filesize_t CurrentSize = 0;
1652    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1653    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1654    if (srcpkgcache_fine == true && pkgcache_fine == false)
1655    {
1656       if (Debug == true)
1657          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1658       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1659          return false;
1660       srcpkgcache_fine = true;
1661       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1662    }
1663    else if (srcpkgcache_fine == false)
1664    {
1665       if (Debug == true)
1666          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1667       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1668
1669       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1670       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1671                Files.end(),Files.end()) == false)
1672          return false;
1673
1674       if (Writeable == true && SrcCacheFile.empty() == false)
1675          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1676             return false;
1677    }
1678
1679    if (pkgcache_fine == false)
1680    {
1681       if (Debug == true)
1682          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1683       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1684                Files.begin(), Files.end()) == false)
1685          return false;
1686
1687       if (Writeable == true && CacheFile.empty() == false)
1688          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1689             return false;
1690    }
1691
1692    if (Debug == true)
1693       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1694
1695    if (volatile_fine == false)
1696    {
1697       if (Gen == nullptr)
1698       {
1699          if (Debug == true)
1700             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1701          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1702             return false;
1703       }
1704
1705       Files = List.GetVolatileFiles();
1706       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1707                Files.begin(), Files.end()) == false)
1708          return false;
1709    }
1710
1711    if (OutMap != nullptr)
1712       *OutMap = Map.release();
1713
1714    if (Debug == true)
1715       std::clog << "Everything is ready for shipping" << std::endl;
1716    return true;
1717 }
1718                                                                         /*}}}*/
1719 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1720 class APT_HIDDEN ScopedErrorMerge {
1721 public:
1722    ScopedErrorMerge() { _error->PushToStack(); }
1723    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1724 };
1725 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1726    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1727 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1728 {
1729    std::vector<pkgIndexFile *> Files;
1730    if (_system->AddStatusFiles(Files) == false)
1731       return false;
1732
1733    ScopedErrorMerge sem;
1734    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1735    if (unlikely(Map->validData()) == false)
1736       return false;
1737    map_filesize_t CurrentSize = 0;
1738    map_filesize_t TotalSize = 0;
1739    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1740
1741    // Build the status cache
1742    if (Progress != NULL)
1743       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1744    pkgCacheGenerator Gen(Map.get(),Progress);
1745    if (_error->PendingError() == true)
1746       return false;
1747    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1748                   Files.begin(), Files.end()) == false)
1749       return false;
1750
1751    if (_error->PendingError() == true)
1752       return false;
1753    *OutMap = Map.release();
1754
1755    return true;
1756 }
1757                                                                         /*}}}*/
1758 // IsDuplicateDescription                                               /*{{{*/
1759 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1760                             MD5SumValue const &CurMd5, std::string const &CurLang)
1761 {
1762    // Descriptions in the same link-list have all the same md5
1763    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1764       return false;
1765    for (; Desc.end() == false; ++Desc)
1766       if (Desc.LanguageCode() == CurLang)
1767          return true;
1768    return false;
1769 }
1770                                                                         /*}}}*/
1771
1772 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1773 pkgCacheListParser::~pkgCacheListParser() {}