apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180 }                                                                       /*}}}*/
 181 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 182 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 183                                         const unsigned long &Len) {
 184    void const * const oldMap = Map.Data();
 185    map_stringitem_t const index = Map.WriteString(String, Len);
 186    if (index != 0)
 187       ReMap(oldMap, Map.Data());
 188    return index;
 189 }
 190                                                                         /*}}}*/
 191 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 192 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 201    void const * const oldMap = Map.Data();
 202    map_pointer_t const index = Map.Allocate(size);
 203    if (index != 0)
 204       ReMap(oldMap, Map.Data());
 205    return index;
 206 }
 207                                                                         /*}}}*/
 208 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 209 // ---------------------------------------------------------------------
 210 /* This provides the generation of the entries in the cache. Each loop
 211    goes through a single package record from the underlying parse engine. */
 212 bool pkgCacheGenerator::MergeList(ListParser &List,
 213                                   pkgCache::VerIterator *OutVer)
 214 {
 215    List.Owner = this;
 216
 217    unsigned int Counter = 0;
 218    while (List.Step() == true)
 219    {
 220       string const PackageName = List.Package();
 221       if (PackageName.empty() == true)
 222          return false;
 223
 224       Counter++;
 225       if (Counter % 100 == 0 && Progress != 0)
 226          Progress->Progress(List.Offset());
 227
 228       string Arch = List.Architecture();
 229       string const Version = List.Version();
 230       if (Version.empty() == true && Arch.empty() == true)
 231       {
 232          // package descriptions
 233          if (MergeListGroup(List, PackageName) == false)
 234             return false;
 235          continue;
 236       }
 237
 238       // Get a pointer to the package structure
 239       pkgCache::PkgIterator Pkg;
 240       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 241       if (NewPackage(Pkg, PackageName, Arch) == false)
 242          // TRANSLATOR: The first placeholder is a package name,
 243          // the other two should be copied verbatim as they include debug info
 244          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 245                               PackageName.c_str(), "NewPackage", 1);
 246
 247
 248       if (Version.empty() == true)
 249       {
 250          if (MergeListPackage(List, Pkg) == false)
 251             return false;
 252       }
 253       else
 254       {
 255          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 256             return false;
 257       }
 258
 259       if (OutVer != 0)
 260          return true;
 261    }
 262
 263    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 264       return _error->Error(_("Wow, you exceeded the number of package "
 265                              "names this APT is capable of."));
 266    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of versions "
 268                              "this APT is capable of."));
 269    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 270       return _error->Error(_("Wow, you exceeded the number of descriptions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 273       return _error->Error(_("Wow, you exceeded the number of dependencies "
 274                              "this APT is capable of."));
 275
 276    return true;
 277 }
 278 // CacheGenerator::MergeListGroup                                       /*{{{*/
 279 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 280 {
 281    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 282    // a group has no data on it's own, only packages have it but these
 283    // stanzas like this come from Translation- files to add descriptions,
 284    // but without a version we don't need a description for it…
 285    if (Grp.end() == true)
 286       return true;
 287    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 288
 289    pkgCache::PkgIterator Pkg;
 290    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 291    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 292       if (MergeListPackage(List, Pkg) == false)
 293          return false;
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298 // CacheGenerator::MergeListPackage                                     /*{{{*/
 299 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 300 {
 301    // we first process the package, then the descriptions
 302    // (for deb this package processing is in fact a no-op)
 303    pkgCache::VerIterator Ver(Cache);
 304    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 305    if (List.UsePackage(Pkg, Ver) == false)
 306       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 307                            Pkg.Name(), "UsePackage", 1);
 308
 309    // Find the right version to write the description
 310    MD5SumValue CurMd5 = List.Description_md5();
 311    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 312    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 313    {
 314       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 315
 316       // a version can only have one md5 describing it
 317       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 318          continue;
 319
 320       map_stringitem_t md5idx = VerDesc->md5sum;
 321       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 322       {
 323          // don't add a new description if we have one for the given
 324          // md5 && language
 325          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 326             continue;
 327
 328          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 329       }
 330
 331       // we can stop here as all "same" versions will share the description
 332       break;
 333    }
 334
 335    return true;
 336 }
 337                                                                         /*}}}*/
 338 // CacheGenerator::MergeListVersion                                     /*{{{*/
 339 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 340                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 341 {
 342    pkgCache::VerIterator Ver = Pkg.VersionList();
 343    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 344    map_pointer_t *LastVer = &Pkg->VersionList;
 345    void const * oldMap = Map.Data();
 346
 347    unsigned short const Hash = List.VersionHash();
 348    if (Ver.end() == false)
 349    {
 350       /* We know the list is sorted so we use that fact in the search.
 351          Insertion of new versions is done with correct sorting */
 352       int Res = 1;
 353       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 354       {
 355          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 356          // Version is higher as current version - insert here
 357          if (Res > 0)
 358             break;
 359          // Versionstrings are equal - is hash also equal?
 360          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 361             break;
 362          // proceed with the next till we have either the right
 363          // or we found another version (which will be lower)
 364       }
 365
 366       /* We already have a version for this item, record that we saw it */
 367       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 368       {
 369          if (List.UsePackage(Pkg,Ver) == false)
 370             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 371                                  Pkg.Name(), "UsePackage", 2);
 372
 373          if (NewFileVer(Ver,List) == false)
 374             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 375                                  Pkg.Name(), "NewFileVer", 1);
 376
 377          // Read only a single record and return
 378          if (OutVer != 0)
 379          {
 380             *OutVer = Ver;
 381             return true;
 382          }
 383
 384          return true;
 385       }
 386    }
 387
 388    // Add a new version
 389    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 390    if (unlikely(verindex == 0))
 391       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 392                            Pkg.Name(), "NewVersion", 1);
 393
 394    if (oldMap != Map.Data())
 395          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 396    *LastVer = verindex;
 397
 398    if (unlikely(List.NewVersion(Ver) == false))
 399       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 400                            Pkg.Name(), "NewVersion", 2);
 401
 402    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "UsePackage", 3);
 405
 406    if (unlikely(NewFileVer(Ver,List) == false))
 407       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 408                            Pkg.Name(), "NewFileVer", 2);
 409
 410    pkgCache::GrpIterator Grp = Pkg.Group();
 411    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 412
 413    /* If it is the first version of this package we need to add implicit
 414       Multi-Arch dependencies to all other package versions in the group now -
 415       otherwise we just add them for this new version */
 416    if (Pkg.VersionList()->NextVer == 0)
 417    {
 418       pkgCache::PkgIterator P = Grp.PackageList();
 419       Dynamic<pkgCache::PkgIterator> DynP(P);
 420       for (; P.end() != true; P = Grp.NextPkg(P))
 421       {
 422          if (P->ID == Pkg->ID)
 423             continue;
 424          pkgCache::VerIterator V = P.VersionList();
 425          Dynamic<pkgCache::VerIterator> DynV(V);
 426          for (; V.end() != true; ++V)
 427             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 428                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 429                                     Pkg.Name(), "AddImplicitDepends", 1);
 430       }
 431    }
 432    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 433       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 434                            Pkg.Name(), "AddImplicitDepends", 2);
 435
 436    // Read only a single record and return
 437    if (OutVer != 0)
 438    {
 439       *OutVer = Ver;
 440       return true;
 441    }
 442
 443    /* Record the Description(s) based on their master md5sum */
 444    MD5SumValue CurMd5 = List.Description_md5();
 445
 446    /* Before we add a new description we first search in the group for
 447       a version with a description of the same MD5 - if so we reuse this
 448       description group instead of creating our own for this version */
 449    for (pkgCache::PkgIterator P = Grp.PackageList();
 450         P.end() == false; P = Grp.NextPkg(P))
 451    {
 452       for (pkgCache::VerIterator V = P.VersionList();
 453            V.end() == false; ++V)
 454       {
 455          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 456             continue;
 457          Ver->DescriptionList = V->DescriptionList;
 458       }
 459    }
 460
 461    // We haven't found reusable descriptions, so add the first description(s)
 462    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 463    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 464    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 465       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 466          return false;
 467    return true;
 468 }
 469                                                                         /*}}}*/
 470 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 471 {
 472    pkgCache::DescIterator Desc;
 473    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 474
 475    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 476    if (unlikely(descindex == 0))
 477       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 478             Ver.ParentPkg().Name(), "NewDescription", 1);
 479
 480    md5idx = Desc->md5sum;
 481    Desc->ParentPkg = Ver.ParentPkg().Index();
 482
 483    // we add at the end, so that the start is constant as we need
 484    // that to be able to efficiently share these lists
 485    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 486    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 487    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 488    *LastNextDesc = descindex;
 489
 490    if (NewFileDesc(Desc,List) == false)
 491       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 492             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 493
 494    return true;
 495 }
 496                                                                         /*}}}*/
 497                                                                         /*}}}*/
 498 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 499 // ---------------------------------------------------------------------
 500 /* This creates a new group structure and adds it to the hash table */
 501 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 502 {
 503    Grp = Cache.FindGrp(Name);
 504    if (Grp.end() == false)
 505       return true;
 506
 507    // Get a structure
 508    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 509    if (unlikely(Group == 0))
 510       return false;
 511
 512    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 513    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 514    if (unlikely(idxName == 0))
 515       return false;
 516    Grp->Name = idxName;
 517
 518    // Insert it into the hash table
 519    unsigned long const Hash = Cache.Hash(Name);
 520    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 521    while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 522       insertAt = &(Cache.GrpP + *insertAt)->Next;
 523    Grp->Next = *insertAt;
 524    *insertAt = Group;
 525
 526    Grp->ID = Cache.HeaderP->GroupCount++;
 527    return true;
 528 }
 529                                                                         /*}}}*/
 530 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 531 // ---------------------------------------------------------------------
 532 /* This creates a new package structure and adds it to the hash table */
 533 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 534                                         StringView Arch) {
 535    pkgCache::GrpIterator Grp;
 536    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 537    if (unlikely(NewGroup(Grp, Name) == false))
 538       return false;
 539
 540    Pkg = Grp.FindPkg(Arch);
 541       if (Pkg.end() == false)
 542          return true;
 543
 544    // Get a structure
 545    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 546    if (unlikely(Package == 0))
 547       return false;
 548    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 549
 550    // Set the name, arch and the ID
 551    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 552    Pkg->Group = Grp.Index();
 553    // all is mapped to the native architecture
 554    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 555    if (unlikely(idxArch == 0))
 556       return false;
 557    Pkg->Arch = idxArch;
 558    Pkg->ID = Cache.HeaderP->PackageCount++;
 559
 560    // Insert the package into our package list
 561    if (Grp->FirstPackage == 0) // the group is new
 562    {
 563       Grp->FirstPackage = Package;
 564       // Insert it into the hash table
 565       map_id_t const Hash = Cache.Hash(Name);
 566       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 567       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 568          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 569       Pkg->NextPackage = *insertAt;
 570       *insertAt = Package;
 571    }
 572    else // Group the Packages together
 573    {
 574       // but first get implicit provides done
 575       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 576       {
 577          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 578          if (M.end() == false) {
 579             pkgCache::PrvIterator Prv;
 580             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 581             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 582             {
 583                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 584                   continue;
 585                pkgCache::VerIterator Ver = Prv.OwnerVer();
 586                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 587                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 588                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 589                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 590                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 591                      return false;
 592             }
 593          }
 594
 595
 596          pkgCache::PkgIterator P;
 597          pkgCache::VerIterator Ver;
 598          Dynamic<pkgCache::PkgIterator> DynP(P);
 599          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 600
 601          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 602             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 603                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 604                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 605                      return false;
 606       }
 607       // and negative dependencies, don't forget negative dependencies
 608       {
 609          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 610          if (M.end() == false) {
 611             pkgCache::DepIterator Dep;
 612             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 613             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 614             {
 615                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 616                   continue;
 617                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 618                      Dep->Type != pkgCache::Dep::Replaces)
 619                   continue;
 620                pkgCache::VerIterator Ver = Dep.ParentVer();
 621                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 622                map_pointer_t * unused = NULL;
 623                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 624                   return false;
 625             }
 626          }
 627       }
 628
 629       // this package is the new last package
 630       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 631       Pkg->NextPackage = LastPkg->NextPackage;
 632       LastPkg->NextPackage = Package;
 633    }
 634    Grp->LastPackage = Package;
 635
 636    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 637    if (Arch == "any")
 638    {
 639       size_t const found = Name.find(':');
 640       StringView const NameA = Name.substr(0, found);
 641       StringView const ArchA = Name.substr(found + 1);
 642       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 643       if (PkgA.end() == false)
 644       {
 645          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 646          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 647          for (; Prv.end() == false; ++Prv)
 648          {
 649             if (Prv.IsMultiArchImplicit())
 650                continue;
 651             pkgCache::VerIterator V = Prv.OwnerVer();
 652             if (ArchA != V.ParentPkg().Arch())
 653                continue;
 654             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 655                return false;
 656          }
 657          pkgCache::VerIterator V = PkgA.VersionList();
 658          Dynamic<pkgCache::VerIterator> DynV(V);
 659          for (; V.end() == false; ++V)
 660          {
 661             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 662                return false;
 663          }
 664       }
 665    }
 666    return true;
 667 }
 668                                                                         /*}}}*/
 669 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 670 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 671                                            pkgCache::PkgIterator &P,
 672                                            pkgCache::VerIterator &V)
 673 {
 674    // copy P.Arch() into a string here as a cache remap
 675    // in NewDepends() later may alter the pointer location
 676    string Arch = P.Arch() == NULL ? "" : P.Arch();
 677    map_pointer_t *OldDepLast = NULL;
 678    /* MultiArch handling introduces a lot of implicit Dependencies:
 679       - MultiArch: same → Co-Installable if they have the same version
 680       - All others conflict with all other group members */
 681    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 682    pkgCache::PkgIterator D = G.PackageList();
 683    Dynamic<pkgCache::PkgIterator> DynD(D);
 684    map_stringitem_t const VerStrIdx = V->VerStr;
 685    for (; D.end() != true; D = G.NextPkg(D))
 686    {
 687       if (Arch == D.Arch() || D->VersionList == 0)
 688          continue;
 689       /* We allow only one installed arch at the time
 690          per group, therefore each group member conflicts
 691          with all other group members */
 692       if (coInstall == true)
 693       {
 694          // Replaces: ${self}:other ( << ${binary:Version})
 695          NewDepends(D, V, VerStrIdx,
 696                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 697                     OldDepLast);
 698          // Breaks: ${self}:other (!= ${binary:Version})
 699          NewDepends(D, V, VerStrIdx,
 700                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 701                     OldDepLast);
 702       } else {
 703          // Conflicts: ${self}:other
 704          NewDepends(D, V, 0,
 705                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 706                     OldDepLast);
 707       }
 708    }
 709    return true;
 710 }
 711 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 712                                            pkgCache::PkgIterator &D)
 713 {
 714    /* MultiArch handling introduces a lot of implicit Dependencies:
 715       - MultiArch: same → Co-Installable if they have the same version
 716       - All others conflict with all other group members */
 717    map_pointer_t *OldDepLast = NULL;
 718    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 719    if (coInstall == true)
 720    {
 721       map_stringitem_t const VerStrIdx = V->VerStr;
 722       // Replaces: ${self}:other ( << ${binary:Version})
 723       NewDepends(D, V, VerStrIdx,
 724                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 725                  OldDepLast);
 726       // Breaks: ${self}:other (!= ${binary:Version})
 727       NewDepends(D, V, VerStrIdx,
 728                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 729                  OldDepLast);
 730    } else {
 731       // Conflicts: ${self}:other
 732       NewDepends(D, V, 0,
 733                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 734                  OldDepLast);
 735    }
 736    return true;
 737 }
 738
 739                                                                         /*}}}*/
 740 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 741 // ---------------------------------------------------------------------
 742 /* */
 743 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 744                                    ListParser &List)
 745 {
 746    if (CurrentFile == 0)
 747       return true;
 748
 749    // Get a structure
 750    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 751    if (VerFile == 0)
 752       return false;
 753
 754    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 755    VF->File = CurrentFile - Cache.PkgFileP;
 756
 757    // Link it to the end of the list
 758    map_pointer_t *Last = &Ver->FileList;
 759    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 760       Last = &V->NextFile;
 761    VF->NextFile = *Last;
 762    *Last = VF.Index();
 763
 764    VF->Offset = List.Offset();
 765    VF->Size = List.Size();
 766    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 767       Cache.HeaderP->MaxVerFileSize = VF->Size;
 768    Cache.HeaderP->VerFileCount++;
 769
 770    return true;
 771 }
 772                                                                         /*}}}*/
 773 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 774 // ---------------------------------------------------------------------
 775 /* This puts a version structure in the linked list */
 776 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 777                                             const string &VerStr,
 778                                             map_pointer_t const ParentPkg,
 779                                             unsigned short const Hash,
 780                                             map_pointer_t const Next)
 781 {
 782    // Get a structure
 783    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 784    if (Version == 0)
 785       return 0;
 786
 787    // Fill it in
 788    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 789    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 790    Ver->NextVer = Next;
 791    Ver->ParentPkg = ParentPkg;
 792    Ver->Hash = Hash;
 793    Ver->ID = Cache.HeaderP->VersionCount++;
 794
 795    // try to find the version string in the group for reuse
 796    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 797    pkgCache::GrpIterator Grp = Pkg.Group();
 798    if (Pkg.end() == false && Grp.end() == false)
 799    {
 800       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 801       {
 802          if (Pkg == P)
 803             continue;
 804          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 805          {
 806             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 807             if (cmp == 0)
 808             {
 809                Ver->VerStr = V->VerStr;
 810                return Version;
 811             }
 812             else if (cmp < 0)
 813                break;
 814          }
 815       }
 816    }
 817    // haven't found the version string, so create
 818    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 819    if (unlikely(idxVerStr == 0))
 820       return 0;
 821    Ver->VerStr = idxVerStr;
 822    return Version;
 823 }
 824                                                                         /*}}}*/
 825 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 826 // ---------------------------------------------------------------------
 827 /* */
 828 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 829                                    ListParser &List)
 830 {
 831    if (CurrentFile == 0)
 832       return true;
 833
 834    // Get a structure
 835    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 836    if (DescFile == 0)
 837       return false;
 838
 839    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 840    DF->File = CurrentFile - Cache.PkgFileP;
 841
 842    // Link it to the end of the list
 843    map_pointer_t *Last = &Desc->FileList;
 844    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 845       Last = &D->NextFile;
 846
 847    DF->NextFile = *Last;
 848    *Last = DF.Index();
 849
 850    DF->Offset = List.Offset();
 851    DF->Size = List.Size();
 852    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 853       Cache.HeaderP->MaxDescFileSize = DF->Size;
 854    Cache.HeaderP->DescFileCount++;
 855
 856    return true;
 857 }
 858                                                                         /*}}}*/
 859 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 860 // ---------------------------------------------------------------------
 861 /* This puts a description structure in the linked list */
 862 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 863                                             const string &Lang,
 864                                             const MD5SumValue &md5sum,
 865                                             map_stringitem_t const idxmd5str)
 866 {
 867    // Get a structure
 868    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 869    if (Description == 0)
 870       return 0;
 871
 872    // Fill it in
 873    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 874    Desc->ID = Cache.HeaderP->DescriptionCount++;
 875    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 876    if (unlikely(idxlanguage_code == 0))
 877       return 0;
 878    Desc->language_code = idxlanguage_code;
 879
 880    if (idxmd5str != 0)
 881       Desc->md5sum = idxmd5str;
 882    else
 883    {
 884       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 885       if (unlikely(idxmd5sum == 0))
 886          return 0;
 887       Desc->md5sum = idxmd5sum;
 888    }
 889
 890    return Description;
 891 }
 892                                                                         /*}}}*/
 893 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 894 // ---------------------------------------------------------------------
 895 /* This creates a dependency element in the tree. It is linked to the
 896    version and to the package that it is pointing to. */
 897 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 898                                    pkgCache::VerIterator &Ver,
 899                                    map_pointer_t const Version,
 900                                    uint8_t const Op,
 901                                    uint8_t const Type,
 902                                    map_pointer_t* &OldDepLast)
 903 {
 904    void const * const oldMap = Map.Data();
 905    // Get a structure
 906    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 907    if (unlikely(Dependency == 0))
 908       return false;
 909
 910    bool isDuplicate = false;
 911    map_pointer_t DependencyData = 0;
 912    map_pointer_t PreviousData = 0;
 913    if (Pkg->RevDepends != 0)
 914    {
 915       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 916       DependencyData = L->DependencyData;
 917       do {
 918          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 919          if (Version > D->Version)
 920             break;
 921          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 922          {
 923             isDuplicate = true;
 924             break;
 925          }
 926          PreviousData = DependencyData;
 927          DependencyData = D->NextData;
 928       } while (DependencyData != 0);
 929    }
 930
 931    if (isDuplicate == false)
 932    {
 933       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 934       if (unlikely(DependencyData == 0))
 935         return false;
 936    }
 937
 938    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 939    Link->ParentVer = Ver.Index();
 940    Link->DependencyData = DependencyData;
 941    Link->ID = Cache.HeaderP->DependsCount++;
 942
 943    pkgCache::DepIterator Dep(Cache, Link);
 944    if (isDuplicate == false)
 945    {
 946       Dep->Type = Type;
 947       Dep->CompareOp = Op;
 948       Dep->Version = Version;
 949       Dep->Package = Pkg.Index();
 950       ++Cache.HeaderP->DependsDataCount;
 951       if (PreviousData != 0)
 952       {
 953          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 954          Dep->NextData = D->NextData;
 955          D->NextData = DependencyData;
 956       }
 957       else if (Pkg->RevDepends != 0)
 958       {
 959          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 960          Dep->NextData = D->DependencyData;
 961       }
 962    }
 963
 964    if (isDuplicate == true || PreviousData != 0)
 965    {
 966       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 967       Link->NextRevDepends = L->NextRevDepends;
 968       L->NextRevDepends = Dependency;
 969    }
 970    else
 971    {
 972       Link->NextRevDepends = Pkg->RevDepends;
 973       Pkg->RevDepends = Dependency;
 974    }
 975
 976
 977    // Do we know where to link the Dependency to?
 978    if (OldDepLast == NULL)
 979    {
 980       OldDepLast = &Ver->DependsList;
 981       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 982          OldDepLast = &D->NextDepends;
 983    } else if (oldMap != Map.Data())
 984       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 985
 986    Dep->NextDepends = *OldDepLast;
 987    *OldDepLast = Dependency;
 988    OldDepLast = &Dep->NextDepends;
 989    return true;
 990 }
 991                                                                         /*}}}*/
 992 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 993 // ---------------------------------------------------------------------
 994 /* This creates a Group and the Package to link this dependency to if
 995    needed and handles also the caching of the old endpoint */
 996 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
 997                                                StringView PackageName,
 998                                                StringView Arch,
 999                                                StringView Version,
1000                                                uint8_t const Op,
1001                                                uint8_t const Type)
1002 {
1003    pkgCache::GrpIterator Grp;
1004    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1005    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1006       return false;
1007
1008    map_stringitem_t idxVersion = 0;
1009    if (Version.empty() == false)
1010    {
1011       int const CmpOp = Op & 0x0F;
1012       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1013       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1014          idxVersion = Ver->VerStr;
1015
1016       if (idxVersion == 0)
1017       {
1018          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1019          if (unlikely(idxVersion == 0))
1020             return false;
1021       }
1022    }
1023
1024    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1025          Type == pkgCache::Dep::Conflicts ||
1026          Type == pkgCache::Dep::Replaces);
1027
1028    pkgCache::PkgIterator Pkg;
1029    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1030    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1031    {
1032       // Locate the target package
1033       Pkg = Grp.FindPkg(Arch);
1034       if (Pkg.end() == true) {
1035          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1036             return false;
1037       }
1038
1039       /* Caching the old end point speeds up generation substantially */
1040       if (OldDepVer != Ver) {
1041          OldDepLast = NULL;
1042          OldDepVer = Ver;
1043       }
1044
1045       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1046    }
1047    else
1048    {
1049       /* Caching the old end point speeds up generation substantially */
1050       if (OldDepVer != Ver) {
1051          OldDepLast = NULL;
1052          OldDepVer = Ver;
1053       }
1054
1055       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1056       {
1057          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1058                return false;
1059       }
1060    }
1061    return true;
1062 }
1063                                                                         /*}}}*/
1064 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1065 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1066                                                 StringView PkgName,
1067                                                 StringView PkgArch,
1068                                                 StringView Version,
1069                                                 uint8_t const Flags)
1070 {
1071    pkgCache const &Cache = Owner->Cache;
1072
1073    // We do not add self referencing provides
1074    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1075         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1076          (Version.empty() || Version == Ver.VerStr()))
1077       return true;
1078
1079    // Locate the target package
1080    pkgCache::PkgIterator Pkg;
1081    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1082    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1083       return false;
1084
1085    map_stringitem_t idxProvideVersion = 0;
1086    if (Version.empty() == false) {
1087       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1088       if (unlikely(idxProvideVersion == 0))
1089          return false;
1090    }
1091    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1092 }
1093 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1094                                     pkgCache::PkgIterator &Pkg,
1095                                     map_pointer_t const ProvideVersion,
1096                                     uint8_t const Flags)
1097 {
1098    // Get a structure
1099    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1100    if (unlikely(Provides == 0))
1101       return false;
1102    ++Cache.HeaderP->ProvidesCount;
1103
1104    // Fill it in
1105    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1106    Prv->Version = Ver.Index();
1107    Prv->ProvideVersion = ProvideVersion;
1108    Prv->Flags = Flags;
1109    Prv->NextPkgProv = Ver->ProvidesList;
1110    Ver->ProvidesList = Prv.Index();
1111
1112    // Link it to the package
1113    Prv->ParentPkg = Pkg.Index();
1114    Prv->NextProvides = Pkg->ProvidesList;
1115    Pkg->ProvidesList = Prv.Index();
1116    return true;
1117 }
1118                                                                         /*}}}*/
1119 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1120 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1121                                 StringView Version, uint8_t const Flags) {
1122    pkgCache &Cache = Owner->Cache;
1123    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1124    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1125
1126    if (Grp.end() == true)
1127       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1128    else
1129    {
1130       map_stringitem_t idxProvideVersion = 0;
1131       if (Version.empty() == false) {
1132          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1133          if (unlikely(idxProvideVersion == 0))
1134             return false;
1135       }
1136
1137       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1138       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1139       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1140       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1141       pkgCache::PkgIterator Pkg;
1142       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1143       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1144       {
1145          if (isImplicit && OwnerPkg == Pkg)
1146             continue;
1147          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1148             continue;
1149          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1150             return false;
1151       }
1152    }
1153    return true;
1154 }
1155                                                                         /*}}}*/
1156 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1157       pkgCache::VerIterator const &Ver)
1158 {
1159    return Hash == Ver->Hash;
1160 }
1161                                                                         /*}}}*/
1162 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1163 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1164                                    unsigned long Flags)
1165 {
1166    if (File.empty() && Site.empty())
1167    {
1168       CurrentRlsFile = NULL;
1169       return true;
1170    }
1171
1172    // Get some space for the structure
1173    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1174    if (unlikely(idxFile == 0))
1175       return false;
1176    CurrentRlsFile = Cache.RlsFileP + idxFile;
1177
1178    // Fill it in
1179    map_stringitem_t const idxFileName = WriteStringInMap(File);
1180    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1181    if (unlikely(idxFileName == 0 || idxSite == 0))
1182       return false;
1183    CurrentRlsFile->FileName = idxFileName;
1184    CurrentRlsFile->Site = idxSite;
1185    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1186    CurrentRlsFile->Flags = Flags;
1187    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1188    RlsFileName = File;
1189    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1190    Cache.HeaderP->ReleaseFileCount++;
1191
1192    return true;
1193 }
1194                                                                         /*}}}*/
1195 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1196 // ---------------------------------------------------------------------
1197 /* This is used to select which file is to be associated with all newly
1198    added versions. The caller is responsible for setting the IMS fields. */
1199 bool pkgCacheGenerator::SelectFile(std::string const &File,
1200                                    pkgIndexFile const &Index,
1201                                    std::string const &Architecture,
1202                                    std::string const &Component,
1203                                    unsigned long const Flags)
1204 {
1205    // Get some space for the structure
1206    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1207    if (unlikely(idxFile == 0))
1208       return false;
1209    CurrentFile = Cache.PkgFileP + idxFile;
1210
1211    // Fill it in
1212    map_stringitem_t const idxFileName = WriteStringInMap(File);
1213    if (unlikely(idxFileName == 0))
1214       return false;
1215    CurrentFile->FileName = idxFileName;
1216    CurrentFile->NextFile = Cache.HeaderP->FileList;
1217    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1218    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1219    if (unlikely(idxIndexType == 0))
1220       return false;
1221    CurrentFile->IndexType = idxIndexType;
1222    if (Architecture.empty())
1223       CurrentFile->Architecture = 0;
1224    else
1225    {
1226       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1227       if (unlikely(arch == 0))
1228          return false;
1229       CurrentFile->Architecture = arch;
1230    }
1231    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1232    if (unlikely(component == 0))
1233       return false;
1234    CurrentFile->Component = component;
1235    CurrentFile->Flags = Flags;
1236    if (CurrentRlsFile != NULL)
1237       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1238    else
1239       CurrentFile->Release = 0;
1240    PkgFileName = File;
1241    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1242    Cache.HeaderP->PackageFileCount++;
1243
1244    if (Progress != 0)
1245       Progress->SubProgress(Index.Size());
1246    return true;
1247 }
1248                                                                         /*}}}*/
1249 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1250 // ---------------------------------------------------------------------
1251 /* This is used to create handles to strings. Given the same text it
1252    always returns the same number */
1253 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1254                                                  unsigned int Size)
1255 {
1256    auto strings = &strMixed;
1257    switch(type) {
1258       case MIXED: strings = &strMixed; break;
1259       case PKGNAME: strings = &strPkgNames; break;
1260       case VERSIONNUMBER: strings = &strVersions; break;
1261       case SECTION: strings = &strSections; break;
1262       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1263    }
1264
1265    auto const item = strings->find({S, Size, nullptr, 0});
1266    if (item != strings->end())
1267       return item->item;
1268
1269    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1270    strings->insert({nullptr, Size, this, idxString});
1271    return idxString;
1272 }
1273                                                                         /*}}}*/
1274 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1275 // ---------------------------------------------------------------------
1276 /* This just verifies that each file in the list of index files exists,
1277    has matching attributes with the cache and the cache does not have
1278    any extra files. */
1279 class APT_HIDDEN ScopedErrorRevert {
1280 public:
1281    ScopedErrorRevert() { _error->PushToStack(); }
1282    ~ScopedErrorRevert() { _error->RevertToStack(); }
1283 };
1284 static bool CheckValidity(const string &CacheFile,
1285                           pkgSourceList &List,
1286                           FileIterator const Start,
1287                           FileIterator const End,
1288                           MMap **OutMap = 0,
1289                           pkgCache **OutCache = 0)
1290 {
1291    ScopedErrorRevert ser;
1292    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1293    // No file, certainly invalid
1294    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1295    {
1296       if (Debug == true)
1297          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1298       return false;
1299    }
1300
1301    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1302    {
1303       if (Debug == true)
1304          std::clog << "sources.list is newer than the cache" << std::endl;
1305       return false;
1306    }
1307
1308    // Map it
1309    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1310    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1311    if (unlikely(Map->validData()) == false)
1312       return false;
1313    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1314    pkgCache &Cache = *CacheP.get();
1315    if (_error->PendingError() || Map->Size() == 0)
1316    {
1317       if (Debug == true)
1318          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1319       return false;
1320    }
1321
1322    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1323    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1324    std::vector<pkgIndexFile *> Files;
1325    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1326    {
1327       if (Debug == true)
1328          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1329       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1330       if (RlsFile.end() == true)
1331       {
1332          if (Debug == true)
1333             std::clog << "FindInCache returned end-Pointer" << std::endl;
1334          return false;
1335       }
1336
1337       RlsVisited[RlsFile->ID] = true;
1338       if (Debug == true)
1339          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1340
1341       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1342       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1343             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1344    }
1345    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1346       if (RlsVisited[I] == false)
1347       {
1348          if (Debug == true)
1349             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1350          return false;
1351       }
1352
1353    std::copy(Start, End, std::back_inserter(Files));
1354
1355    /* Now we check every index file, see if it is in the cache,
1356       verify the IMS data and check that it is on the disk too.. */
1357    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1358    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1359    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1360    {
1361       if (Debug == true)
1362          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1363       if ((*PkgFile)->Exists() == false)
1364       {
1365          if (Debug == true)
1366             std::clog << "file doesn't exist" << std::endl;
1367          continue;
1368       }
1369
1370       // FindInCache is also expected to do an IMS check.
1371       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1372       if (File.end() == true)
1373       {
1374          if (Debug == true)
1375             std::clog << "FindInCache returned end-Pointer" << std::endl;
1376          return false;
1377       }
1378
1379       Visited[File->ID] = true;
1380       if (Debug == true)
1381          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1382    }
1383
1384    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1385       if (Visited[I] == false)
1386       {
1387          if (Debug == true)
1388             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1389          return false;
1390       }
1391
1392    if (_error->PendingError() == true)
1393    {
1394       if (Debug == true)
1395       {
1396          std::clog << "Validity failed because of pending errors:" << std::endl;
1397          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1398       }
1399       return false;
1400    }
1401
1402    if (OutMap != 0)
1403       *OutMap = Map.release();
1404    if (OutCache != 0)
1405       *OutCache = CacheP.release();
1406    return true;
1407 }
1408                                                                         /*}}}*/
1409 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1410 // ---------------------------------------------------------------------
1411 /* Size is kind of an abstract notion that is only used for the progress
1412    meter */
1413 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1414 {
1415    map_filesize_t TotalSize = 0;
1416    if (List !=  NULL)
1417    {
1418       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1419       {
1420          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1421          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1422             if ((*j)->HasPackages() == true)
1423                TotalSize += (*j)->Size();
1424       }
1425    }
1426
1427    for (; Start < End; ++Start)
1428    {
1429       if ((*Start)->HasPackages() == false)
1430          continue;
1431       TotalSize += (*Start)->Size();
1432    }
1433    return TotalSize;
1434 }
1435                                                                         /*}}}*/
1436 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1437 static bool BuildCache(pkgCacheGenerator &Gen,
1438                        OpProgress * const Progress,
1439                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1440                        pkgSourceList const * const List,
1441                        FileIterator const Start, FileIterator const End)
1442 {
1443    bool mergeFailure = false;
1444
1445    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1446       if (I->HasPackages() == false || mergeFailure)
1447          return;
1448
1449       if (I->Exists() == false)
1450          return;
1451
1452       if (I->FindInCache(Gen.GetCache()).end() == false)
1453       {
1454          _error->Warning("Duplicate sources.list entry %s",
1455                I->Describe().c_str());
1456          return;
1457       }
1458
1459       map_filesize_t const Size = I->Size();
1460       if (Progress != NULL)
1461          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1462       CurrentSize += Size;
1463
1464       if (I->Merge(Gen,Progress) == false)
1465          mergeFailure = true;
1466    };
1467
1468    if (List !=  NULL)
1469    {
1470       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1471       {
1472          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1473          {
1474             _error->Warning("Duplicate sources.list entry %s",
1475                   (*i)->Describe().c_str());
1476             continue;
1477          }
1478
1479          if ((*i)->Merge(Gen, Progress) == false)
1480             return false;
1481
1482          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1483          if (Indexes != NULL)
1484             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1485          if (mergeFailure)
1486             return false;
1487       }
1488    }
1489
1490    if (Start != End)
1491    {
1492       Gen.SelectReleaseFile("", "");
1493       std::for_each(Start, End, indexFileMerge);
1494       if (mergeFailure)
1495          return false;
1496    }
1497    return true;
1498 }
1499                                                                         /*}}}*/
1500 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1501 // ---------------------------------------------------------------------
1502 /* This makes sure that the status cache (the cache that has all
1503    index files from the sources list and all local ones) is ready
1504    to be mmaped. If OutMap is not zero then a MMap object representing
1505    the cache will be stored there. This is pretty much mandetory if you
1506    are using AllowMem. AllowMem lets the function be run as non-root
1507    where it builds the cache 'fast' into a memory buffer. */
1508 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1509 {
1510    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1511    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1512    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1513    Flags |= MMap::Moveable;
1514    if (_config->FindB("APT::Cache-Fallback", false) == true)
1515       Flags |= MMap::Fallback;
1516    if (CacheF != NULL)
1517       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1518    else
1519       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1520 }
1521 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1522       std::string const &FileName)
1523 {
1524    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1525    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1526       return false;
1527
1528    fchmod(SCacheF.Fd(),0644);
1529
1530    // Write out the main data
1531    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1532       return _error->Error(_("IO Error saving source cache"));
1533
1534    // Write out the proper header
1535    Gen->GetCache().HeaderP->Dirty = false;
1536    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1537    if (SCacheF.Seek(0) == false ||
1538          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1539       return _error->Error(_("IO Error saving source cache"));
1540    Gen->GetCache().HeaderP->Dirty = true;
1541    return true;
1542 }
1543 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1544       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1545 {
1546    Map.reset(CreateDynamicMMap(NULL, 0));
1547    if (unlikely(Map->validData()) == false)
1548       return false;
1549    FileFd CacheF(FileName, FileFd::ReadOnly);
1550    if (CacheF.IsOpen() == false || CacheF.Failed())
1551       return false;
1552    _error->PushToStack();
1553    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1554    bool const newError = _error->PendingError();
1555    _error->MergeWithStack();
1556    if (alloc == 0 && newError)
1557       return false;
1558    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1559       return false;
1560    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1561    return true;
1562 }
1563 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1564                         MMap **OutMap, bool AllowMem)
1565    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1566 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1567                         MMap **OutMap,bool)
1568 {
1569    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1570 }
1571 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1572                         MMap **OutMap,pkgCache **OutCache, bool)
1573 {
1574    // FIXME: deprecate the ignored AllowMem parameter
1575    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1576
1577    std::vector<pkgIndexFile *> Files;
1578    if (_system->AddStatusFiles(Files) == false)
1579       return false;
1580
1581    // Decide if we can write to the files..
1582    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1583    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1584
1585    // ensure the cache directory exists
1586    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1587    {
1588       string dir = _config->FindDir("Dir::Cache");
1589       size_t const len = dir.size();
1590       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1591          dir = dir.substr(0, len - 5);
1592       if (CacheFile.empty() == false)
1593          CreateDirectory(dir, flNotFile(CacheFile));
1594       if (SrcCacheFile.empty() == false)
1595          CreateDirectory(dir, flNotFile(SrcCacheFile));
1596    }
1597
1598    if (Progress != NULL)
1599       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1600
1601    bool pkgcache_fine = false;
1602    bool srcpkgcache_fine = false;
1603    bool volatile_fine = List.GetVolatileFiles().empty();
1604
1605    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1606                      volatile_fine ? OutCache : NULL) == true)
1607    {
1608       if (Debug == true)
1609          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1610       pkgcache_fine = true;
1611       srcpkgcache_fine = true;
1612    }
1613    if (pkgcache_fine == false)
1614    {
1615       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1616       {
1617          if (Debug == true)
1618             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1619          srcpkgcache_fine = true;
1620       }
1621    }
1622
1623    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1624    {
1625       if (Progress != NULL)
1626          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1627       return true;
1628    }
1629
1630    bool Writeable = false;
1631    if (srcpkgcache_fine == false || pkgcache_fine == false)
1632    {
1633       if (CacheFile.empty() == false)
1634          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1635       else if (SrcCacheFile.empty() == false)
1636          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1637
1638       if (Debug == true)
1639          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1640    }
1641
1642    // At this point we know we need to construct something, so get storage ready
1643    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1644    if (unlikely(Map->validData()) == false)
1645       return false;
1646    if (Debug == true)
1647       std::clog << "Open memory Map (not filebased)" << std::endl;
1648
1649    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1650    map_filesize_t CurrentSize = 0;
1651    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1652    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1653    if (srcpkgcache_fine == true && pkgcache_fine == false)
1654    {
1655       if (Debug == true)
1656          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1657       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1658          return false;
1659       srcpkgcache_fine = true;
1660       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1661    }
1662    else if (srcpkgcache_fine == false)
1663    {
1664       if (Debug == true)
1665          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1666       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1667
1668       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1669       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1670                Files.end(),Files.end()) == false)
1671          return false;
1672
1673       if (Writeable == true && SrcCacheFile.empty() == false)
1674          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1675             return false;
1676    }
1677
1678    if (pkgcache_fine == false)
1679    {
1680       if (Debug == true)
1681          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1682       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1683                Files.begin(), Files.end()) == false)
1684          return false;
1685
1686       if (Writeable == true && CacheFile.empty() == false)
1687          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1688             return false;
1689    }
1690
1691    if (Debug == true)
1692       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1693
1694    if (volatile_fine == false)
1695    {
1696       if (Gen == nullptr)
1697       {
1698          if (Debug == true)
1699             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1700          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1701             return false;
1702       }
1703
1704       Files = List.GetVolatileFiles();
1705       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1706                Files.begin(), Files.end()) == false)
1707          return false;
1708    }
1709
1710    if (OutMap != nullptr)
1711       *OutMap = Map.release();
1712
1713    if (Debug == true)
1714       std::clog << "Everything is ready for shipping" << std::endl;
1715    return true;
1716 }
1717                                                                         /*}}}*/
1718 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1719 class APT_HIDDEN ScopedErrorMerge {
1720 public:
1721    ScopedErrorMerge() { _error->PushToStack(); }
1722    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1723 };
1724 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1725    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1726 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1727 {
1728    std::vector<pkgIndexFile *> Files;
1729    if (_system->AddStatusFiles(Files) == false)
1730       return false;
1731
1732    ScopedErrorMerge sem;
1733    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1734    if (unlikely(Map->validData()) == false)
1735       return false;
1736    map_filesize_t CurrentSize = 0;
1737    map_filesize_t TotalSize = 0;
1738    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1739
1740    // Build the status cache
1741    if (Progress != NULL)
1742       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1743    pkgCacheGenerator Gen(Map.get(),Progress);
1744    if (_error->PendingError() == true)
1745       return false;
1746    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1747                   Files.begin(), Files.end()) == false)
1748       return false;
1749
1750    if (_error->PendingError() == true)
1751       return false;
1752    *OutMap = Map.release();
1753
1754    return true;
1755 }
1756                                                                         /*}}}*/
1757 // IsDuplicateDescription                                               /*{{{*/
1758 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1759                             MD5SumValue const &CurMd5, std::string const &CurLang)
1760 {
1761    // Descriptions in the same link-list have all the same md5
1762    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1763       return false;
1764    for (; Desc.end() == false; ++Desc)
1765       if (Desc.LanguageCode() == CurLang)
1766          return true;
1767    return false;
1768 }
1769                                                                         /*}}}*/
1770
1771 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1772 pkgCacheListParser::~pkgCacheListParser() {}