apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52
  53 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* We set the dirty flag and make sure that is written to the disk */
  56 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  57                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  58                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  59 {
  60    if (Map.Size() == 0)
  61    {
  62       // Setup the map interface..
  63       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  64       _error->PushToStack();
  65       Map.RawAllocate(sizeof(pkgCache::Header));
  66       bool const newError = _error->PendingError();
  67       _error->MergeWithStack();
  68       if (newError)
  69          return;
  70
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72
  73       // Starting header
  74       *Cache.HeaderP = pkgCache::Header();
  75
  76       // make room for the hashtables for packages and groups
  77       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  78          return;
  79
  80       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  81       if (unlikely(idxVerSysName == 0))
  82          return;
  83       Cache.HeaderP->VerSysName = idxVerSysName;
  84       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  85       if (unlikely(idxArchitecture == 0))
  86          return;
  87       Cache.HeaderP->Architecture = idxArchitecture;
  88
  89       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  90       if (archs.size() > 1)
  91       {
  92          std::vector<std::string>::const_iterator a = archs.begin();
  93          std::string list = *a;
  94          for (++a; a != archs.end(); ++a)
  95             list.append(",").append(*a);
  96          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  97          if (unlikely(idxArchitectures == 0))
  98             return;
  99          Cache.HeaderP->SetArchitectures(idxArchitectures);
 100       }
 101       else
 102          Cache.HeaderP->SetArchitectures(idxArchitecture);
 103
 104       Cache.ReMap();
 105    }
 106    else
 107    {
 108       // Map directly from the existing file
 109       Cache.ReMap();
 110       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 111       if (Cache.VS != _system->VS)
 112       {
 113          _error->Error(_("Cache has an incompatible versioning system"));
 114          return;
 115       }
 116    }
 117
 118    Cache.HeaderP->Dirty = true;
 119    Map.Sync(0,sizeof(pkgCache::Header));
 120 }
 121                                                                         /*}}}*/
 122 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* We sync the data then unset the dirty flag in two steps so as to
 125    advoid a problem during a crash */
 126 pkgCacheGenerator::~pkgCacheGenerator()
 127 {
 128    if (_error->PendingError() == true || Map.validData() == false)
 129       return;
 130    if (Map.Sync() == false)
 131       return;
 132
 133    Cache.HeaderP->Dirty = false;
 134    Cache.HeaderP->CacheFileSize = Map.Size();
 135    Map.Sync(0,sizeof(pkgCache::Header));
 136 }
 137                                                                         /*}}}*/
 138 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 139    if (oldMap == newMap)
 140       return;
 141
 142    if (_config->FindB("Debug::pkgCacheGen", false))
 143       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 144
 145    Cache.ReMap(false);
 146
 147    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 148    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 149
 150    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 151         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 152       (*i)->ReMap(oldMap, newMap);
 153    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 154         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 155       (*i)->ReMap(oldMap, newMap);
 156    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174 }                                                                       /*}}}*/
 175 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 176 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 177                                         const unsigned long &Len) {
 178    void const * const oldMap = Map.Data();
 179    map_stringitem_t const index = Map.WriteString(String, Len);
 180    if (index != 0)
 181       ReMap(oldMap, Map.Data());
 182    return index;
 183 }
 184                                                                         /*}}}*/
 185 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 186 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 187    void const * const oldMap = Map.Data();
 188    map_stringitem_t const index = Map.WriteString(String);
 189    if (index != 0)
 190       ReMap(oldMap, Map.Data());
 191    return index;
 192 }
 193                                                                         /*}}}*/
 194 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 195    void const * const oldMap = Map.Data();
 196    map_pointer_t const index = Map.Allocate(size);
 197    if (index != 0)
 198       ReMap(oldMap, Map.Data());
 199    return index;
 200 }
 201                                                                         /*}}}*/
 202 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 203 // ---------------------------------------------------------------------
 204 /* This provides the generation of the entries in the cache. Each loop
 205    goes through a single package record from the underlying parse engine. */
 206 bool pkgCacheGenerator::MergeList(ListParser &List,
 207                                   pkgCache::VerIterator *OutVer)
 208 {
 209    List.Owner = this;
 210
 211    unsigned int Counter = 0;
 212    while (List.Step() == true)
 213    {
 214       string const PackageName = List.Package();
 215       if (PackageName.empty() == true)
 216          return false;
 217
 218       Counter++;
 219       if (Counter % 100 == 0 && Progress != 0)
 220          Progress->Progress(List.Offset());
 221
 222       string Arch = List.Architecture();
 223       string const Version = List.Version();
 224       if (Version.empty() == true && Arch.empty() == true)
 225       {
 226          // package descriptions
 227          if (MergeListGroup(List, PackageName) == false)
 228             return false;
 229          continue;
 230       }
 231
 232       // Get a pointer to the package structure
 233       pkgCache::PkgIterator Pkg;
 234       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 235       if (NewPackage(Pkg, PackageName, Arch) == false)
 236          // TRANSLATOR: The first placeholder is a package name,
 237          // the other two should be copied verbatim as they include debug info
 238          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 239                               PackageName.c_str(), "NewPackage", 1);
 240
 241
 242       if (Version.empty() == true)
 243       {
 244          if (MergeListPackage(List, Pkg) == false)
 245             return false;
 246       }
 247       else
 248       {
 249          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 250             return false;
 251       }
 252
 253       if (OutVer != 0)
 254          return true;
 255    }
 256
 257    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 258       return _error->Error(_("Wow, you exceeded the number of package "
 259                              "names this APT is capable of."));
 260    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 261       return _error->Error(_("Wow, you exceeded the number of versions "
 262                              "this APT is capable of."));
 263    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 264       return _error->Error(_("Wow, you exceeded the number of descriptions "
 265                              "this APT is capable of."));
 266    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of dependencies "
 268                              "this APT is capable of."));
 269
 270    return true;
 271 }
 272 // CacheGenerator::MergeListGroup                                       /*{{{*/
 273 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 274 {
 275    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 276    // a group has no data on it's own, only packages have it but these
 277    // stanzas like this come from Translation- files to add descriptions,
 278    // but without a version we don't need a description for it…
 279    if (Grp.end() == true)
 280       return true;
 281    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 282
 283    pkgCache::PkgIterator Pkg;
 284    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 285    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 286       if (MergeListPackage(List, Pkg) == false)
 287          return false;
 288
 289    return true;
 290 }
 291                                                                         /*}}}*/
 292 // CacheGenerator::MergeListPackage                                     /*{{{*/
 293 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 294 {
 295    // we first process the package, then the descriptions
 296    // (for deb this package processing is in fact a no-op)
 297    pkgCache::VerIterator Ver(Cache);
 298    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 299    if (List.UsePackage(Pkg, Ver) == false)
 300       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 301                            Pkg.Name(), "UsePackage", 1);
 302
 303    // Find the right version to write the description
 304    MD5SumValue CurMd5 = List.Description_md5();
 305    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 306       return true;
 307    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 308    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 309    {
 310       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 311
 312       // a version can only have one md5 describing it
 313       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 314          continue;
 315
 316       map_stringitem_t md5idx = VerDesc->md5sum;
 317       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 318       {
 319          // don't add a new description if we have one for the given
 320          // md5 && language
 321          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 322             continue;
 323
 324          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 325       }
 326
 327       // we can stop here as all "same" versions will share the description
 328       break;
 329    }
 330
 331    return true;
 332 }
 333                                                                         /*}}}*/
 334 // CacheGenerator::MergeListVersion                                     /*{{{*/
 335 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 336                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 337 {
 338    pkgCache::VerIterator Ver = Pkg.VersionList();
 339    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 340    map_pointer_t *LastVer = &Pkg->VersionList;
 341    void const * oldMap = Map.Data();
 342
 343    unsigned short const Hash = List.VersionHash();
 344    if (Ver.end() == false)
 345    {
 346       /* We know the list is sorted so we use that fact in the search.
 347          Insertion of new versions is done with correct sorting */
 348       int Res = 1;
 349       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 350       {
 351          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 352          // Version is higher as current version - insert here
 353          if (Res > 0)
 354             break;
 355          // Versionstrings are equal - is hash also equal?
 356          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 357             break;
 358          // proceed with the next till we have either the right
 359          // or we found another version (which will be lower)
 360       }
 361
 362       /* We already have a version for this item, record that we saw it */
 363       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 364       {
 365          if (List.UsePackage(Pkg,Ver) == false)
 366             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 367                                  Pkg.Name(), "UsePackage", 2);
 368
 369          if (NewFileVer(Ver,List) == false)
 370             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 371                                  Pkg.Name(), "NewFileVer", 1);
 372
 373          // Read only a single record and return
 374          if (OutVer != 0)
 375          {
 376             *OutVer = Ver;
 377             return true;
 378          }
 379
 380          return true;
 381       }
 382    }
 383
 384    // Add a new version
 385    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 386    if (unlikely(verindex == 0))
 387       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 388                            Pkg.Name(), "NewVersion", 1);
 389
 390    if (oldMap != Map.Data())
 391          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 392    *LastVer = verindex;
 393
 394    if (unlikely(List.NewVersion(Ver) == false))
 395       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 396                            Pkg.Name(), "NewVersion", 2);
 397
 398    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 399       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 400                            Pkg.Name(), "UsePackage", 3);
 401
 402    if (unlikely(NewFileVer(Ver,List) == false))
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "NewFileVer", 2);
 405
 406    pkgCache::GrpIterator Grp = Pkg.Group();
 407    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 408
 409    /* If it is the first version of this package we need to add implicit
 410       Multi-Arch dependencies to all other package versions in the group now -
 411       otherwise we just add them for this new version */
 412    if (Pkg.VersionList()->NextVer == 0)
 413    {
 414       pkgCache::PkgIterator P = Grp.PackageList();
 415       Dynamic<pkgCache::PkgIterator> DynP(P);
 416       for (; P.end() != true; P = Grp.NextPkg(P))
 417       {
 418          if (P->ID == Pkg->ID)
 419             continue;
 420          pkgCache::VerIterator V = P.VersionList();
 421          Dynamic<pkgCache::VerIterator> DynV(V);
 422          for (; V.end() != true; ++V)
 423             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 424                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                                     Pkg.Name(), "AddImplicitDepends", 1);
 426       }
 427    }
 428    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 429       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 430                            Pkg.Name(), "AddImplicitDepends", 2);
 431
 432    // Read only a single record and return
 433    if (OutVer != 0)
 434    {
 435       *OutVer = Ver;
 436       return true;
 437    }
 438
 439    /* Record the Description(s) based on their master md5sum */
 440    MD5SumValue CurMd5 = List.Description_md5();
 441    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 442       return true;
 443
 444    /* Before we add a new description we first search in the group for
 445       a version with a description of the same MD5 - if so we reuse this
 446       description group instead of creating our own for this version */
 447    for (pkgCache::PkgIterator P = Grp.PackageList();
 448         P.end() == false; P = Grp.NextPkg(P))
 449    {
 450       for (pkgCache::VerIterator V = P.VersionList();
 451            V.end() == false; ++V)
 452       {
 453          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 454             continue;
 455          Ver->DescriptionList = V->DescriptionList;
 456       }
 457    }
 458
 459    // We haven't found reusable descriptions, so add the first description(s)
 460    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 461    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 462    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 463       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 464          return false;
 465    return true;
 466 }
 467                                                                         /*}}}*/
 468 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 469 {
 470    pkgCache::DescIterator Desc;
 471    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 472
 473    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 474    if (unlikely(descindex == 0))
 475       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 476             Ver.ParentPkg().Name(), "NewDescription", 1);
 477
 478    md5idx = Desc->md5sum;
 479    Desc->ParentPkg = Ver.ParentPkg().Index();
 480
 481    // we add at the end, so that the start is constant as we need
 482    // that to be able to efficiently share these lists
 483    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 484    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 485    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 486    *LastNextDesc = descindex;
 487
 488    if (NewFileDesc(Desc,List) == false)
 489       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 490             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 491
 492    return true;
 493 }
 494                                                                         /*}}}*/
 495                                                                         /*}}}*/
 496 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* This creates a new group structure and adds it to the hash table */
 499 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 500 {
 501    Grp = Cache.FindGrp(Name);
 502    if (Grp.end() == false)
 503       return true;
 504
 505    // Get a structure
 506    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 507    if (unlikely(Group == 0))
 508       return false;
 509
 510    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 511    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 512    if (unlikely(idxName == 0))
 513       return false;
 514    Grp->Name = idxName;
 515
 516    // Insert it into the hash table
 517    unsigned long const Hash = Cache.Hash(Name);
 518    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 519    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 520       insertAt = &(Cache.GrpP + *insertAt)->Next;
 521    Grp->Next = *insertAt;
 522    *insertAt = Group;
 523
 524    Grp->ID = Cache.HeaderP->GroupCount++;
 525    return true;
 526 }
 527                                                                         /*}}}*/
 528 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 529 // ---------------------------------------------------------------------
 530 /* This creates a new package structure and adds it to the hash table */
 531 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 532                                         const string &Arch) {
 533    pkgCache::GrpIterator Grp;
 534    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 535    if (unlikely(NewGroup(Grp, Name) == false))
 536       return false;
 537
 538    Pkg = Grp.FindPkg(Arch);
 539       if (Pkg.end() == false)
 540          return true;
 541
 542    // Get a structure
 543    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 544    if (unlikely(Package == 0))
 545       return false;
 546    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 547
 548    // Set the name, arch and the ID
 549    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 550    Pkg->Group = Grp.Index();
 551    // all is mapped to the native architecture
 552    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 553    if (unlikely(idxArch == 0))
 554       return false;
 555    Pkg->Arch = idxArch;
 556    Pkg->ID = Cache.HeaderP->PackageCount++;
 557
 558    // Insert the package into our package list
 559    if (Grp->FirstPackage == 0) // the group is new
 560    {
 561       Grp->FirstPackage = Package;
 562       // Insert it into the hash table
 563       map_id_t const Hash = Cache.Hash(Name);
 564       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 565       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 566          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 567       Pkg->NextPackage = *insertAt;
 568       *insertAt = Package;
 569    }
 570    else // Group the Packages together
 571    {
 572       // but first get implicit provides done
 573       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 574       {
 575          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 576          if (M.end() == false) {
 577             pkgCache::PrvIterator Prv;
 578             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 579             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 580             {
 581                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 582                   continue;
 583                pkgCache::VerIterator Ver = Prv.OwnerVer();
 584                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 585                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 586                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 587                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 588                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 589                      return false;
 590             }
 591          }
 592
 593
 594          pkgCache::PkgIterator P;
 595          pkgCache::VerIterator Ver;
 596          Dynamic<pkgCache::PkgIterator> DynP(P);
 597          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 598
 599          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 600             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 601                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 602                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 603                      return false;
 604       }
 605       // and negative dependencies, don't forget negative dependencies
 606       {
 607          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 608          if (M.end() == false) {
 609             pkgCache::DepIterator Dep;
 610             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 611             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 612             {
 613                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 614                   continue;
 615                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 616                      Dep->Type != pkgCache::Dep::Replaces)
 617                   continue;
 618                pkgCache::VerIterator Ver = Dep.ParentVer();
 619                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 620                map_pointer_t * unused = NULL;
 621                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 622                   return false;
 623             }
 624          }
 625       }
 626
 627       // this package is the new last package
 628       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 629       Pkg->NextPackage = LastPkg->NextPackage;
 630       LastPkg->NextPackage = Package;
 631    }
 632    Grp->LastPackage = Package;
 633
 634    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 635    if (Arch == "any")
 636    {
 637       size_t const found = Name.find(':');
 638       std::string const NameA = Name.substr(0, found);
 639       std::string const ArchA = Name.substr(found + 1);
 640       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 641       if (PkgA.end() == false)
 642       {
 643          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 644          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 645          for (; Prv.end() == false; ++Prv)
 646          {
 647             if (Prv.IsMultiArchImplicit())
 648                continue;
 649             pkgCache::VerIterator V = Prv.OwnerVer();
 650             if (ArchA != V.ParentPkg().Arch())
 651                continue;
 652             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 653                return false;
 654          }
 655          pkgCache::VerIterator V = PkgA.VersionList();
 656          Dynamic<pkgCache::VerIterator> DynV(V);
 657          for (; V.end() == false; ++V)
 658          {
 659             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 660                return false;
 661          }
 662       }
 663    }
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 668 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 669                                            pkgCache::PkgIterator &P,
 670                                            pkgCache::VerIterator &V)
 671 {
 672    // copy P.Arch() into a string here as a cache remap
 673    // in NewDepends() later may alter the pointer location
 674    string Arch = P.Arch() == NULL ? "" : P.Arch();
 675    map_pointer_t *OldDepLast = NULL;
 676    /* MultiArch handling introduces a lot of implicit Dependencies:
 677       - MultiArch: same → Co-Installable if they have the same version
 678       - All others conflict with all other group members */
 679    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 680    pkgCache::PkgIterator D = G.PackageList();
 681    Dynamic<pkgCache::PkgIterator> DynD(D);
 682    map_stringitem_t const VerStrIdx = V->VerStr;
 683    for (; D.end() != true; D = G.NextPkg(D))
 684    {
 685       if (Arch == D.Arch() || D->VersionList == 0)
 686          continue;
 687       /* We allow only one installed arch at the time
 688          per group, therefore each group member conflicts
 689          with all other group members */
 690       if (coInstall == true)
 691       {
 692          // Replaces: ${self}:other ( << ${binary:Version})
 693          NewDepends(D, V, VerStrIdx,
 694                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 695                     OldDepLast);
 696          // Breaks: ${self}:other (!= ${binary:Version})
 697          NewDepends(D, V, VerStrIdx,
 698                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 699                     OldDepLast);
 700       } else {
 701          // Conflicts: ${self}:other
 702          NewDepends(D, V, 0,
 703                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 704                     OldDepLast);
 705       }
 706    }
 707    return true;
 708 }
 709 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 710                                            pkgCache::PkgIterator &D)
 711 {
 712    /* MultiArch handling introduces a lot of implicit Dependencies:
 713       - MultiArch: same → Co-Installable if they have the same version
 714       - All others conflict with all other group members */
 715    map_pointer_t *OldDepLast = NULL;
 716    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 717    if (coInstall == true)
 718    {
 719       map_stringitem_t const VerStrIdx = V->VerStr;
 720       // Replaces: ${self}:other ( << ${binary:Version})
 721       NewDepends(D, V, VerStrIdx,
 722                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 723                  OldDepLast);
 724       // Breaks: ${self}:other (!= ${binary:Version})
 725       NewDepends(D, V, VerStrIdx,
 726                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 727                  OldDepLast);
 728    } else {
 729       // Conflicts: ${self}:other
 730       NewDepends(D, V, 0,
 731                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 732                  OldDepLast);
 733    }
 734    return true;
 735 }
 736
 737                                                                         /*}}}*/
 738 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 739 // ---------------------------------------------------------------------
 740 /* */
 741 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 742                                    ListParser &List)
 743 {
 744    if (CurrentFile == 0)
 745       return true;
 746
 747    // Get a structure
 748    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 749    if (VerFile == 0)
 750       return false;
 751
 752    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 753    VF->File = CurrentFile - Cache.PkgFileP;
 754
 755    // Link it to the end of the list
 756    map_pointer_t *Last = &Ver->FileList;
 757    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 758       Last = &V->NextFile;
 759    VF->NextFile = *Last;
 760    *Last = VF.Index();
 761
 762    VF->Offset = List.Offset();
 763    VF->Size = List.Size();
 764    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 765       Cache.HeaderP->MaxVerFileSize = VF->Size;
 766    Cache.HeaderP->VerFileCount++;
 767
 768    return true;
 769 }
 770                                                                         /*}}}*/
 771 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 772 // ---------------------------------------------------------------------
 773 /* This puts a version structure in the linked list */
 774 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 775                                             const string &VerStr,
 776                                             map_pointer_t const ParentPkg,
 777                                             unsigned short const Hash,
 778                                             map_pointer_t const Next)
 779 {
 780    // Get a structure
 781    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 782    if (Version == 0)
 783       return 0;
 784
 785    // Fill it in
 786    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 787    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 788    Ver->NextVer = Next;
 789    Ver->ParentPkg = ParentPkg;
 790    Ver->Hash = Hash;
 791    Ver->ID = Cache.HeaderP->VersionCount++;
 792
 793    // try to find the version string in the group for reuse
 794    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 795    pkgCache::GrpIterator Grp = Pkg.Group();
 796    if (Pkg.end() == false && Grp.end() == false)
 797    {
 798       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 799       {
 800          if (Pkg == P)
 801             continue;
 802          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 803          {
 804             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 805             if (cmp == 0)
 806             {
 807                Ver->VerStr = V->VerStr;
 808                return Version;
 809             }
 810             else if (cmp < 0)
 811                break;
 812          }
 813       }
 814    }
 815    // haven't found the version string, so create
 816    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 817    if (unlikely(idxVerStr == 0))
 818       return 0;
 819    Ver->VerStr = idxVerStr;
 820    return Version;
 821 }
 822                                                                         /*}}}*/
 823 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 824 // ---------------------------------------------------------------------
 825 /* */
 826 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 827                                    ListParser &List)
 828 {
 829    if (CurrentFile == 0)
 830       return true;
 831
 832    // Get a structure
 833    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 834    if (DescFile == 0)
 835       return false;
 836
 837    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 838    DF->File = CurrentFile - Cache.PkgFileP;
 839
 840    // Link it to the end of the list
 841    map_pointer_t *Last = &Desc->FileList;
 842    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 843       Last = &D->NextFile;
 844
 845    DF->NextFile = *Last;
 846    *Last = DF.Index();
 847
 848    DF->Offset = List.Offset();
 849    DF->Size = List.Size();
 850    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 851       Cache.HeaderP->MaxDescFileSize = DF->Size;
 852    Cache.HeaderP->DescFileCount++;
 853
 854    return true;
 855 }
 856                                                                         /*}}}*/
 857 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 858 // ---------------------------------------------------------------------
 859 /* This puts a description structure in the linked list */
 860 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 861                                             const string &Lang,
 862                                             const MD5SumValue &md5sum,
 863                                             map_stringitem_t const idxmd5str)
 864 {
 865    // Get a structure
 866    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 867    if (Description == 0)
 868       return 0;
 869
 870    // Fill it in
 871    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 872    Desc->ID = Cache.HeaderP->DescriptionCount++;
 873    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 874    if (unlikely(idxlanguage_code == 0))
 875       return 0;
 876    Desc->language_code = idxlanguage_code;
 877
 878    if (idxmd5str != 0)
 879       Desc->md5sum = idxmd5str;
 880    else
 881    {
 882       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 883       if (unlikely(idxmd5sum == 0))
 884          return 0;
 885       Desc->md5sum = idxmd5sum;
 886    }
 887
 888    return Description;
 889 }
 890                                                                         /*}}}*/
 891 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 892 // ---------------------------------------------------------------------
 893 /* This creates a dependency element in the tree. It is linked to the
 894    version and to the package that it is pointing to. */
 895 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 896                                    pkgCache::VerIterator &Ver,
 897                                    map_pointer_t const Version,
 898                                    uint8_t const Op,
 899                                    uint8_t const Type,
 900                                    map_pointer_t* &OldDepLast)
 901 {
 902    void const * const oldMap = Map.Data();
 903    // Get a structure
 904    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 905    if (unlikely(Dependency == 0))
 906       return false;
 907
 908    bool isDuplicate = false;
 909    map_pointer_t DependencyData = 0;
 910    map_pointer_t PreviousData = 0;
 911    if (Pkg->RevDepends != 0)
 912    {
 913       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 914       DependencyData = L->DependencyData;
 915       do {
 916          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 917          if (Version > D->Version)
 918             break;
 919          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 920          {
 921             isDuplicate = true;
 922             break;
 923          }
 924          PreviousData = DependencyData;
 925          DependencyData = D->NextData;
 926       } while (DependencyData != 0);
 927    }
 928
 929    if (isDuplicate == false)
 930    {
 931       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 932       if (unlikely(DependencyData == 0))
 933         return false;
 934    }
 935
 936    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 937    Link->ParentVer = Ver.Index();
 938    Link->DependencyData = DependencyData;
 939    Link->ID = Cache.HeaderP->DependsCount++;
 940
 941    pkgCache::DepIterator Dep(Cache, Link);
 942    if (isDuplicate == false)
 943    {
 944       Dep->Type = Type;
 945       Dep->CompareOp = Op;
 946       Dep->Version = Version;
 947       Dep->Package = Pkg.Index();
 948       ++Cache.HeaderP->DependsDataCount;
 949       if (PreviousData != 0)
 950       {
 951          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 952          Dep->NextData = D->NextData;
 953          D->NextData = DependencyData;
 954       }
 955       else if (Pkg->RevDepends != 0)
 956       {
 957          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 958          Dep->NextData = D->DependencyData;
 959       }
 960    }
 961
 962    if (isDuplicate == true || PreviousData != 0)
 963    {
 964       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 965       Link->NextRevDepends = L->NextRevDepends;
 966       L->NextRevDepends = Dependency;
 967    }
 968    else
 969    {
 970       Link->NextRevDepends = Pkg->RevDepends;
 971       Pkg->RevDepends = Dependency;
 972    }
 973
 974
 975    // Do we know where to link the Dependency to?
 976    if (OldDepLast == NULL)
 977    {
 978       OldDepLast = &Ver->DependsList;
 979       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 980          OldDepLast = &D->NextDepends;
 981    } else if (oldMap != Map.Data())
 982       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 983
 984    Dep->NextDepends = *OldDepLast;
 985    *OldDepLast = Dependency;
 986    OldDepLast = &Dep->NextDepends;
 987    return true;
 988 }
 989                                                                         /*}}}*/
 990 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 991 // ---------------------------------------------------------------------
 992 /* This creates a Group and the Package to link this dependency to if
 993    needed and handles also the caching of the old endpoint */
 994 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
 995                                                const string &PackageName,
 996                                                const string &Arch,
 997                                                const string &Version,
 998                                                uint8_t const Op,
 999                                                uint8_t const Type)
1000 {
1001    pkgCache::GrpIterator Grp;
1002    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1003    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1004       return false;
1005
1006    map_stringitem_t idxVersion = 0;
1007    if (Version.empty() == false)
1008    {
1009       int const CmpOp = Op & 0x0F;
1010       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1011       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1012          idxVersion = Ver->VerStr;
1013
1014       if (idxVersion == 0)
1015       {
1016          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1017          if (unlikely(idxVersion == 0))
1018             return false;
1019       }
1020    }
1021
1022    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1023          Type == pkgCache::Dep::Conflicts ||
1024          Type == pkgCache::Dep::Replaces);
1025
1026    pkgCache::PkgIterator Pkg;
1027    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1028    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1029    {
1030       // Locate the target package
1031       Pkg = Grp.FindPkg(Arch);
1032       if (Pkg.end() == true) {
1033          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1034             return false;
1035       }
1036
1037       /* Caching the old end point speeds up generation substantially */
1038       if (OldDepVer != Ver) {
1039          OldDepLast = NULL;
1040          OldDepVer = Ver;
1041       }
1042
1043       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1044    }
1045    else
1046    {
1047       /* Caching the old end point speeds up generation substantially */
1048       if (OldDepVer != Ver) {
1049          OldDepLast = NULL;
1050          OldDepVer = Ver;
1051       }
1052
1053       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1054       {
1055          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1056                return false;
1057       }
1058    }
1059    return true;
1060 }
1061                                                                         /*}}}*/
1062 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1063 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1064                                                 const string &PkgName,
1065                                                 const string &PkgArch,
1066                                                 const string &Version,
1067                                                 uint8_t const Flags)
1068 {
1069    pkgCache const &Cache = Owner->Cache;
1070
1071    // We do not add self referencing provides
1072    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1073         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1074          (Version.empty() || Version == Ver.VerStr()))
1075       return true;
1076
1077    // Locate the target package
1078    pkgCache::PkgIterator Pkg;
1079    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1080    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1081       return false;
1082
1083    map_stringitem_t idxProvideVersion = 0;
1084    if (Version.empty() == false) {
1085       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1086       if (unlikely(idxProvideVersion == 0))
1087          return false;
1088    }
1089    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1090 }
1091 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1092                                     pkgCache::PkgIterator &Pkg,
1093                                     map_pointer_t const ProvideVersion,
1094                                     uint8_t const Flags)
1095 {
1096    // Get a structure
1097    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1098    if (unlikely(Provides == 0))
1099       return false;
1100    ++Cache.HeaderP->ProvidesCount;
1101
1102    // Fill it in
1103    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1104    Prv->Version = Ver.Index();
1105    Prv->ProvideVersion = ProvideVersion;
1106    Prv->Flags = Flags;
1107    Prv->NextPkgProv = Ver->ProvidesList;
1108    Ver->ProvidesList = Prv.Index();
1109
1110    // Link it to the package
1111    Prv->ParentPkg = Pkg.Index();
1112    Prv->NextProvides = Pkg->ProvidesList;
1113    Pkg->ProvidesList = Prv.Index();
1114    return true;
1115 }
1116                                                                         /*}}}*/
1117 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1118 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1119                                 string const &Version, uint8_t const Flags) {
1120    pkgCache &Cache = Owner->Cache;
1121    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1122    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1123
1124    if (Grp.end() == true)
1125       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1126    else
1127    {
1128       map_stringitem_t idxProvideVersion = 0;
1129       if (Version.empty() == false) {
1130          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1131          if (unlikely(idxProvideVersion == 0))
1132             return false;
1133       }
1134
1135       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1136       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1137       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1138       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1139       pkgCache::PkgIterator Pkg;
1140       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1141       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1142       {
1143          if (isImplicit && OwnerPkg == Pkg)
1144             continue;
1145          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1146             continue;
1147          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1148             return false;
1149       }
1150    }
1151    return true;
1152 }
1153                                                                         /*}}}*/
1154 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1155       pkgCache::VerIterator const &Ver)
1156 {
1157    return Hash == Ver->Hash;
1158 }
1159                                                                         /*}}}*/
1160 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1161 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1162                                    unsigned long Flags)
1163 {
1164    if (File.empty() && Site.empty())
1165    {
1166       CurrentRlsFile = NULL;
1167       return true;
1168    }
1169
1170    // Get some space for the structure
1171    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1172    if (unlikely(idxFile == 0))
1173       return false;
1174    CurrentRlsFile = Cache.RlsFileP + idxFile;
1175
1176    // Fill it in
1177    map_stringitem_t const idxFileName = WriteStringInMap(File);
1178    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1179    if (unlikely(idxFileName == 0 || idxSite == 0))
1180       return false;
1181    CurrentRlsFile->FileName = idxFileName;
1182    CurrentRlsFile->Site = idxSite;
1183    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1184    CurrentRlsFile->Flags = Flags;
1185    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1186    RlsFileName = File;
1187    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1188    Cache.HeaderP->ReleaseFileCount++;
1189
1190    return true;
1191 }
1192                                                                         /*}}}*/
1193 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1194 // ---------------------------------------------------------------------
1195 /* This is used to select which file is to be associated with all newly
1196    added versions. The caller is responsible for setting the IMS fields. */
1197 bool pkgCacheGenerator::SelectFile(std::string const &File,
1198                                    pkgIndexFile const &Index,
1199                                    std::string const &Architecture,
1200                                    std::string const &Component,
1201                                    unsigned long const Flags)
1202 {
1203    // Get some space for the structure
1204    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1205    if (unlikely(idxFile == 0))
1206       return false;
1207    CurrentFile = Cache.PkgFileP + idxFile;
1208
1209    // Fill it in
1210    map_stringitem_t const idxFileName = WriteStringInMap(File);
1211    if (unlikely(idxFileName == 0))
1212       return false;
1213    CurrentFile->FileName = idxFileName;
1214    CurrentFile->NextFile = Cache.HeaderP->FileList;
1215    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1216    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1217    if (unlikely(idxIndexType == 0))
1218       return false;
1219    CurrentFile->IndexType = idxIndexType;
1220    if (Architecture.empty())
1221       CurrentFile->Architecture = 0;
1222    else
1223    {
1224       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1225       if (unlikely(arch == 0))
1226          return false;
1227       CurrentFile->Architecture = arch;
1228    }
1229    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1230    if (unlikely(component == 0))
1231       return false;
1232    CurrentFile->Component = component;
1233    CurrentFile->Flags = Flags;
1234    if (CurrentRlsFile != NULL)
1235       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1236    else
1237       CurrentFile->Release = 0;
1238    PkgFileName = File;
1239    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1240    Cache.HeaderP->PackageFileCount++;
1241
1242    if (Progress != 0)
1243       Progress->SubProgress(Index.Size());
1244    return true;
1245 }
1246                                                                         /*}}}*/
1247 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1248 // ---------------------------------------------------------------------
1249 /* This is used to create handles to strings. Given the same text it
1250    always returns the same number */
1251 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1252                                                  unsigned int Size)
1253 {
1254    std::string const key(S, Size);
1255
1256    std::unordered_map<std::string,map_stringitem_t> * strings;
1257    switch(type) {
1258       case MIXED: strings = &strMixed; break;
1259       case PKGNAME: strings = &strPkgNames; break;
1260       case VERSIONNUMBER: strings = &strVersions; break;
1261       case SECTION: strings = &strSections; break;
1262       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1263    }
1264
1265    std::unordered_map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1266    if (item != strings->end())
1267       return item->second;
1268
1269    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1270    strings->insert(std::make_pair(key, idxString));
1271    return idxString;
1272 }
1273                                                                         /*}}}*/
1274 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1275 // ---------------------------------------------------------------------
1276 /* This just verifies that each file in the list of index files exists,
1277    has matching attributes with the cache and the cache does not have
1278    any extra files. */
1279 class APT_HIDDEN ScopedErrorRevert {
1280 public:
1281    ScopedErrorRevert() { _error->PushToStack(); }
1282    ~ScopedErrorRevert() { _error->RevertToStack(); }
1283 };
1284 static bool CheckValidity(const string &CacheFile,
1285                           pkgSourceList &List,
1286                           FileIterator const Start,
1287                           FileIterator const End,
1288                           MMap **OutMap = 0,
1289                           pkgCache **OutCache = 0)
1290 {
1291    ScopedErrorRevert ser;
1292    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1293    // No file, certainly invalid
1294    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1295    {
1296       if (Debug == true)
1297          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1298       return false;
1299    }
1300
1301    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1302    {
1303       if (Debug == true)
1304          std::clog << "sources.list is newer than the cache" << std::endl;
1305       return false;
1306    }
1307
1308    // Map it
1309    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1310    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1311    if (unlikely(Map->validData()) == false)
1312       return false;
1313    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1314    pkgCache &Cache = *CacheP.get();
1315    if (_error->PendingError() || Map->Size() == 0)
1316    {
1317       if (Debug == true)
1318          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1319       return false;
1320    }
1321
1322    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1323    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1324    std::vector<pkgIndexFile *> Files;
1325    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1326    {
1327       if (Debug == true)
1328          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1329       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1330       if (RlsFile.end() == true)
1331       {
1332          if (Debug == true)
1333             std::clog << "FindInCache returned end-Pointer" << std::endl;
1334          return false;
1335       }
1336
1337       RlsVisited[RlsFile->ID] = true;
1338       if (Debug == true)
1339          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1340
1341       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1342       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1343             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1344    }
1345    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1346       if (RlsVisited[I] == false)
1347       {
1348          if (Debug == true)
1349             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1350          return false;
1351       }
1352
1353    std::copy(Start, End, std::back_inserter(Files));
1354
1355    /* Now we check every index file, see if it is in the cache,
1356       verify the IMS data and check that it is on the disk too.. */
1357    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1358    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1359    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1360    {
1361       if (Debug == true)
1362          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1363       if ((*PkgFile)->Exists() == false)
1364       {
1365          if (Debug == true)
1366             std::clog << "file doesn't exist" << std::endl;
1367          continue;
1368       }
1369
1370       // FindInCache is also expected to do an IMS check.
1371       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1372       if (File.end() == true)
1373       {
1374          if (Debug == true)
1375             std::clog << "FindInCache returned end-Pointer" << std::endl;
1376          return false;
1377       }
1378
1379       Visited[File->ID] = true;
1380       if (Debug == true)
1381          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1382    }
1383
1384    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1385       if (Visited[I] == false)
1386       {
1387          if (Debug == true)
1388             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1389          return false;
1390       }
1391
1392    if (_error->PendingError() == true)
1393    {
1394       if (Debug == true)
1395       {
1396          std::clog << "Validity failed because of pending errors:" << std::endl;
1397          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1398       }
1399       return false;
1400    }
1401
1402    if (OutMap != 0)
1403       *OutMap = Map.release();
1404    if (OutCache != 0)
1405       *OutCache = CacheP.release();
1406    return true;
1407 }
1408                                                                         /*}}}*/
1409 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1410 // ---------------------------------------------------------------------
1411 /* Size is kind of an abstract notion that is only used for the progress
1412    meter */
1413 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1414 {
1415    map_filesize_t TotalSize = 0;
1416    if (List !=  NULL)
1417    {
1418       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1419       {
1420          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1421          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1422             if ((*j)->HasPackages() == true)
1423                TotalSize += (*j)->Size();
1424       }
1425    }
1426
1427    for (; Start < End; ++Start)
1428    {
1429       if ((*Start)->HasPackages() == false)
1430          continue;
1431       TotalSize += (*Start)->Size();
1432    }
1433    return TotalSize;
1434 }
1435                                                                         /*}}}*/
1436 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1437 static bool BuildCache(pkgCacheGenerator &Gen,
1438                        OpProgress * const Progress,
1439                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1440                        pkgSourceList const * const List,
1441                        FileIterator const Start, FileIterator const End)
1442 {
1443    bool mergeFailure = false;
1444
1445    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1446       if (I->HasPackages() == false || mergeFailure)
1447          return;
1448
1449       if (I->Exists() == false)
1450          return;
1451
1452       if (I->FindInCache(Gen.GetCache()).end() == false)
1453       {
1454          _error->Warning("Duplicate sources.list entry %s",
1455                I->Describe().c_str());
1456          return;
1457       }
1458
1459       map_filesize_t const Size = I->Size();
1460       if (Progress != NULL)
1461          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1462       CurrentSize += Size;
1463
1464       if (I->Merge(Gen,Progress) == false)
1465          mergeFailure = true;
1466    };
1467
1468    if (List !=  NULL)
1469    {
1470       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1471       {
1472          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1473          {
1474             _error->Warning("Duplicate sources.list entry %s",
1475                   (*i)->Describe().c_str());
1476             continue;
1477          }
1478
1479          if ((*i)->Merge(Gen, Progress) == false)
1480             return false;
1481
1482          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1483          if (Indexes != NULL)
1484             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1485          if (mergeFailure)
1486             return false;
1487       }
1488    }
1489
1490    if (Start != End)
1491    {
1492       Gen.SelectReleaseFile("", "");
1493       std::for_each(Start, End, indexFileMerge);
1494       if (mergeFailure)
1495          return false;
1496    }
1497    return true;
1498 }
1499                                                                         /*}}}*/
1500 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1501 // ---------------------------------------------------------------------
1502 /* This makes sure that the status cache (the cache that has all
1503    index files from the sources list and all local ones) is ready
1504    to be mmaped. If OutMap is not zero then a MMap object representing
1505    the cache will be stored there. This is pretty much mandetory if you
1506    are using AllowMem. AllowMem lets the function be run as non-root
1507    where it builds the cache 'fast' into a memory buffer. */
1508 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1509 {
1510    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1511    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1512    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1513    Flags |= MMap::Moveable;
1514    if (_config->FindB("APT::Cache-Fallback", false) == true)
1515       Flags |= MMap::Fallback;
1516    if (CacheF != NULL)
1517       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1518    else
1519       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1520 }
1521 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1522       std::string const &FileName)
1523 {
1524    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1525    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1526       return false;
1527
1528    fchmod(SCacheF.Fd(),0644);
1529
1530    // Write out the main data
1531    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1532       return _error->Error(_("IO Error saving source cache"));
1533    SCacheF.Sync();
1534
1535    // Write out the proper header
1536    Gen->GetCache().HeaderP->Dirty = false;
1537    if (SCacheF.Seek(0) == false ||
1538          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1539       return _error->Error(_("IO Error saving source cache"));
1540    Gen->GetCache().HeaderP->Dirty = true;
1541    SCacheF.Sync();
1542    return true;
1543 }
1544 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1545       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1546 {
1547    Map.reset(CreateDynamicMMap(NULL, 0));
1548    if (unlikely(Map->validData()) == false)
1549       return false;
1550    FileFd CacheF(FileName, FileFd::ReadOnly);
1551    if (CacheF.IsOpen() == false || CacheF.Failed())
1552       return false;
1553    _error->PushToStack();
1554    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1555    bool const newError = _error->PendingError();
1556    _error->MergeWithStack();
1557    if (alloc == 0 && newError)
1558       return false;
1559    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1560       return false;
1561    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1562    return true;
1563 }
1564 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1565                         MMap **OutMap, bool AllowMem)
1566    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1567 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1568                         MMap **OutMap,bool)
1569 {
1570    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1571 }
1572 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1573                         MMap **OutMap,pkgCache **OutCache, bool)
1574 {
1575    // FIXME: deprecate the ignored AllowMem parameter
1576    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1577
1578    std::vector<pkgIndexFile *> Files;
1579    if (_system->AddStatusFiles(Files) == false)
1580       return false;
1581
1582    // Decide if we can write to the files..
1583    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1584    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1585
1586    // ensure the cache directory exists
1587    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1588    {
1589       string dir = _config->FindDir("Dir::Cache");
1590       size_t const len = dir.size();
1591       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1592          dir = dir.substr(0, len - 5);
1593       if (CacheFile.empty() == false)
1594          CreateDirectory(dir, flNotFile(CacheFile));
1595       if (SrcCacheFile.empty() == false)
1596          CreateDirectory(dir, flNotFile(SrcCacheFile));
1597    }
1598
1599    if (Progress != NULL)
1600       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1601
1602    bool pkgcache_fine = false;
1603    bool srcpkgcache_fine = false;
1604    bool volatile_fine = List.GetVolatileFiles().empty();
1605
1606    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1607                      volatile_fine ? OutCache : NULL) == true)
1608    {
1609       if (Debug == true)
1610          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1611       pkgcache_fine = true;
1612       srcpkgcache_fine = true;
1613    }
1614    if (pkgcache_fine == false)
1615    {
1616       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1617       {
1618          if (Debug == true)
1619             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1620          srcpkgcache_fine = true;
1621       }
1622    }
1623
1624    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1625    {
1626       if (Progress != NULL)
1627          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1628       return true;
1629    }
1630
1631    bool Writeable = false;
1632    if (srcpkgcache_fine == false || pkgcache_fine == false)
1633    {
1634       if (CacheFile.empty() == false)
1635          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1636       else if (SrcCacheFile.empty() == false)
1637          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1638
1639       if (Debug == true)
1640          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1641    }
1642
1643    // At this point we know we need to construct something, so get storage ready
1644    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1645    if (unlikely(Map->validData()) == false)
1646       return false;
1647    if (Debug == true)
1648       std::clog << "Open memory Map (not filebased)" << std::endl;
1649
1650    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1651    map_filesize_t CurrentSize = 0;
1652    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1653    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1654    if (srcpkgcache_fine == true && pkgcache_fine == false)
1655    {
1656       if (Debug == true)
1657          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1658       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1659          return false;
1660       srcpkgcache_fine = true;
1661       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1662    }
1663    else if (srcpkgcache_fine == false)
1664    {
1665       if (Debug == true)
1666          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1667       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1668
1669       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1670       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1671                Files.end(),Files.end()) == false)
1672          return false;
1673
1674       if (Writeable == true && SrcCacheFile.empty() == false)
1675          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1676             return false;
1677    }
1678
1679    if (pkgcache_fine == false)
1680    {
1681       if (Debug == true)
1682          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1683       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1684                Files.begin(), Files.end()) == false)
1685          return false;
1686
1687       if (Writeable == true && CacheFile.empty() == false)
1688          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1689             return false;
1690    }
1691
1692    if (Debug == true)
1693       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1694
1695    if (volatile_fine == false)
1696    {
1697       if (Gen == nullptr)
1698       {
1699          if (Debug == true)
1700             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1701          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1702             return false;
1703       }
1704
1705       Files = List.GetVolatileFiles();
1706       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1707                Files.begin(), Files.end()) == false)
1708          return false;
1709    }
1710
1711    if (OutMap != nullptr)
1712       *OutMap = Map.release();
1713
1714    if (Debug == true)
1715       std::clog << "Everything is ready for shipping" << std::endl;
1716    return true;
1717 }
1718                                                                         /*}}}*/
1719 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1720 class APT_HIDDEN ScopedErrorMerge {
1721 public:
1722    ScopedErrorMerge() { _error->PushToStack(); }
1723    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1724 };
1725 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1726    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1727 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1728 {
1729    std::vector<pkgIndexFile *> Files;
1730    if (_system->AddStatusFiles(Files) == false)
1731       return false;
1732
1733    ScopedErrorMerge sem;
1734    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1735    if (unlikely(Map->validData()) == false)
1736       return false;
1737    map_filesize_t CurrentSize = 0;
1738    map_filesize_t TotalSize = 0;
1739    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1740
1741    // Build the status cache
1742    if (Progress != NULL)
1743       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1744    pkgCacheGenerator Gen(Map.get(),Progress);
1745    if (_error->PendingError() == true)
1746       return false;
1747    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1748                   Files.begin(), Files.end()) == false)
1749       return false;
1750
1751    if (_error->PendingError() == true)
1752       return false;
1753    *OutMap = Map.release();
1754
1755    return true;
1756 }
1757                                                                         /*}}}*/
1758 // IsDuplicateDescription                                               /*{{{*/
1759 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1760                             MD5SumValue const &CurMd5, std::string const &CurLang)
1761 {
1762    // Descriptions in the same link-list have all the same md5
1763    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1764       return false;
1765    for (; Desc.end() == false; ++Desc)
1766       if (Desc.LanguageCode() == CurLang)
1767          return true;
1768    return false;
1769 }
1770                                                                         /*}}}*/
1771
1772 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1773 pkgCacheListParser::~pkgCacheListParser() {}