apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <memory>
  40 #include <algorithm>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  47 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  48
  49 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  50                             MD5SumValue const &CurMd5, std::string const &CurLang);
  51
  52 using std::string;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (_error->PendingError() == true)
  62       return;
  63
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  69          return;
  70
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72
  73       // Starting header
  74       *Cache.HeaderP = pkgCache::Header();
  75
  76       // make room for the hashtables for packages and groups
  77       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  78          return;
  79
  80       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  81       if (unlikely(idxVerSysName == 0))
  82          return;
  83       Cache.HeaderP->VerSysName = idxVerSysName;
  84       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  85       if (unlikely(idxArchitecture == 0))
  86          return;
  87       Cache.HeaderP->Architecture = idxArchitecture;
  88
  89       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  90       if (archs.size() > 1)
  91       {
  92          std::vector<std::string>::const_iterator a = archs.begin();
  93          std::string list = *a;
  94          for (++a; a != archs.end(); ++a)
  95             list.append(",").append(*a);
  96          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  97          if (unlikely(idxArchitectures == 0))
  98             return;
  99          Cache.HeaderP->SetArchitectures(idxArchitectures);
 100       }
 101       else
 102          Cache.HeaderP->SetArchitectures(idxArchitecture);
 103
 104       Cache.ReMap();
 105    }
 106    else
 107    {
 108       // Map directly from the existing file
 109       Cache.ReMap();
 110       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 111       if (Cache.VS != _system->VS)
 112       {
 113          _error->Error(_("Cache has an incompatible versioning system"));
 114          return;
 115       }
 116    }
 117
 118    Cache.HeaderP->Dirty = true;
 119    Map.Sync(0,sizeof(pkgCache::Header));
 120 }
 121                                                                         /*}}}*/
 122 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* We sync the data then unset the dirty flag in two steps so as to
 125    advoid a problem during a crash */
 126 pkgCacheGenerator::~pkgCacheGenerator()
 127 {
 128    if (_error->PendingError() == true)
 129       return;
 130    if (Map.Sync() == false)
 131       return;
 132
 133    Cache.HeaderP->Dirty = false;
 134    Cache.HeaderP->CacheFileSize = Map.Size();
 135    Map.Sync(0,sizeof(pkgCache::Header));
 136 }
 137                                                                         /*}}}*/
 138 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 139    if (oldMap == newMap)
 140       return;
 141
 142    if (_config->FindB("Debug::pkgCacheGen", false))
 143       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 144
 145    Cache.ReMap(false);
 146
 147    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 148    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 149
 150    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 151         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 152       (*i)->ReMap(oldMap, newMap);
 153    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 154         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 155       (*i)->ReMap(oldMap, newMap);
 156    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174 }                                                                       /*}}}*/
 175 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 176 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 177                                         const unsigned long &Len) {
 178    void const * const oldMap = Map.Data();
 179    map_stringitem_t const index = Map.WriteString(String, Len);
 180    if (index != 0)
 181       ReMap(oldMap, Map.Data());
 182    return index;
 183 }
 184                                                                         /*}}}*/
 185 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 186 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 187    void const * const oldMap = Map.Data();
 188    map_stringitem_t const index = Map.WriteString(String);
 189    if (index != 0)
 190       ReMap(oldMap, Map.Data());
 191    return index;
 192 }
 193                                                                         /*}}}*/
 194 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 195    void const * const oldMap = Map.Data();
 196    map_pointer_t const index = Map.Allocate(size);
 197    if (index != 0)
 198       ReMap(oldMap, Map.Data());
 199    return index;
 200 }
 201                                                                         /*}}}*/
 202 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 203 // ---------------------------------------------------------------------
 204 /* This provides the generation of the entries in the cache. Each loop
 205    goes through a single package record from the underlying parse engine. */
 206 bool pkgCacheGenerator::MergeList(ListParser &List,
 207                                   pkgCache::VerIterator *OutVer)
 208 {
 209    List.Owner = this;
 210
 211    unsigned int Counter = 0;
 212    while (List.Step() == true)
 213    {
 214       string const PackageName = List.Package();
 215       if (PackageName.empty() == true)
 216          return false;
 217
 218       Counter++;
 219       if (Counter % 100 == 0 && Progress != 0)
 220          Progress->Progress(List.Offset());
 221
 222       string Arch = List.Architecture();
 223       string const Version = List.Version();
 224       if (Version.empty() == true && Arch.empty() == true)
 225       {
 226          // package descriptions
 227          if (MergeListGroup(List, PackageName) == false)
 228             return false;
 229          continue;
 230       }
 231
 232       // Get a pointer to the package structure
 233       pkgCache::PkgIterator Pkg;
 234       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 235       if (NewPackage(Pkg, PackageName, Arch) == false)
 236          // TRANSLATOR: The first placeholder is a package name,
 237          // the other two should be copied verbatim as they include debug info
 238          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 239                               PackageName.c_str(), "NewPackage", 1);
 240
 241
 242       if (Version.empty() == true)
 243       {
 244          if (MergeListPackage(List, Pkg) == false)
 245             return false;
 246       }
 247       else
 248       {
 249          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 250             return false;
 251       }
 252
 253       if (OutVer != 0)
 254          return true;
 255    }
 256
 257    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 258       return _error->Error(_("Wow, you exceeded the number of package "
 259                              "names this APT is capable of."));
 260    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 261       return _error->Error(_("Wow, you exceeded the number of versions "
 262                              "this APT is capable of."));
 263    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 264       return _error->Error(_("Wow, you exceeded the number of descriptions "
 265                              "this APT is capable of."));
 266    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of dependencies "
 268                              "this APT is capable of."));
 269
 270    return true;
 271 }
 272 // CacheGenerator::MergeListGroup                                       /*{{{*/
 273 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 274 {
 275    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 276    // a group has no data on it's own, only packages have it but these
 277    // stanzas like this come from Translation- files to add descriptions,
 278    // but without a version we don't need a description for it…
 279    if (Grp.end() == true)
 280       return true;
 281    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 282
 283    pkgCache::PkgIterator Pkg;
 284    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 285    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 286       if (MergeListPackage(List, Pkg) == false)
 287          return false;
 288
 289    return true;
 290 }
 291                                                                         /*}}}*/
 292 // CacheGenerator::MergeListPackage                                     /*{{{*/
 293 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 294 {
 295    // we first process the package, then the descriptions
 296    // (for deb this package processing is in fact a no-op)
 297    pkgCache::VerIterator Ver(Cache);
 298    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 299    if (List.UsePackage(Pkg, Ver) == false)
 300       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 301                            Pkg.Name(), "UsePackage", 1);
 302
 303    // Find the right version to write the description
 304    MD5SumValue CurMd5 = List.Description_md5();
 305    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 306       return true;
 307    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 308    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 309    {
 310       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 311
 312       // a version can only have one md5 describing it
 313       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 314          continue;
 315
 316       map_stringitem_t md5idx = VerDesc->md5sum;
 317       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 318       {
 319          // don't add a new description if we have one for the given
 320          // md5 && language
 321          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 322             continue;
 323
 324          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 325       }
 326
 327       // we can stop here as all "same" versions will share the description
 328       break;
 329    }
 330
 331    return true;
 332 }
 333                                                                         /*}}}*/
 334 // CacheGenerator::MergeListVersion                                     /*{{{*/
 335 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 336                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 337 {
 338    pkgCache::VerIterator Ver = Pkg.VersionList();
 339    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 340    map_pointer_t *LastVer = &Pkg->VersionList;
 341    void const * oldMap = Map.Data();
 342
 343    unsigned short const Hash = List.VersionHash();
 344    if (Ver.end() == false)
 345    {
 346       /* We know the list is sorted so we use that fact in the search.
 347          Insertion of new versions is done with correct sorting */
 348       int Res = 1;
 349       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 350       {
 351          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 352          // Version is higher as current version - insert here
 353          if (Res > 0)
 354             break;
 355          // Versionstrings are equal - is hash also equal?
 356          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 357             break;
 358          // proceed with the next till we have either the right
 359          // or we found another version (which will be lower)
 360       }
 361
 362       /* We already have a version for this item, record that we saw it */
 363       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 364       {
 365          if (List.UsePackage(Pkg,Ver) == false)
 366             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 367                                  Pkg.Name(), "UsePackage", 2);
 368
 369          if (NewFileVer(Ver,List) == false)
 370             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 371                                  Pkg.Name(), "NewFileVer", 1);
 372
 373          // Read only a single record and return
 374          if (OutVer != 0)
 375          {
 376             *OutVer = Ver;
 377             return true;
 378          }
 379
 380          return true;
 381       }
 382    }
 383
 384    // Add a new version
 385    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 386    if (verindex == 0 && _error->PendingError())
 387       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 388                            Pkg.Name(), "NewVersion", 1);
 389
 390    if (oldMap != Map.Data())
 391          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 392    *LastVer = verindex;
 393
 394    if (unlikely(List.NewVersion(Ver) == false))
 395       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 396                            Pkg.Name(), "NewVersion", 2);
 397
 398    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 399       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 400                            Pkg.Name(), "UsePackage", 3);
 401
 402    if (unlikely(NewFileVer(Ver,List) == false))
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "NewFileVer", 2);
 405
 406    pkgCache::GrpIterator Grp = Pkg.Group();
 407    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 408
 409    /* If it is the first version of this package we need to add implicit
 410       Multi-Arch dependencies to all other package versions in the group now -
 411       otherwise we just add them for this new version */
 412    if (Pkg.VersionList()->NextVer == 0)
 413    {
 414       pkgCache::PkgIterator P = Grp.PackageList();
 415       Dynamic<pkgCache::PkgIterator> DynP(P);
 416       for (; P.end() != true; P = Grp.NextPkg(P))
 417       {
 418          if (P->ID == Pkg->ID)
 419             continue;
 420          pkgCache::VerIterator V = P.VersionList();
 421          Dynamic<pkgCache::VerIterator> DynV(V);
 422          for (; V.end() != true; ++V)
 423             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 424                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 425                                     Pkg.Name(), "AddImplicitDepends", 1);
 426       }
 427    }
 428    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 429       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 430                            Pkg.Name(), "AddImplicitDepends", 2);
 431
 432    // Read only a single record and return
 433    if (OutVer != 0)
 434    {
 435       *OutVer = Ver;
 436       return true;
 437    }
 438
 439    /* Record the Description(s) based on their master md5sum */
 440    MD5SumValue CurMd5 = List.Description_md5();
 441    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 442       return true;
 443
 444    /* Before we add a new description we first search in the group for
 445       a version with a description of the same MD5 - if so we reuse this
 446       description group instead of creating our own for this version */
 447    for (pkgCache::PkgIterator P = Grp.PackageList();
 448         P.end() == false; P = Grp.NextPkg(P))
 449    {
 450       for (pkgCache::VerIterator V = P.VersionList();
 451            V.end() == false; ++V)
 452       {
 453          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 454             continue;
 455          Ver->DescriptionList = V->DescriptionList;
 456       }
 457    }
 458
 459    // We haven't found reusable descriptions, so add the first description(s)
 460    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 461    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 462    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 463       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 464          return false;
 465    return true;
 466 }
 467                                                                         /*}}}*/
 468 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 469 {
 470    pkgCache::DescIterator Desc;
 471    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 472
 473    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 474    if (unlikely(descindex == 0 && _error->PendingError()))
 475       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 476             Ver.ParentPkg().Name(), "NewDescription", 1);
 477
 478    md5idx = Desc->md5sum;
 479    Desc->ParentPkg = Ver.ParentPkg().Index();
 480
 481    // we add at the end, so that the start is constant as we need
 482    // that to be able to efficiently share these lists
 483    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 484    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 485    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 486    *LastNextDesc = descindex;
 487
 488    if (NewFileDesc(Desc,List) == false)
 489       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 490             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 491
 492    return true;
 493 }
 494                                                                         /*}}}*/
 495                                                                         /*}}}*/
 496 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 497 // ---------------------------------------------------------------------
 498 /* This creates a new group structure and adds it to the hash table */
 499 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 500 {
 501    Grp = Cache.FindGrp(Name);
 502    if (Grp.end() == false)
 503       return true;
 504
 505    // Get a structure
 506    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 507    if (unlikely(Group == 0))
 508       return false;
 509
 510    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 511    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 512    if (unlikely(idxName == 0))
 513       return false;
 514    Grp->Name = idxName;
 515
 516    // Insert it into the hash table
 517    unsigned long const Hash = Cache.Hash(Name);
 518    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 519    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 520       insertAt = &(Cache.GrpP + *insertAt)->Next;
 521    Grp->Next = *insertAt;
 522    *insertAt = Group;
 523
 524    Grp->ID = Cache.HeaderP->GroupCount++;
 525    return true;
 526 }
 527                                                                         /*}}}*/
 528 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 529 // ---------------------------------------------------------------------
 530 /* This creates a new package structure and adds it to the hash table */
 531 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 532                                         const string &Arch) {
 533    pkgCache::GrpIterator Grp;
 534    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 535    if (unlikely(NewGroup(Grp, Name) == false))
 536       return false;
 537
 538    Pkg = Grp.FindPkg(Arch);
 539       if (Pkg.end() == false)
 540          return true;
 541
 542    // Get a structure
 543    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 544    if (unlikely(Package == 0))
 545       return false;
 546    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 547
 548    // Set the name, arch and the ID
 549    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 550    Pkg->Group = Grp.Index();
 551    // all is mapped to the native architecture
 552    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 553    if (unlikely(idxArch == 0))
 554       return false;
 555    Pkg->Arch = idxArch;
 556    Pkg->ID = Cache.HeaderP->PackageCount++;
 557
 558    // Insert the package into our package list
 559    if (Grp->FirstPackage == 0) // the group is new
 560    {
 561       Grp->FirstPackage = Package;
 562       // Insert it into the hash table
 563       map_id_t const Hash = Cache.Hash(Name);
 564       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 565       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 566          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 567       Pkg->NextPackage = *insertAt;
 568       *insertAt = Package;
 569    }
 570    else // Group the Packages together
 571    {
 572       // but first get implicit provides done
 573       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 574       {
 575          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 576          if (M.end() == false)
 577             for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 578             {
 579                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 580                   continue;
 581                pkgCache::VerIterator Ver = Prv.OwnerVer();
 582                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 583                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 584                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 585                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 586                      return false;
 587             }
 588
 589          for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 590             for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
 591                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 592                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 593                      return false;
 594       }
 595       // and negative dependencies, don't forget negative dependencies
 596       {
 597          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 598          if (M.end() == false)
 599             for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 600             {
 601                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 602                   continue;
 603                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 604                      Dep->Type != pkgCache::Dep::Replaces)
 605                   continue;
 606                pkgCache::VerIterator Ver = Dep.ParentVer();
 607                map_pointer_t * unused = NULL;
 608                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 609                   return false;
 610             }
 611       }
 612
 613       // this package is the new last package
 614       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 615       Pkg->NextPackage = LastPkg->NextPackage;
 616       LastPkg->NextPackage = Package;
 617    }
 618    Grp->LastPackage = Package;
 619    return true;
 620 }
 621                                                                         /*}}}*/
 622 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 623 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 624                                            pkgCache::PkgIterator &P,
 625                                            pkgCache::VerIterator &V)
 626 {
 627    // copy P.Arch() into a string here as a cache remap
 628    // in NewDepends() later may alter the pointer location
 629    string Arch = P.Arch() == NULL ? "" : P.Arch();
 630    map_pointer_t *OldDepLast = NULL;
 631    /* MultiArch handling introduces a lot of implicit Dependencies:
 632       - MultiArch: same → Co-Installable if they have the same version
 633       - All others conflict with all other group members */
 634    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 635    pkgCache::PkgIterator D = G.PackageList();
 636    Dynamic<pkgCache::PkgIterator> DynD(D);
 637    map_stringitem_t const VerStrIdx = V->VerStr;
 638    for (; D.end() != true; D = G.NextPkg(D))
 639    {
 640       if (Arch == D.Arch() || D->VersionList == 0)
 641          continue;
 642       /* We allow only one installed arch at the time
 643          per group, therefore each group member conflicts
 644          with all other group members */
 645       if (coInstall == true)
 646       {
 647          // Replaces: ${self}:other ( << ${binary:Version})
 648          NewDepends(D, V, VerStrIdx,
 649                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 650                     OldDepLast);
 651          // Breaks: ${self}:other (!= ${binary:Version})
 652          NewDepends(D, V, VerStrIdx,
 653                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 654                     OldDepLast);
 655       } else {
 656          // Conflicts: ${self}:other
 657          NewDepends(D, V, 0,
 658                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 659                     OldDepLast);
 660       }
 661    }
 662    return true;
 663 }
 664 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 665                                            pkgCache::PkgIterator &D)
 666 {
 667    /* MultiArch handling introduces a lot of implicit Dependencies:
 668       - MultiArch: same → Co-Installable if they have the same version
 669       - All others conflict with all other group members */
 670    map_pointer_t *OldDepLast = NULL;
 671    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 672    if (coInstall == true)
 673    {
 674       map_stringitem_t const VerStrIdx = V->VerStr;
 675       // Replaces: ${self}:other ( << ${binary:Version})
 676       NewDepends(D, V, VerStrIdx,
 677                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 678                  OldDepLast);
 679       // Breaks: ${self}:other (!= ${binary:Version})
 680       NewDepends(D, V, VerStrIdx,
 681                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 682                  OldDepLast);
 683    } else {
 684       // Conflicts: ${self}:other
 685       NewDepends(D, V, 0,
 686                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 687                  OldDepLast);
 688    }
 689    return true;
 690 }
 691
 692                                                                         /*}}}*/
 693 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 694 // ---------------------------------------------------------------------
 695 /* */
 696 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 697                                    ListParser &List)
 698 {
 699    if (CurrentFile == 0)
 700       return true;
 701
 702    // Get a structure
 703    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 704    if (VerFile == 0)
 705       return false;
 706
 707    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 708    VF->File = CurrentFile - Cache.PkgFileP;
 709
 710    // Link it to the end of the list
 711    map_pointer_t *Last = &Ver->FileList;
 712    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 713       Last = &V->NextFile;
 714    VF->NextFile = *Last;
 715    *Last = VF.Index();
 716
 717    VF->Offset = List.Offset();
 718    VF->Size = List.Size();
 719    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 720       Cache.HeaderP->MaxVerFileSize = VF->Size;
 721    Cache.HeaderP->VerFileCount++;
 722
 723    return true;
 724 }
 725                                                                         /*}}}*/
 726 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 727 // ---------------------------------------------------------------------
 728 /* This puts a version structure in the linked list */
 729 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 730                                             const string &VerStr,
 731                                             map_pointer_t const ParentPkg,
 732                                             unsigned short const Hash,
 733                                             map_pointer_t const Next)
 734 {
 735    // Get a structure
 736    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 737    if (Version == 0)
 738       return 0;
 739
 740    // Fill it in
 741    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 742    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 743    Ver->NextVer = Next;
 744    Ver->ParentPkg = ParentPkg;
 745    Ver->Hash = Hash;
 746    Ver->ID = Cache.HeaderP->VersionCount++;
 747
 748    // try to find the version string in the group for reuse
 749    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 750    pkgCache::GrpIterator Grp = Pkg.Group();
 751    if (Pkg.end() == false && Grp.end() == false)
 752    {
 753       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 754       {
 755          if (Pkg == P)
 756             continue;
 757          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 758          {
 759             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 760             if (cmp == 0)
 761             {
 762                Ver->VerStr = V->VerStr;
 763                return Version;
 764             }
 765             else if (cmp < 0)
 766                break;
 767          }
 768       }
 769    }
 770    // haven't found the version string, so create
 771    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 772    if (unlikely(idxVerStr == 0))
 773       return 0;
 774    Ver->VerStr = idxVerStr;
 775    return Version;
 776 }
 777                                                                         /*}}}*/
 778 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* */
 781 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 782                                    ListParser &List)
 783 {
 784    if (CurrentFile == 0)
 785       return true;
 786
 787    // Get a structure
 788    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 789    if (DescFile == 0)
 790       return false;
 791
 792    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 793    DF->File = CurrentFile - Cache.PkgFileP;
 794
 795    // Link it to the end of the list
 796    map_pointer_t *Last = &Desc->FileList;
 797    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 798       Last = &D->NextFile;
 799
 800    DF->NextFile = *Last;
 801    *Last = DF.Index();
 802
 803    DF->Offset = List.Offset();
 804    DF->Size = List.Size();
 805    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 806       Cache.HeaderP->MaxDescFileSize = DF->Size;
 807    Cache.HeaderP->DescFileCount++;
 808
 809    return true;
 810 }
 811                                                                         /*}}}*/
 812 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 813 // ---------------------------------------------------------------------
 814 /* This puts a description structure in the linked list */
 815 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 816                                             const string &Lang,
 817                                             const MD5SumValue &md5sum,
 818                                             map_stringitem_t const idxmd5str)
 819 {
 820    // Get a structure
 821    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 822    if (Description == 0)
 823       return 0;
 824
 825    // Fill it in
 826    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 827    Desc->ID = Cache.HeaderP->DescriptionCount++;
 828    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 829    if (unlikely(idxlanguage_code == 0))
 830       return 0;
 831    Desc->language_code = idxlanguage_code;
 832
 833    if (idxmd5str != 0)
 834       Desc->md5sum = idxmd5str;
 835    else
 836    {
 837       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 838       if (unlikely(idxmd5sum == 0))
 839          return 0;
 840       Desc->md5sum = idxmd5sum;
 841    }
 842
 843    return Description;
 844 }
 845                                                                         /*}}}*/
 846 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* This creates a dependency element in the tree. It is linked to the
 849    version and to the package that it is pointing to. */
 850 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 851                                    pkgCache::VerIterator &Ver,
 852                                    map_pointer_t const Version,
 853                                    uint8_t const Op,
 854                                    uint8_t const Type,
 855                                    map_pointer_t* &OldDepLast)
 856 {
 857    void const * const oldMap = Map.Data();
 858    // Get a structure
 859    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 860    if (unlikely(Dependency == 0))
 861       return false;
 862
 863    bool isDuplicate = false;
 864    map_pointer_t DependencyData = 0;
 865    map_pointer_t PreviousData = 0;
 866    if (Pkg->RevDepends != 0)
 867    {
 868       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 869       DependencyData = L->DependencyData;
 870       do {
 871          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 872          if (Version > D->Version)
 873             break;
 874          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 875          {
 876             isDuplicate = true;
 877             break;
 878          }
 879          PreviousData = DependencyData;
 880          DependencyData = D->NextData;
 881       } while (DependencyData != 0);
 882    }
 883
 884    if (isDuplicate == false)
 885    {
 886       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 887       if (unlikely(DependencyData == 0))
 888         return false;
 889    }
 890
 891    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 892    Link->ParentVer = Ver.Index();
 893    Link->DependencyData = DependencyData;
 894    Link->ID = Cache.HeaderP->DependsCount++;
 895
 896    pkgCache::DepIterator Dep(Cache, Link);
 897    if (isDuplicate == false)
 898    {
 899       Dep->Type = Type;
 900       Dep->CompareOp = Op;
 901       Dep->Version = Version;
 902       Dep->Package = Pkg.Index();
 903       ++Cache.HeaderP->DependsDataCount;
 904       if (PreviousData != 0)
 905       {
 906          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 907          Dep->NextData = D->NextData;
 908          D->NextData = DependencyData;
 909       }
 910       else if (Pkg->RevDepends != 0)
 911       {
 912          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 913          Dep->NextData = D->DependencyData;
 914       }
 915    }
 916
 917    if (isDuplicate == true || PreviousData != 0)
 918    {
 919       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 920       Link->NextRevDepends = L->NextRevDepends;
 921       L->NextRevDepends = Dependency;
 922    }
 923    else
 924    {
 925       Link->NextRevDepends = Pkg->RevDepends;
 926       Pkg->RevDepends = Dependency;
 927    }
 928
 929
 930    // Do we know where to link the Dependency to?
 931    if (OldDepLast == NULL)
 932    {
 933       OldDepLast = &Ver->DependsList;
 934       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 935          OldDepLast = &D->NextDepends;
 936    } else if (oldMap != Map.Data())
 937       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 938
 939    Dep->NextDepends = *OldDepLast;
 940    *OldDepLast = Dependency;
 941    OldDepLast = &Dep->NextDepends;
 942    return true;
 943 }
 944                                                                         /*}}}*/
 945 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 946 // ---------------------------------------------------------------------
 947 /* This creates a Group and the Package to link this dependency to if
 948    needed and handles also the caching of the old endpoint */
 949 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
 950                                                const string &PackageName,
 951                                                const string &Arch,
 952                                                const string &Version,
 953                                                uint8_t const Op,
 954                                                uint8_t const Type)
 955 {
 956    pkgCache::GrpIterator Grp;
 957    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 958    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 959       return false;
 960
 961    map_stringitem_t idxVersion = 0;
 962    if (Version.empty() == false)
 963    {
 964       int const CmpOp = Op & 0x0F;
 965       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 966       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 967          idxVersion = Ver->VerStr;
 968
 969       if (idxVersion == 0)
 970       {
 971          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
 972          if (unlikely(idxVersion == 0))
 973             return false;
 974       }
 975    }
 976
 977    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
 978          Type == pkgCache::Dep::Conflicts ||
 979          Type == pkgCache::Dep::Replaces);
 980
 981    pkgCache::PkgIterator Pkg;
 982    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 983    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
 984    {
 985       // Locate the target package
 986       Pkg = Grp.FindPkg(Arch);
 987       if (Pkg.end() == true) {
 988          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 989             return false;
 990       }
 991
 992       /* Caching the old end point speeds up generation substantially */
 993       if (OldDepVer != Ver) {
 994          OldDepLast = NULL;
 995          OldDepVer = Ver;
 996       }
 997
 998       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
 999    }
1000    else
1001    {
1002       /* Caching the old end point speeds up generation substantially */
1003       if (OldDepVer != Ver) {
1004          OldDepLast = NULL;
1005          OldDepVer = Ver;
1006       }
1007
1008       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1009       {
1010          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1011                return false;
1012       }
1013    }
1014    return true;
1015 }
1016                                                                         /*}}}*/
1017 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1018 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1019                                                 const string &PkgName,
1020                                                 const string &PkgArch,
1021                                                 const string &Version,
1022                                                 uint8_t const Flags)
1023 {
1024    pkgCache const &Cache = Owner->Cache;
1025
1026    // We do not add self referencing provides
1027    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1028         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1029       return true;
1030
1031    // Locate the target package
1032    pkgCache::PkgIterator Pkg;
1033    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1034    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1035       return false;
1036
1037    map_stringitem_t idxProvideVersion = 0;
1038    if (Version.empty() == false) {
1039       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1040       if (unlikely(idxProvideVersion == 0))
1041          return false;
1042    }
1043    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1044 }
1045 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1046                                     pkgCache::PkgIterator &Pkg,
1047                                     map_pointer_t const ProvideVersion,
1048                                     uint8_t const Flags)
1049 {
1050    // Get a structure
1051    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1052    if (unlikely(Provides == 0))
1053       return false;
1054    ++Cache.HeaderP->ProvidesCount;
1055
1056    // Fill it in
1057    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1058    Prv->Version = Ver.Index();
1059    Prv->ProvideVersion = ProvideVersion;
1060    Prv->Flags = Flags;
1061    Prv->NextPkgProv = Ver->ProvidesList;
1062    Ver->ProvidesList = Prv.Index();
1063
1064    // Link it to the package
1065    Prv->ParentPkg = Pkg.Index();
1066    Prv->NextProvides = Pkg->ProvidesList;
1067    Pkg->ProvidesList = Prv.Index();
1068    return true;
1069 }
1070                                                                         /*}}}*/
1071 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1072 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1073                                 string const &Version, uint8_t const Flags) {
1074    pkgCache &Cache = Owner->Cache;
1075    pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1076    if (Grp.end() == true)
1077       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1078    else
1079    {
1080       map_stringitem_t idxProvideVersion = 0;
1081       if (Version.empty() == false) {
1082          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1083          if (unlikely(idxProvideVersion == 0))
1084             return false;
1085       }
1086
1087       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1088       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1089       pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1090       for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1091       {
1092          if (isImplicit && OwnerPkg == Pkg)
1093             continue;
1094          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1095             continue;
1096          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1097             return false;
1098       }
1099    }
1100    return true;
1101 }
1102                                                                         /*}}}*/
1103 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1104       pkgCache::VerIterator const &Ver)
1105 {
1106    return Hash == Ver->Hash;
1107 }
1108                                                                         /*}}}*/
1109 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1110 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1111                                    unsigned long Flags)
1112 {
1113    if (File.empty() && Site.empty())
1114    {
1115       CurrentRlsFile = NULL;
1116       return true;
1117    }
1118
1119    // Get some space for the structure
1120    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1121    if (unlikely(idxFile == 0))
1122       return false;
1123    CurrentRlsFile = Cache.RlsFileP + idxFile;
1124
1125    // Fill it in
1126    map_stringitem_t const idxFileName = WriteStringInMap(File);
1127    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1128    if (unlikely(idxFileName == 0 || idxSite == 0))
1129       return false;
1130    CurrentRlsFile->FileName = idxFileName;
1131    CurrentRlsFile->Site = idxSite;
1132    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1133    CurrentRlsFile->Flags = Flags;
1134    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1135    RlsFileName = File;
1136    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1137    Cache.HeaderP->ReleaseFileCount++;
1138
1139    return true;
1140 }
1141                                                                         /*}}}*/
1142 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1143 // ---------------------------------------------------------------------
1144 /* This is used to select which file is to be associated with all newly
1145    added versions. The caller is responsible for setting the IMS fields. */
1146 bool pkgCacheGenerator::SelectFile(std::string const &File,
1147                                    pkgIndexFile const &Index,
1148                                    std::string const &Architecture,
1149                                    std::string const &Component,
1150                                    unsigned long const Flags)
1151 {
1152    // Get some space for the structure
1153    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1154    if (unlikely(idxFile == 0))
1155       return false;
1156    CurrentFile = Cache.PkgFileP + idxFile;
1157
1158    // Fill it in
1159    map_stringitem_t const idxFileName = WriteStringInMap(File);
1160    if (unlikely(idxFileName == 0))
1161       return false;
1162    CurrentFile->FileName = idxFileName;
1163    CurrentFile->NextFile = Cache.HeaderP->FileList;
1164    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1165    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1166    if (unlikely(idxIndexType == 0))
1167       return false;
1168    CurrentFile->IndexType = idxIndexType;
1169    if (Architecture.empty())
1170       CurrentFile->Architecture = 0;
1171    else
1172    {
1173       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1174       if (unlikely(arch == 0))
1175          return false;
1176       CurrentFile->Architecture = arch;
1177    }
1178    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1179    if (unlikely(component == 0))
1180       return false;
1181    CurrentFile->Component = component;
1182    CurrentFile->Flags = Flags;
1183    if (CurrentRlsFile != NULL)
1184       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1185    else
1186       CurrentFile->Release = 0;
1187    PkgFileName = File;
1188    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1189    Cache.HeaderP->PackageFileCount++;
1190
1191    if (Progress != 0)
1192       Progress->SubProgress(Index.Size());
1193    return true;
1194 }
1195                                                                         /*}}}*/
1196 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1197 // ---------------------------------------------------------------------
1198 /* This is used to create handles to strings. Given the same text it
1199    always returns the same number */
1200 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1201                                                  unsigned int Size)
1202 {
1203    std::string const key(S, Size);
1204
1205    std::map<std::string,map_stringitem_t> * strings;
1206    switch(type) {
1207       case MIXED: strings = &strMixed; break;
1208       case PKGNAME: strings = &strPkgNames; break;
1209       case VERSIONNUMBER: strings = &strVersions; break;
1210       case SECTION: strings = &strSections; break;
1211       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1212    }
1213
1214    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1215    if (item != strings->end())
1216       return item->second;
1217
1218    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1219    strings->insert(std::make_pair(key, idxString));
1220    return idxString;
1221 }
1222                                                                         /*}}}*/
1223 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1224 // ---------------------------------------------------------------------
1225 /* This just verifies that each file in the list of index files exists,
1226    has matching attributes with the cache and the cache does not have
1227    any extra files. */
1228 static bool CheckValidity(const string &CacheFile,
1229                           pkgSourceList &List,
1230                           FileIterator const Start,
1231                           FileIterator const End,
1232                           MMap **OutMap = 0)
1233 {
1234    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1235    // No file, certainly invalid
1236    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1237    {
1238       if (Debug == true)
1239          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1240       return false;
1241    }
1242
1243    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1244    {
1245       if (Debug == true)
1246          std::clog << "sources.list is newer than the cache" << std::endl;
1247       return false;
1248    }
1249
1250    // Map it
1251    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1252    SPtr<MMap> Map = new MMap(CacheF,0);
1253    pkgCache Cache(Map);
1254    if (_error->PendingError() == true || Map->Size() == 0)
1255    {
1256       if (Debug == true)
1257          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1258       _error->Discard();
1259       return false;
1260    }
1261
1262    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1263    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1264    std::vector<pkgIndexFile *> Files;
1265    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1266    {
1267       if (Debug == true)
1268          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1269       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1270       if (RlsFile.end() == true)
1271       {
1272          if (Debug == true)
1273             std::clog << "FindInCache returned end-Pointer" << std::endl;
1274          return false;
1275       }
1276
1277       RlsVisited[RlsFile->ID] = true;
1278       if (Debug == true)
1279          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1280
1281       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1282       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1283             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1284    }
1285    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1286       if (RlsVisited[I] == false)
1287       {
1288          if (Debug == true)
1289             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1290          return false;
1291       }
1292
1293    std::copy(Start, End, std::back_inserter(Files));
1294
1295    /* Now we check every index file, see if it is in the cache,
1296       verify the IMS data and check that it is on the disk too.. */
1297    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1298    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1299    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1300    {
1301       if (Debug == true)
1302          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1303       if ((*PkgFile)->Exists() == false)
1304       {
1305          if (Debug == true)
1306             std::clog << "file doesn't exist" << std::endl;
1307          continue;
1308       }
1309
1310       // FindInCache is also expected to do an IMS check.
1311       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1312       if (File.end() == true)
1313       {
1314          if (Debug == true)
1315             std::clog << "FindInCache returned end-Pointer" << std::endl;
1316          return false;
1317       }
1318
1319       Visited[File->ID] = true;
1320       if (Debug == true)
1321          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1322    }
1323
1324    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1325       if (Visited[I] == false)
1326       {
1327          if (Debug == true)
1328             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1329          return false;
1330       }
1331
1332    if (_error->PendingError() == true)
1333    {
1334       if (Debug == true)
1335       {
1336          std::clog << "Validity failed because of pending errors:" << std::endl;
1337          _error->DumpErrors();
1338       }
1339       _error->Discard();
1340       return false;
1341    }
1342
1343    if (OutMap != 0)
1344       *OutMap = Map.UnGuard();
1345    return true;
1346 }
1347                                                                         /*}}}*/
1348 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1349 // ---------------------------------------------------------------------
1350 /* Size is kind of an abstract notion that is only used for the progress
1351    meter */
1352 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1353 {
1354    map_filesize_t TotalSize = 0;
1355    if (List !=  NULL)
1356    {
1357       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1358       {
1359          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1360          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1361             if ((*j)->HasPackages() == true)
1362                TotalSize += (*j)->Size();
1363       }
1364    }
1365
1366    for (; Start < End; ++Start)
1367    {
1368       if ((*Start)->HasPackages() == false)
1369          continue;
1370       TotalSize += (*Start)->Size();
1371    }
1372    return TotalSize;
1373 }
1374                                                                         /*}}}*/
1375 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1376 static bool BuildCache(pkgCacheGenerator &Gen,
1377                        OpProgress * const Progress,
1378                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1379                        pkgSourceList const * const List,
1380                        FileIterator const Start, FileIterator const End)
1381 {
1382    std::vector<pkgIndexFile *> Files;
1383    bool mergeFailure = false;
1384
1385    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1386       if (I->HasPackages() == false || mergeFailure)
1387          return;
1388
1389       if (I->Exists() == false)
1390          return;
1391
1392       if (I->FindInCache(Gen.GetCache()).end() == false)
1393       {
1394          _error->Warning("Duplicate sources.list entry %s",
1395                I->Describe().c_str());
1396          return;
1397       }
1398
1399       map_filesize_t const Size = I->Size();
1400       if (Progress != NULL)
1401          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1402       CurrentSize += Size;
1403
1404       if (I->Merge(Gen,Progress) == false)
1405          mergeFailure = true;
1406    };
1407
1408    if (List !=  NULL)
1409    {
1410       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1411       {
1412          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1413          {
1414             _error->Warning("Duplicate sources.list entry %s",
1415                   (*i)->Describe().c_str());
1416             continue;
1417          }
1418
1419          if ((*i)->Merge(Gen, Progress) == false)
1420             return false;
1421
1422          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1423          if (Indexes != NULL)
1424             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1425          if (mergeFailure)
1426             return false;
1427       }
1428    }
1429
1430    if (Start != End)
1431    {
1432       Gen.SelectReleaseFile("", "");
1433       std::for_each(Start, End, indexFileMerge);
1434       if (mergeFailure)
1435          return false;
1436    }
1437    return true;
1438 }
1439                                                                         /*}}}*/
1440 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1441 // ---------------------------------------------------------------------
1442 /* This makes sure that the status cache (the cache that has all
1443    index files from the sources list and all local ones) is ready
1444    to be mmaped. If OutMap is not zero then a MMap object representing
1445    the cache will be stored there. This is pretty much mandetory if you
1446    are using AllowMem. AllowMem lets the function be run as non-root
1447    where it builds the cache 'fast' into a memory buffer. */
1448 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1449 {
1450    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1451    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1452    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1453    Flags |= MMap::Moveable;
1454    if (_config->FindB("APT::Cache-Fallback", false) == true)
1455       Flags |= MMap::Fallback;
1456    if (CacheF != NULL)
1457       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1458    else
1459       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1460 }
1461 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1462       std::string const &FileName)
1463 {
1464    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1465    if (_error->PendingError() == true)
1466       return false;
1467
1468    fchmod(SCacheF.Fd(),0644);
1469
1470    // Write out the main data
1471    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1472       return _error->Error(_("IO Error saving source cache"));
1473    SCacheF.Sync();
1474
1475    // Write out the proper header
1476    Gen->GetCache().HeaderP->Dirty = false;
1477    if (SCacheF.Seek(0) == false ||
1478          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1479       return _error->Error(_("IO Error saving source cache"));
1480    Gen->GetCache().HeaderP->Dirty = true;
1481    SCacheF.Sync();
1482    return true;
1483 }
1484 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1485       SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1486 {
1487    Map = CreateDynamicMMap(NULL, 0);
1488    FileFd CacheF(FileName, FileFd::ReadOnly);
1489    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1490    if ((alloc == 0 && _error->PendingError())
1491          || CacheF.Read((unsigned char *)Map->Data() + alloc,
1492             CacheF.Size()) == false)
1493       return false;
1494    Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1495    return true;
1496 }
1497 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1498                         MMap **OutMap, bool AllowMem)
1499    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1500 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1501                         MMap **OutMap,bool AllowMem)
1502 {
1503    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1504
1505    std::vector<pkgIndexFile *> Files;
1506    if (_system->AddStatusFiles(Files) == false)
1507       return false;
1508
1509    // Decide if we can write to the files..
1510    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1511    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1512
1513    // ensure the cache directory exists
1514    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1515    {
1516       string dir = _config->FindDir("Dir::Cache");
1517       size_t const len = dir.size();
1518       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1519          dir = dir.substr(0, len - 5);
1520       if (CacheFile.empty() == false)
1521          CreateDirectory(dir, flNotFile(CacheFile));
1522       if (SrcCacheFile.empty() == false)
1523          CreateDirectory(dir, flNotFile(SrcCacheFile));
1524    }
1525
1526    if (Progress != NULL)
1527       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1528
1529    bool pkgcache_fine = false;
1530    bool srcpkgcache_fine = false;
1531    bool volatile_fine = List.GetVolatileFiles().empty();
1532
1533    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
1534    {
1535       if (Debug == true)
1536          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1537       pkgcache_fine = true;
1538       srcpkgcache_fine = true;
1539    }
1540    if (pkgcache_fine == false)
1541    {
1542       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1543       {
1544          if (Debug == true)
1545             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1546          srcpkgcache_fine = true;
1547       }
1548    }
1549
1550    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1551    {
1552       if (Progress != NULL)
1553          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1554       return true;
1555    }
1556
1557    bool Writeable = false;
1558    if (srcpkgcache_fine == false || pkgcache_fine == false)
1559    {
1560       if (CacheFile.empty() == false)
1561          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1562       else if (SrcCacheFile.empty() == false)
1563          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1564
1565       if (Debug == true)
1566          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1567
1568       if (Writeable == false && AllowMem == false)
1569       {
1570          if (CacheFile.empty() == false)
1571             return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1572          else if (SrcCacheFile.empty() == false)
1573             return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
1574          else
1575             return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
1576       }
1577    }
1578
1579    // At this point we know we need to construct something, so get storage ready
1580    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1581    if (Debug == true)
1582       std::clog << "Open memory Map (not filebased)" << std::endl;
1583
1584    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1585    map_filesize_t CurrentSize = 0;
1586    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1587    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1588    if (srcpkgcache_fine == true && pkgcache_fine == false)
1589    {
1590       if (Debug == true)
1591          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1592       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1593          return false;
1594       srcpkgcache_fine = true;
1595       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1596    }
1597    else if (srcpkgcache_fine == false)
1598    {
1599       if (Debug == true)
1600          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1601       Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1602
1603       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1604       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1605                Files.end(),Files.end()) == false)
1606          return false;
1607
1608       if (Writeable == true && SrcCacheFile.empty() == false)
1609          if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
1610             return false;
1611    }
1612
1613    if (pkgcache_fine == false)
1614    {
1615       if (Debug == true)
1616          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1617       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1618                Files.begin(), Files.end()) == false)
1619          return false;
1620
1621       if (Writeable == true && CacheFile.empty() == false)
1622          if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
1623             return false;
1624    }
1625
1626    if (Debug == true)
1627       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1628
1629    if (volatile_fine == false)
1630    {
1631       if (Gen == nullptr)
1632       {
1633          if (Debug == true)
1634             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1635          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1636             return false;
1637       }
1638
1639       Files = List.GetVolatileFiles();
1640       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1641                Files.begin(), Files.end()) == false)
1642          return false;
1643    }
1644
1645    if (OutMap != nullptr)
1646       *OutMap = Map.UnGuard();
1647
1648    if (Debug == true)
1649       std::clog << "Everything is ready for shipping" << std::endl;
1650    return true;
1651 }
1652                                                                         /*}}}*/
1653 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1654 // ---------------------------------------------------------------------
1655 /* */
1656 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1657    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1658 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1659 {
1660    std::vector<pkgIndexFile *> Files;
1661    if (_system->AddStatusFiles(Files) == false)
1662       return false;
1663
1664    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1665    map_filesize_t CurrentSize = 0;
1666    map_filesize_t TotalSize = 0;
1667
1668    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1669
1670    // Build the status cache
1671    if (Progress != NULL)
1672       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1673    pkgCacheGenerator Gen(Map.Get(),Progress);
1674    if (_error->PendingError() == true)
1675       return false;
1676    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1677                   Files.begin(), Files.end()) == false)
1678       return false;
1679
1680    if (_error->PendingError() == true)
1681       return false;
1682    *OutMap = Map.UnGuard();
1683
1684    return true;
1685 }
1686                                                                         /*}}}*/
1687 // IsDuplicateDescription                                               /*{{{*/
1688 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1689                             MD5SumValue const &CurMd5, std::string const &CurLang)
1690 {
1691    // Descriptions in the same link-list have all the same md5
1692    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1693       return false;
1694    for (; Desc.end() == false; ++Desc)
1695       if (Desc.LanguageCode() == CurLang)
1696          return true;
1697    return false;
1698 }
1699                                                                         /*}}}*/
1700
1701 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1702 pkgCacheListParser::~pkgCacheListParser() {}