apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 181       // Ignore views outside of the cache.
 182       if (ViewP->data() < static_cast<const char*>(oldMap)
 183          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 184          continue;
 185       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 186       *ViewP = StringView(data , ViewP->size());
 187    }
 188 }                                                                       /*}}}*/
 189 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 190 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 191                                         const unsigned long &Len) {
 192    size_t oldSize = Map.Size();
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String, Len);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data(), oldSize);
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 201 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 202    size_t oldSize = Map.Size();
 203    void const * const oldMap = Map.Data();
 204    map_stringitem_t const index = Map.WriteString(String);
 205    if (index != 0)
 206       ReMap(oldMap, Map.Data(), oldSize);
 207    return index;
 208 }
 209                                                                         /*}}}*/
 210 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 211    size_t oldSize = Map.Size();
 212    void const * const oldMap = Map.Data();
 213    map_pointer_t const index = Map.Allocate(size);
 214    if (index != 0)
 215       ReMap(oldMap, Map.Data(), oldSize);
 216    return index;
 217 }
 218                                                                         /*}}}*/
 219 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 220 // ---------------------------------------------------------------------
 221 /* This provides the generation of the entries in the cache. Each loop
 222    goes through a single package record from the underlying parse engine. */
 223 bool pkgCacheGenerator::MergeList(ListParser &List,
 224                                   pkgCache::VerIterator *OutVer)
 225 {
 226    List.Owner = this;
 227
 228    unsigned int Counter = 0;
 229    while (List.Step() == true)
 230    {
 231       string const PackageName = List.Package();
 232       if (PackageName.empty() == true)
 233          return false;
 234
 235       Counter++;
 236       if (Counter % 100 == 0 && Progress != 0)
 237          Progress->Progress(List.Offset());
 238
 239       string Arch = List.Architecture();
 240       string const Version = List.Version();
 241       if (Version.empty() == true && Arch.empty() == true)
 242       {
 243          // package descriptions
 244          if (MergeListGroup(List, PackageName) == false)
 245             return false;
 246          continue;
 247       }
 248
 249       // Get a pointer to the package structure
 250       pkgCache::PkgIterator Pkg;
 251       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 252       if (NewPackage(Pkg, PackageName, Arch) == false)
 253          // TRANSLATOR: The first placeholder is a package name,
 254          // the other two should be copied verbatim as they include debug info
 255          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 256                               PackageName.c_str(), "NewPackage", 1);
 257
 258
 259       if (Version.empty() == true)
 260       {
 261          if (MergeListPackage(List, Pkg) == false)
 262             return false;
 263       }
 264       else
 265       {
 266          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 267             return false;
 268       }
 269
 270       if (OutVer != 0)
 271          return true;
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    return true;
 288 }
 289 // CacheGenerator::MergeListGroup                                       /*{{{*/
 290 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 291 {
 292    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 293    // a group has no data on it's own, only packages have it but these
 294    // stanzas like this come from Translation- files to add descriptions,
 295    // but without a version we don't need a description for it…
 296    if (Grp.end() == true)
 297       return true;
 298    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 299
 300    pkgCache::PkgIterator Pkg;
 301    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 302    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 303       if (MergeListPackage(List, Pkg) == false)
 304          return false;
 305
 306    return true;
 307 }
 308                                                                         /*}}}*/
 309 // CacheGenerator::MergeListPackage                                     /*{{{*/
 310 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 311 {
 312    // we first process the package, then the descriptions
 313    // (for deb this package processing is in fact a no-op)
 314    pkgCache::VerIterator Ver(Cache);
 315    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 316    if (List.UsePackage(Pkg, Ver) == false)
 317       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 318                            Pkg.Name(), "UsePackage", 1);
 319
 320    // Find the right version to write the description
 321    MD5SumValue CurMd5 = List.Description_md5();
 322    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 323    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 324    {
 325       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 326
 327       // a version can only have one md5 describing it
 328       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 329          continue;
 330
 331       map_stringitem_t md5idx = VerDesc->md5sum;
 332       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 333       {
 334          // don't add a new description if we have one for the given
 335          // md5 && language
 336          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 337             continue;
 338
 339          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 340       }
 341
 342       // we can stop here as all "same" versions will share the description
 343       break;
 344    }
 345
 346    return true;
 347 }
 348                                                                         /*}}}*/
 349 // CacheGenerator::MergeListVersion                                     /*{{{*/
 350 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 351                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 352 {
 353    pkgCache::VerIterator Ver = Pkg.VersionList();
 354    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 355    map_pointer_t *LastVer = &Pkg->VersionList;
 356    void const * oldMap = Map.Data();
 357
 358    unsigned short const Hash = List.VersionHash();
 359    if (Ver.end() == false)
 360    {
 361       /* We know the list is sorted so we use that fact in the search.
 362          Insertion of new versions is done with correct sorting */
 363       int Res = 1;
 364       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 365       {
 366          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 367          // Version is higher as current version - insert here
 368          if (Res > 0)
 369             break;
 370          // Versionstrings are equal - is hash also equal?
 371          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 372             break;
 373          // proceed with the next till we have either the right
 374          // or we found another version (which will be lower)
 375       }
 376
 377       /* We already have a version for this item, record that we saw it */
 378       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 379       {
 380          if (List.UsePackage(Pkg,Ver) == false)
 381             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 382                                  Pkg.Name(), "UsePackage", 2);
 383
 384          if (NewFileVer(Ver,List) == false)
 385             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 386                                  Pkg.Name(), "NewFileVer", 1);
 387
 388          // Read only a single record and return
 389          if (OutVer != 0)
 390          {
 391             *OutVer = Ver;
 392             return true;
 393          }
 394
 395          return true;
 396       }
 397    }
 398
 399    // Add a new version
 400    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 401    if (unlikely(verindex == 0))
 402       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 403                            Pkg.Name(), "NewVersion", 1);
 404
 405    if (oldMap != Map.Data())
 406          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 407    *LastVer = verindex;
 408
 409    if (unlikely(List.NewVersion(Ver) == false))
 410       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 411                            Pkg.Name(), "NewVersion", 2);
 412
 413    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 414       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 415                            Pkg.Name(), "UsePackage", 3);
 416
 417    if (unlikely(NewFileVer(Ver,List) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "NewFileVer", 2);
 420
 421    pkgCache::GrpIterator Grp = Pkg.Group();
 422    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 423
 424    /* If it is the first version of this package we need to add implicit
 425       Multi-Arch dependencies to all other package versions in the group now -
 426       otherwise we just add them for this new version */
 427    if (Pkg.VersionList()->NextVer == 0)
 428    {
 429       pkgCache::PkgIterator P = Grp.PackageList();
 430       Dynamic<pkgCache::PkgIterator> DynP(P);
 431       for (; P.end() != true; P = Grp.NextPkg(P))
 432       {
 433          if (P->ID == Pkg->ID)
 434             continue;
 435          pkgCache::VerIterator V = P.VersionList();
 436          Dynamic<pkgCache::VerIterator> DynV(V);
 437          for (; V.end() != true; ++V)
 438             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 439                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 440                                     Pkg.Name(), "AddImplicitDepends", 1);
 441       }
 442    }
 443    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 444       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 445                            Pkg.Name(), "AddImplicitDepends", 2);
 446
 447    // Read only a single record and return
 448    if (OutVer != 0)
 449    {
 450       *OutVer = Ver;
 451       return true;
 452    }
 453
 454    /* Record the Description(s) based on their master md5sum */
 455    MD5SumValue CurMd5 = List.Description_md5();
 456
 457    /* Before we add a new description we first search in the group for
 458       a version with a description of the same MD5 - if so we reuse this
 459       description group instead of creating our own for this version */
 460    for (pkgCache::PkgIterator P = Grp.PackageList();
 461         P.end() == false; P = Grp.NextPkg(P))
 462    {
 463       for (pkgCache::VerIterator V = P.VersionList();
 464            V.end() == false; ++V)
 465       {
 466          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 467             continue;
 468          Ver->DescriptionList = V->DescriptionList;
 469       }
 470    }
 471
 472    // We haven't found reusable descriptions, so add the first description(s)
 473    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 474    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 475    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 476       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 477          return false;
 478    return true;
 479 }
 480                                                                         /*}}}*/
 481 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 482 {
 483    pkgCache::DescIterator Desc;
 484    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 485
 486    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 487    if (unlikely(descindex == 0))
 488       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 489             Ver.ParentPkg().Name(), "NewDescription", 1);
 490
 491    md5idx = Desc->md5sum;
 492    Desc->ParentPkg = Ver.ParentPkg().Index();
 493
 494    // we add at the end, so that the start is constant as we need
 495    // that to be able to efficiently share these lists
 496    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 497    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 498    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 499    *LastNextDesc = descindex;
 500
 501    if (NewFileDesc(Desc,List) == false)
 502       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 503             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 504
 505    return true;
 506 }
 507                                                                         /*}}}*/
 508                                                                         /*}}}*/
 509 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* This creates a new group structure and adds it to the hash table */
 512 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 513 {
 514    Dynamic<StringView> DName(Name);
 515    Grp = Cache.FindGrp(Name);
 516    if (Grp.end() == false)
 517       return true;
 518
 519    // Get a structure
 520    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 521    if (unlikely(Group == 0))
 522       return false;
 523
 524    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 525    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 526    if (unlikely(idxName == 0))
 527       return false;
 528    Grp->Name = idxName;
 529
 530    // Insert it into the hash table
 531    unsigned long const Hash = Cache.Hash(Name);
 532    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 533
 534    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 535       insertAt = &(Cache.GrpP + *insertAt)->Next;
 536    Grp->Next = *insertAt;
 537    *insertAt = Group;
 538
 539    Grp->ID = Cache.HeaderP->GroupCount++;
 540    return true;
 541 }
 542                                                                         /*}}}*/
 543 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* This creates a new package structure and adds it to the hash table */
 546 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 547                                         StringView Arch) {
 548    pkgCache::GrpIterator Grp;
 549    Dynamic<StringView> DName(Name);
 550    Dynamic<StringView> DArch(Arch);
 551    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 552    if (unlikely(NewGroup(Grp, Name) == false))
 553       return false;
 554
 555    Pkg = Grp.FindPkg(Arch);
 556       if (Pkg.end() == false)
 557          return true;
 558
 559    // Get a structure
 560    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 561    if (unlikely(Package == 0))
 562       return false;
 563    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 564
 565    // Set the name, arch and the ID
 566    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 567    Pkg->Group = Grp.Index();
 568    // all is mapped to the native architecture
 569    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 570    if (unlikely(idxArch == 0))
 571       return false;
 572    Pkg->Arch = idxArch;
 573    Pkg->ID = Cache.HeaderP->PackageCount++;
 574
 575    // Insert the package into our package list
 576    if (Grp->FirstPackage == 0) // the group is new
 577    {
 578       Grp->FirstPackage = Package;
 579       // Insert it into the hash table
 580       map_id_t const Hash = Cache.Hash(Name);
 581       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 582       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 583          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 584       Pkg->NextPackage = *insertAt;
 585       *insertAt = Package;
 586    }
 587    else // Group the Packages together
 588    {
 589       // if sibling is provided by another package, this one is too
 590       {
 591          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 592          if (M.end() == false) {
 593             pkgCache::PrvIterator Prv;
 594             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 595             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 596             {
 597                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 598                   continue;
 599                pkgCache::VerIterator Ver = Prv.OwnerVer();
 600                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 601                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 602                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 603                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 604                {
 605                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 606                      continue;
 607                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 608                      return false;
 609                }
 610             }
 611          }
 612       }
 613       // let M-A:foreign package siblings provide this package
 614       {
 615          pkgCache::PkgIterator P;
 616          pkgCache::VerIterator Ver;
 617          Dynamic<pkgCache::PkgIterator> DynP(P);
 618          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 619
 620          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 621          {
 622             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 623                continue;
 624             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 625                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 626                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 627                      return false;
 628          }
 629       }
 630       // and negative dependencies, don't forget negative dependencies
 631       {
 632          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 633          if (M.end() == false) {
 634             pkgCache::DepIterator Dep;
 635             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 636             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 637             {
 638                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 639                   continue;
 640                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 641                      Dep->Type != pkgCache::Dep::Replaces)
 642                   continue;
 643                pkgCache::VerIterator Ver = Dep.ParentVer();
 644                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 645                map_pointer_t * unused = NULL;
 646                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 647                   return false;
 648             }
 649          }
 650       }
 651
 652       // this package is the new last package
 653       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 654       Pkg->NextPackage = LastPkg->NextPackage;
 655       LastPkg->NextPackage = Package;
 656    }
 657    Grp->LastPackage = Package;
 658
 659    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 660    if (Arch == "any")
 661    {
 662       size_t const found = Name.rfind(':');
 663       StringView ArchA = Name.substr(found + 1);
 664       if (ArchA != "any")
 665       {
 666          // ArchA is used inside the loop which might remap (NameA is not used)
 667          Dynamic<StringView> DynArchA(ArchA);
 668          StringView NameA = Name.substr(0, found);
 669          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 670          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 671          if (PkgA.end())
 672          {
 673             Dynamic<StringView> DynNameA(NameA);
 674             if (NewPackage(PkgA, NameA, ArchA) == false)
 675                return false;
 676          }
 677          if (unlikely(PkgA.end()))
 678             return _error->Fatal("NewPackage was successful for %s:%s,"
 679                   "but the package doesn't exist anyhow!",
 680                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 681          else
 682          {
 683             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 684             for (; Prv.end() == false; ++Prv)
 685             {
 686                if (Prv.IsMultiArchImplicit())
 687                   continue;
 688                pkgCache::VerIterator V = Prv.OwnerVer();
 689                if (ArchA != V.ParentPkg().Arch())
 690                   continue;
 691                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 692                   return false;
 693             }
 694             pkgCache::VerIterator V = PkgA.VersionList();
 695             Dynamic<pkgCache::VerIterator> DynV(V);
 696             for (; V.end() == false; ++V)
 697             {
 698                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 699                   return false;
 700             }
 701          }
 702       }
 703    }
 704    return true;
 705 }
 706                                                                         /*}}}*/
 707 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 708 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 709                                            pkgCache::PkgIterator &P,
 710                                            pkgCache::VerIterator &V)
 711 {
 712    // copy P.Arch() into a string here as a cache remap
 713    // in NewDepends() later may alter the pointer location
 714    string Arch = P.Arch() == NULL ? "" : P.Arch();
 715    map_pointer_t *OldDepLast = NULL;
 716    /* MultiArch handling introduces a lot of implicit Dependencies:
 717       - MultiArch: same → Co-Installable if they have the same version
 718       - All others conflict with all other group members */
 719    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 720    pkgCache::PkgIterator D = G.PackageList();
 721    Dynamic<pkgCache::PkgIterator> DynD(D);
 722    map_stringitem_t const VerStrIdx = V->VerStr;
 723    for (; D.end() != true; D = G.NextPkg(D))
 724    {
 725       if (Arch == D.Arch() || D->VersionList == 0)
 726          continue;
 727       /* We allow only one installed arch at the time
 728          per group, therefore each group member conflicts
 729          with all other group members */
 730       if (coInstall == true)
 731       {
 732          // Replaces: ${self}:other ( << ${binary:Version})
 733          NewDepends(D, V, VerStrIdx,
 734                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 735                     OldDepLast);
 736          // Breaks: ${self}:other (!= ${binary:Version})
 737          NewDepends(D, V, VerStrIdx,
 738                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 739                     OldDepLast);
 740       } else {
 741          // Conflicts: ${self}:other
 742          NewDepends(D, V, 0,
 743                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 744                     OldDepLast);
 745       }
 746    }
 747    return true;
 748 }
 749 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 750                                            pkgCache::PkgIterator &D)
 751 {
 752    /* MultiArch handling introduces a lot of implicit Dependencies:
 753       - MultiArch: same → Co-Installable if they have the same version
 754       - All others conflict with all other group members */
 755    map_pointer_t *OldDepLast = NULL;
 756    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 757    if (coInstall == true)
 758    {
 759       map_stringitem_t const VerStrIdx = V->VerStr;
 760       // Replaces: ${self}:other ( << ${binary:Version})
 761       NewDepends(D, V, VerStrIdx,
 762                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 763                  OldDepLast);
 764       // Breaks: ${self}:other (!= ${binary:Version})
 765       NewDepends(D, V, VerStrIdx,
 766                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 767                  OldDepLast);
 768    } else {
 769       // Conflicts: ${self}:other
 770       NewDepends(D, V, 0,
 771                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 772                  OldDepLast);
 773    }
 774    return true;
 775 }
 776
 777                                                                         /*}}}*/
 778 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* */
 781 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 782                                    ListParser &List)
 783 {
 784    if (CurrentFile == 0)
 785       return true;
 786
 787    // Get a structure
 788    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 789    if (VerFile == 0)
 790       return false;
 791
 792    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 793    VF->File = CurrentFile - Cache.PkgFileP;
 794
 795    // Link it to the end of the list
 796    map_pointer_t *Last = &Ver->FileList;
 797    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 798       Last = &V->NextFile;
 799    VF->NextFile = *Last;
 800    *Last = VF.Index();
 801
 802    VF->Offset = List.Offset();
 803    VF->Size = List.Size();
 804    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 805       Cache.HeaderP->MaxVerFileSize = VF->Size;
 806    Cache.HeaderP->VerFileCount++;
 807
 808    return true;
 809 }
 810                                                                         /*}}}*/
 811 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 812 // ---------------------------------------------------------------------
 813 /* This puts a version structure in the linked list */
 814 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 815                                             const string &VerStr,
 816                                             map_pointer_t const ParentPkg,
 817                                             unsigned short const Hash,
 818                                             map_pointer_t const Next)
 819 {
 820    // Get a structure
 821    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 822    if (Version == 0)
 823       return 0;
 824
 825    // Fill it in
 826    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 827    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 828    Ver->NextVer = Next;
 829    Ver->ParentPkg = ParentPkg;
 830    Ver->Hash = Hash;
 831    Ver->ID = Cache.HeaderP->VersionCount++;
 832
 833    // try to find the version string in the group for reuse
 834    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 835    pkgCache::GrpIterator Grp = Pkg.Group();
 836    if (Pkg.end() == false && Grp.end() == false)
 837    {
 838       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 839       {
 840          if (Pkg == P)
 841             continue;
 842          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 843          {
 844             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 845             if (cmp == 0)
 846             {
 847                Ver->VerStr = V->VerStr;
 848                return Version;
 849             }
 850             else if (cmp < 0)
 851                break;
 852          }
 853       }
 854    }
 855    // haven't found the version string, so create
 856    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 857    if (unlikely(idxVerStr == 0))
 858       return 0;
 859    Ver->VerStr = idxVerStr;
 860    return Version;
 861 }
 862                                                                         /*}}}*/
 863 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 864 // ---------------------------------------------------------------------
 865 /* */
 866 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 867                                    ListParser &List)
 868 {
 869    if (CurrentFile == 0)
 870       return true;
 871
 872    // Get a structure
 873    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 874    if (DescFile == 0)
 875       return false;
 876
 877    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 878    DF->File = CurrentFile - Cache.PkgFileP;
 879
 880    // Link it to the end of the list
 881    map_pointer_t *Last = &Desc->FileList;
 882    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 883       Last = &D->NextFile;
 884
 885    DF->NextFile = *Last;
 886    *Last = DF.Index();
 887
 888    DF->Offset = List.Offset();
 889    DF->Size = List.Size();
 890    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 891       Cache.HeaderP->MaxDescFileSize = DF->Size;
 892    Cache.HeaderP->DescFileCount++;
 893
 894    return true;
 895 }
 896                                                                         /*}}}*/
 897 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 898 // ---------------------------------------------------------------------
 899 /* This puts a description structure in the linked list */
 900 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 901                                             const string &Lang,
 902                                             const MD5SumValue &md5sum,
 903                                             map_stringitem_t const idxmd5str)
 904 {
 905    // Get a structure
 906    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 907    if (Description == 0)
 908       return 0;
 909
 910    // Fill it in
 911    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 912    Desc->ID = Cache.HeaderP->DescriptionCount++;
 913    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 914    if (unlikely(idxlanguage_code == 0))
 915       return 0;
 916    Desc->language_code = idxlanguage_code;
 917
 918    if (idxmd5str != 0)
 919       Desc->md5sum = idxmd5str;
 920    else
 921    {
 922       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 923       if (unlikely(idxmd5sum == 0))
 924          return 0;
 925       Desc->md5sum = idxmd5sum;
 926    }
 927
 928    return Description;
 929 }
 930                                                                         /*}}}*/
 931 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 932 // ---------------------------------------------------------------------
 933 /* This creates a dependency element in the tree. It is linked to the
 934    version and to the package that it is pointing to. */
 935 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 936                                    pkgCache::VerIterator &Ver,
 937                                    map_pointer_t const Version,
 938                                    uint8_t const Op,
 939                                    uint8_t const Type,
 940                                    map_pointer_t* &OldDepLast)
 941 {
 942    void const * const oldMap = Map.Data();
 943    // Get a structure
 944    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 945    if (unlikely(Dependency == 0))
 946       return false;
 947
 948    bool isDuplicate = false;
 949    map_pointer_t DependencyData = 0;
 950    map_pointer_t PreviousData = 0;
 951    if (Pkg->RevDepends != 0)
 952    {
 953       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 954       DependencyData = L->DependencyData;
 955       do {
 956          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 957          if (Version > D->Version)
 958             break;
 959          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 960          {
 961             isDuplicate = true;
 962             break;
 963          }
 964          PreviousData = DependencyData;
 965          DependencyData = D->NextData;
 966       } while (DependencyData != 0);
 967    }
 968
 969    if (isDuplicate == false)
 970    {
 971       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 972       if (unlikely(DependencyData == 0))
 973         return false;
 974    }
 975
 976    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 977    Link->ParentVer = Ver.Index();
 978    Link->DependencyData = DependencyData;
 979    Link->ID = Cache.HeaderP->DependsCount++;
 980
 981    pkgCache::DepIterator Dep(Cache, Link);
 982    if (isDuplicate == false)
 983    {
 984       Dep->Type = Type;
 985       Dep->CompareOp = Op;
 986       Dep->Version = Version;
 987       Dep->Package = Pkg.Index();
 988       ++Cache.HeaderP->DependsDataCount;
 989       if (PreviousData != 0)
 990       {
 991          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 992          Dep->NextData = D->NextData;
 993          D->NextData = DependencyData;
 994       }
 995       else if (Pkg->RevDepends != 0)
 996       {
 997          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 998          Dep->NextData = D->DependencyData;
 999       }
1000    }
1001
1002    if (isDuplicate == true || PreviousData != 0)
1003    {
1004       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1005       Link->NextRevDepends = L->NextRevDepends;
1006       L->NextRevDepends = Dependency;
1007    }
1008    else
1009    {
1010       Link->NextRevDepends = Pkg->RevDepends;
1011       Pkg->RevDepends = Dependency;
1012    }
1013
1014
1015    // Do we know where to link the Dependency to?
1016    if (OldDepLast == NULL)
1017    {
1018       OldDepLast = &Ver->DependsList;
1019       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1020          OldDepLast = &D->NextDepends;
1021    } else if (oldMap != Map.Data())
1022       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1023
1024    Dep->NextDepends = *OldDepLast;
1025    *OldDepLast = Dependency;
1026    OldDepLast = &Dep->NextDepends;
1027    return true;
1028 }
1029                                                                         /*}}}*/
1030 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1031 // ---------------------------------------------------------------------
1032 /* This creates a Group and the Package to link this dependency to if
1033    needed and handles also the caching of the old endpoint */
1034 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1035                                                StringView PackageName,
1036                                                StringView Arch,
1037                                                StringView Version,
1038                                                uint8_t const Op,
1039                                                uint8_t const Type)
1040 {
1041    pkgCache::GrpIterator Grp;
1042    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1043    Dynamic<StringView> DynPackageName(PackageName);
1044    Dynamic<StringView> DynArch(Arch);
1045    Dynamic<StringView> DynVersion(Version);
1046    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1047       return false;
1048
1049    map_stringitem_t idxVersion = 0;
1050    if (Version.empty() == false)
1051    {
1052       int const CmpOp = Op & 0x0F;
1053       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1054       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1055          idxVersion = Ver->VerStr;
1056
1057       if (idxVersion == 0)
1058       {
1059          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1060          if (unlikely(idxVersion == 0))
1061             return false;
1062       }
1063    }
1064
1065    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1066          Type == pkgCache::Dep::Conflicts ||
1067          Type == pkgCache::Dep::Replaces);
1068
1069    pkgCache::PkgIterator Pkg;
1070    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1071    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1072    {
1073       // Locate the target package
1074       Pkg = Grp.FindPkg(Arch);
1075       if (Pkg.end() == true) {
1076          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1077             return false;
1078       }
1079
1080       /* Caching the old end point speeds up generation substantially */
1081       if (OldDepVer != Ver) {
1082          OldDepLast = NULL;
1083          OldDepVer = Ver;
1084       }
1085
1086       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1087    }
1088    else
1089    {
1090       /* Caching the old end point speeds up generation substantially */
1091       if (OldDepVer != Ver) {
1092          OldDepLast = NULL;
1093          OldDepVer = Ver;
1094       }
1095
1096       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1097       {
1098          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1099                return false;
1100       }
1101    }
1102    return true;
1103 }
1104                                                                         /*}}}*/
1105 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1106 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1107                                                 StringView PkgName,
1108                                                 StringView PkgArch,
1109                                                 StringView Version,
1110                                                 uint8_t const Flags)
1111 {
1112    pkgCache const &Cache = Owner->Cache;
1113    Dynamic<StringView> DynPkgName(PkgName);
1114    Dynamic<StringView> DynArch(PkgArch);
1115    Dynamic<StringView> DynVersion(Version);
1116
1117    // We do not add self referencing provides
1118    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1119         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1120          (Version.empty() || Version == Ver.VerStr()))
1121       return true;
1122
1123    // Locate the target package
1124    pkgCache::PkgIterator Pkg;
1125    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1126    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1127       return false;
1128
1129    map_stringitem_t idxProvideVersion = 0;
1130    if (Version.empty() == false) {
1131       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1132       if (unlikely(idxProvideVersion == 0))
1133          return false;
1134    }
1135    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1136 }
1137 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1138                                     pkgCache::PkgIterator &Pkg,
1139                                     map_pointer_t const ProvideVersion,
1140                                     uint8_t const Flags)
1141 {
1142    // Get a structure
1143    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1144    if (unlikely(Provides == 0))
1145       return false;
1146    ++Cache.HeaderP->ProvidesCount;
1147
1148    // Fill it in
1149    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1150    Prv->Version = Ver.Index();
1151    Prv->ProvideVersion = ProvideVersion;
1152    Prv->Flags = Flags;
1153    Prv->NextPkgProv = Ver->ProvidesList;
1154    Ver->ProvidesList = Prv.Index();
1155
1156    // Link it to the package
1157    Prv->ParentPkg = Pkg.Index();
1158    Prv->NextProvides = Pkg->ProvidesList;
1159    Pkg->ProvidesList = Prv.Index();
1160    return true;
1161 }
1162                                                                         /*}}}*/
1163 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1164 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1165                                 StringView Version, uint8_t const Flags) {
1166    pkgCache &Cache = Owner->Cache;
1167    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1168    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1169    Dynamic<StringView> DynPackage(Package);
1170    Dynamic<StringView> DynVersion(Version);
1171
1172    if (Grp.end() == true)
1173       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1174    else
1175    {
1176       map_stringitem_t idxProvideVersion = 0;
1177       if (Version.empty() == false) {
1178          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1179          if (unlikely(idxProvideVersion == 0))
1180             return false;
1181       }
1182
1183       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1184       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1185       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1186       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1187       pkgCache::PkgIterator Pkg;
1188       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1189       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1190       {
1191          if (isImplicit && OwnerPkg == Pkg)
1192             continue;
1193          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1194             continue;
1195          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1196             return false;
1197       }
1198    }
1199    return true;
1200 }
1201                                                                         /*}}}*/
1202 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1203       pkgCache::VerIterator const &Ver)
1204 {
1205    return Hash == Ver->Hash;
1206 }
1207                                                                         /*}}}*/
1208 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1209 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1210                                    unsigned long Flags)
1211 {
1212    if (File.empty() && Site.empty())
1213    {
1214       CurrentRlsFile = NULL;
1215       return true;
1216    }
1217
1218    // Get some space for the structure
1219    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1220    if (unlikely(idxFile == 0))
1221       return false;
1222    CurrentRlsFile = Cache.RlsFileP + idxFile;
1223
1224    // Fill it in
1225    map_stringitem_t const idxFileName = WriteStringInMap(File);
1226    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1227    if (unlikely(idxFileName == 0 || idxSite == 0))
1228       return false;
1229    CurrentRlsFile->FileName = idxFileName;
1230    CurrentRlsFile->Site = idxSite;
1231    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1232    CurrentRlsFile->Flags = Flags;
1233    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1234    RlsFileName = File;
1235    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1236    Cache.HeaderP->ReleaseFileCount++;
1237
1238    return true;
1239 }
1240                                                                         /*}}}*/
1241 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1242 // ---------------------------------------------------------------------
1243 /* This is used to select which file is to be associated with all newly
1244    added versions. The caller is responsible for setting the IMS fields. */
1245 bool pkgCacheGenerator::SelectFile(std::string const &File,
1246                                    pkgIndexFile const &Index,
1247                                    std::string const &Architecture,
1248                                    std::string const &Component,
1249                                    unsigned long const Flags)
1250 {
1251    // Get some space for the structure
1252    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1253    if (unlikely(idxFile == 0))
1254       return false;
1255    CurrentFile = Cache.PkgFileP + idxFile;
1256
1257    // Fill it in
1258    map_stringitem_t const idxFileName = WriteStringInMap(File);
1259    if (unlikely(idxFileName == 0))
1260       return false;
1261    CurrentFile->FileName = idxFileName;
1262    CurrentFile->NextFile = Cache.HeaderP->FileList;
1263    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1264    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1265    if (unlikely(idxIndexType == 0))
1266       return false;
1267    CurrentFile->IndexType = idxIndexType;
1268    if (Architecture.empty())
1269       CurrentFile->Architecture = 0;
1270    else
1271    {
1272       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1273       if (unlikely(arch == 0))
1274          return false;
1275       CurrentFile->Architecture = arch;
1276    }
1277    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1278    if (unlikely(component == 0))
1279       return false;
1280    CurrentFile->Component = component;
1281    CurrentFile->Flags = Flags;
1282    if (CurrentRlsFile != NULL)
1283       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1284    else
1285       CurrentFile->Release = 0;
1286    PkgFileName = File;
1287    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1288    Cache.HeaderP->PackageFileCount++;
1289
1290    if (Progress != 0)
1291       Progress->SubProgress(Index.Size());
1292    return true;
1293 }
1294                                                                         /*}}}*/
1295 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1296 // ---------------------------------------------------------------------
1297 /* This is used to create handles to strings. Given the same text it
1298    always returns the same number */
1299 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1300                                                  unsigned int Size)
1301 {
1302    auto strings = &strMixed;
1303    switch(type) {
1304       case MIXED: strings = &strMixed; break;
1305       case PKGNAME: strings = &strPkgNames; break;
1306       case VERSIONNUMBER: strings = &strVersions; break;
1307       case SECTION: strings = &strSections; break;
1308       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1309    }
1310
1311    auto const item = strings->find({S, Size, nullptr, 0});
1312    if (item != strings->end())
1313       return item->item;
1314
1315    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1316    strings->insert({nullptr, Size, this, idxString});
1317    return idxString;
1318 }
1319                                                                         /*}}}*/
1320 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1321 // ---------------------------------------------------------------------
1322 /* This just verifies that each file in the list of index files exists,
1323    has matching attributes with the cache and the cache does not have
1324    any extra files. */
1325 class APT_HIDDEN ScopedErrorRevert {
1326 public:
1327    ScopedErrorRevert() { _error->PushToStack(); }
1328    ~ScopedErrorRevert() { _error->RevertToStack(); }
1329 };
1330 static bool CheckValidity(const string &CacheFile,
1331                           pkgSourceList &List,
1332                           FileIterator const Start,
1333                           FileIterator const End,
1334                           MMap **OutMap = 0,
1335                           pkgCache **OutCache = 0)
1336 {
1337    ScopedErrorRevert ser;
1338    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1339    // No file, certainly invalid
1340    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1341    {
1342       if (Debug == true)
1343          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1344       return false;
1345    }
1346
1347    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1348    {
1349       if (Debug == true)
1350          std::clog << "sources.list is newer than the cache" << std::endl;
1351       return false;
1352    }
1353
1354    // Map it
1355    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1356    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1357    if (unlikely(Map->validData()) == false)
1358       return false;
1359    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1360    pkgCache &Cache = *CacheP.get();
1361    if (_error->PendingError() || Map->Size() == 0)
1362    {
1363       if (Debug == true)
1364          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1365       return false;
1366    }
1367
1368    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1369    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1370    std::vector<pkgIndexFile *> Files;
1371    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1372    {
1373       if (Debug == true)
1374          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1375       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1376       if (RlsFile.end() == true)
1377       {
1378          if (Debug == true)
1379             std::clog << "FindInCache returned end-Pointer" << std::endl;
1380          return false;
1381       }
1382
1383       RlsVisited[RlsFile->ID] = true;
1384       if (Debug == true)
1385          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1386
1387       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1388       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1389             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1390    }
1391    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1392       if (RlsVisited[I] == false)
1393       {
1394          if (Debug == true)
1395             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1396          return false;
1397       }
1398
1399    std::copy(Start, End, std::back_inserter(Files));
1400
1401    /* Now we check every index file, see if it is in the cache,
1402       verify the IMS data and check that it is on the disk too.. */
1403    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1404    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1405    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1406    {
1407       if (Debug == true)
1408          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1409       if ((*PkgFile)->Exists() == false)
1410       {
1411          if (Debug == true)
1412             std::clog << "file doesn't exist" << std::endl;
1413          continue;
1414       }
1415
1416       // FindInCache is also expected to do an IMS check.
1417       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1418       if (File.end() == true)
1419       {
1420          if (Debug == true)
1421             std::clog << "FindInCache returned end-Pointer" << std::endl;
1422          return false;
1423       }
1424
1425       Visited[File->ID] = true;
1426       if (Debug == true)
1427          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1428    }
1429
1430    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1431       if (Visited[I] == false)
1432       {
1433          if (Debug == true)
1434             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1435          return false;
1436       }
1437
1438    if (_error->PendingError() == true)
1439    {
1440       if (Debug == true)
1441       {
1442          std::clog << "Validity failed because of pending errors:" << std::endl;
1443          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1444       }
1445       return false;
1446    }
1447
1448    if (OutMap != 0)
1449       *OutMap = Map.release();
1450    if (OutCache != 0)
1451       *OutCache = CacheP.release();
1452    return true;
1453 }
1454                                                                         /*}}}*/
1455 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1456 // ---------------------------------------------------------------------
1457 /* Size is kind of an abstract notion that is only used for the progress
1458    meter */
1459 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1460 {
1461    map_filesize_t TotalSize = 0;
1462    if (List !=  NULL)
1463    {
1464       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1465       {
1466          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1467          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1468             if ((*j)->HasPackages() == true)
1469                TotalSize += (*j)->Size();
1470       }
1471    }
1472
1473    for (; Start < End; ++Start)
1474    {
1475       if ((*Start)->HasPackages() == false)
1476          continue;
1477       TotalSize += (*Start)->Size();
1478    }
1479    return TotalSize;
1480 }
1481                                                                         /*}}}*/
1482 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1483 static bool BuildCache(pkgCacheGenerator &Gen,
1484                        OpProgress * const Progress,
1485                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1486                        pkgSourceList const * const List,
1487                        FileIterator const Start, FileIterator const End)
1488 {
1489    bool mergeFailure = false;
1490
1491    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1492       if (I->HasPackages() == false || mergeFailure)
1493          return;
1494
1495       if (I->Exists() == false)
1496          return;
1497
1498       if (I->FindInCache(Gen.GetCache()).end() == false)
1499       {
1500          _error->Warning("Duplicate sources.list entry %s",
1501                I->Describe().c_str());
1502          return;
1503       }
1504
1505       map_filesize_t const Size = I->Size();
1506       if (Progress != NULL)
1507          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1508       CurrentSize += Size;
1509
1510       if (I->Merge(Gen,Progress) == false)
1511          mergeFailure = true;
1512    };
1513
1514    if (List !=  NULL)
1515    {
1516       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1517       {
1518          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1519          {
1520             _error->Warning("Duplicate sources.list entry %s",
1521                   (*i)->Describe().c_str());
1522             continue;
1523          }
1524
1525          if ((*i)->Merge(Gen, Progress) == false)
1526             return false;
1527
1528          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1529          if (Indexes != NULL)
1530             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1531          if (mergeFailure)
1532             return false;
1533       }
1534    }
1535
1536    if (Start != End)
1537    {
1538       Gen.SelectReleaseFile("", "");
1539       std::for_each(Start, End, indexFileMerge);
1540       if (mergeFailure)
1541          return false;
1542    }
1543    return true;
1544 }
1545                                                                         /*}}}*/
1546 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1547 // ---------------------------------------------------------------------
1548 /* This makes sure that the status cache (the cache that has all
1549    index files from the sources list and all local ones) is ready
1550    to be mmaped. If OutMap is not zero then a MMap object representing
1551    the cache will be stored there. This is pretty much mandetory if you
1552    are using AllowMem. AllowMem lets the function be run as non-root
1553    where it builds the cache 'fast' into a memory buffer. */
1554 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1555 {
1556    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1557    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1558    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1559    Flags |= MMap::Moveable;
1560    if (_config->FindB("APT::Cache-Fallback", false) == true)
1561       Flags |= MMap::Fallback;
1562    if (CacheF != NULL)
1563       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1564    else
1565       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1566 }
1567 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1568       std::string const &FileName)
1569 {
1570    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1571    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1572       return false;
1573
1574    fchmod(SCacheF.Fd(),0644);
1575
1576    // Write out the main data
1577    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1578       return _error->Error(_("IO Error saving source cache"));
1579
1580    // Write out the proper header
1581    Gen->GetCache().HeaderP->Dirty = false;
1582    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1583    if (SCacheF.Seek(0) == false ||
1584          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1585       return _error->Error(_("IO Error saving source cache"));
1586    Gen->GetCache().HeaderP->Dirty = true;
1587    return true;
1588 }
1589 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1590       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1591 {
1592    Map.reset(CreateDynamicMMap(NULL, 0));
1593    if (unlikely(Map->validData()) == false)
1594       return false;
1595    FileFd CacheF(FileName, FileFd::ReadOnly);
1596    if (CacheF.IsOpen() == false || CacheF.Failed())
1597       return false;
1598    _error->PushToStack();
1599    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1600    bool const newError = _error->PendingError();
1601    _error->MergeWithStack();
1602    if (alloc == 0 && newError)
1603       return false;
1604    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1605       return false;
1606    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1607    return true;
1608 }
1609 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1610                         MMap **OutMap, bool AllowMem)
1611    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1612 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1613                         MMap **OutMap,bool)
1614 {
1615    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1616 }
1617 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1618                         MMap **OutMap,pkgCache **OutCache, bool)
1619 {
1620    // FIXME: deprecate the ignored AllowMem parameter
1621    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1622
1623    std::vector<pkgIndexFile *> Files;
1624    if (_system->AddStatusFiles(Files) == false)
1625       return false;
1626
1627    // Decide if we can write to the files..
1628    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1629    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1630
1631    // ensure the cache directory exists
1632    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1633    {
1634       string dir = _config->FindDir("Dir::Cache");
1635       size_t const len = dir.size();
1636       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1637          dir = dir.substr(0, len - 5);
1638       if (CacheFile.empty() == false)
1639          CreateDirectory(dir, flNotFile(CacheFile));
1640       if (SrcCacheFile.empty() == false)
1641          CreateDirectory(dir, flNotFile(SrcCacheFile));
1642    }
1643
1644    if (Progress != NULL)
1645       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1646
1647    bool pkgcache_fine = false;
1648    bool srcpkgcache_fine = false;
1649    bool volatile_fine = List.GetVolatileFiles().empty();
1650
1651    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1652                      volatile_fine ? OutCache : NULL) == true)
1653    {
1654       if (Debug == true)
1655          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1656       pkgcache_fine = true;
1657       srcpkgcache_fine = true;
1658    }
1659    if (pkgcache_fine == false)
1660    {
1661       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1662       {
1663          if (Debug == true)
1664             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1665          srcpkgcache_fine = true;
1666       }
1667    }
1668
1669    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1670    {
1671       if (Progress != NULL)
1672          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1673       return true;
1674    }
1675
1676    bool Writeable = false;
1677    if (srcpkgcache_fine == false || pkgcache_fine == false)
1678    {
1679       if (CacheFile.empty() == false)
1680          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1681       else if (SrcCacheFile.empty() == false)
1682          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1683
1684       if (Debug == true)
1685          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1686    }
1687
1688    // At this point we know we need to construct something, so get storage ready
1689    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1690    if (unlikely(Map->validData()) == false)
1691       return false;
1692    if (Debug == true)
1693       std::clog << "Open memory Map (not filebased)" << std::endl;
1694
1695    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1696    map_filesize_t CurrentSize = 0;
1697    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1698    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1699    if (srcpkgcache_fine == true && pkgcache_fine == false)
1700    {
1701       if (Debug == true)
1702          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1703       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1704          return false;
1705       srcpkgcache_fine = true;
1706       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1707    }
1708    else if (srcpkgcache_fine == false)
1709    {
1710       if (Debug == true)
1711          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1712       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1713
1714       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1715       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1716                Files.end(),Files.end()) == false)
1717          return false;
1718
1719       if (Writeable == true && SrcCacheFile.empty() == false)
1720          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1721             return false;
1722    }
1723
1724    if (pkgcache_fine == false)
1725    {
1726       if (Debug == true)
1727          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1728       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1729                Files.begin(), Files.end()) == false)
1730          return false;
1731
1732       if (Writeable == true && CacheFile.empty() == false)
1733          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1734             return false;
1735    }
1736
1737    if (Debug == true)
1738       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1739
1740    if (volatile_fine == false)
1741    {
1742       if (Gen == nullptr)
1743       {
1744          if (Debug == true)
1745             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1746          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1747             return false;
1748       }
1749
1750       Files = List.GetVolatileFiles();
1751       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1752                Files.begin(), Files.end()) == false)
1753          return false;
1754    }
1755
1756    if (OutMap != nullptr)
1757       *OutMap = Map.release();
1758
1759    if (Debug == true)
1760       std::clog << "Everything is ready for shipping" << std::endl;
1761    return true;
1762 }
1763                                                                         /*}}}*/
1764 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1765 class APT_HIDDEN ScopedErrorMerge {
1766 public:
1767    ScopedErrorMerge() { _error->PushToStack(); }
1768    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1769 };
1770 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1771    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1772 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1773 {
1774    std::vector<pkgIndexFile *> Files;
1775    if (_system->AddStatusFiles(Files) == false)
1776       return false;
1777
1778    ScopedErrorMerge sem;
1779    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1780    if (unlikely(Map->validData()) == false)
1781       return false;
1782    map_filesize_t CurrentSize = 0;
1783    map_filesize_t TotalSize = 0;
1784    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1785
1786    // Build the status cache
1787    if (Progress != NULL)
1788       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1789    pkgCacheGenerator Gen(Map.get(),Progress);
1790    if (_error->PendingError() == true)
1791       return false;
1792    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1793                   Files.begin(), Files.end()) == false)
1794       return false;
1795
1796    if (_error->PendingError() == true)
1797       return false;
1798    *OutMap = Map.release();
1799
1800    return true;
1801 }
1802                                                                         /*}}}*/
1803 // IsDuplicateDescription                                               /*{{{*/
1804 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1805                             MD5SumValue const &CurMd5, std::string const &CurLang)
1806 {
1807    // Descriptions in the same link-list have all the same md5
1808    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1809       return false;
1810    for (; Desc.end() == false; ++Desc)
1811       if (Desc.LanguageCode() == CurLang)
1812          return true;
1813    return false;
1814 }
1815                                                                         /*}}}*/
1816
1817 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1818 pkgCacheListParser::~pkgCacheListParser() {}