apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180 }                                                                       /*}}}*/
 181 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 182 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 183                                         const unsigned long &Len) {
 184    size_t oldSize = Map.Size();
 185    void const * const oldMap = Map.Data();
 186    map_stringitem_t const index = Map.WriteString(String, Len);
 187    if (index != 0)
 188       ReMap(oldMap, Map.Data(), oldSize);
 189    return index;
 190 }
 191                                                                         /*}}}*/
 192 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 193 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 194    size_t oldSize = Map.Size();
 195    void const * const oldMap = Map.Data();
 196    map_stringitem_t const index = Map.WriteString(String);
 197    if (index != 0)
 198       ReMap(oldMap, Map.Data(), oldSize);
 199    return index;
 200 }
 201                                                                         /*}}}*/
 202 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 203    size_t oldSize = Map.Size();
 204    void const * const oldMap = Map.Data();
 205    map_pointer_t const index = Map.Allocate(size);
 206    if (index != 0)
 207       ReMap(oldMap, Map.Data(), oldSize);
 208    return index;
 209 }
 210                                                                         /*}}}*/
 211 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 212 // ---------------------------------------------------------------------
 213 /* This provides the generation of the entries in the cache. Each loop
 214    goes through a single package record from the underlying parse engine. */
 215 bool pkgCacheGenerator::MergeList(ListParser &List,
 216                                   pkgCache::VerIterator *OutVer)
 217 {
 218    List.Owner = this;
 219
 220    unsigned int Counter = 0;
 221    while (List.Step() == true)
 222    {
 223       string const PackageName = List.Package();
 224       if (PackageName.empty() == true)
 225          return false;
 226
 227       Counter++;
 228       if (Counter % 100 == 0 && Progress != 0)
 229          Progress->Progress(List.Offset());
 230
 231       string Arch = List.Architecture();
 232       string const Version = List.Version();
 233       if (Version.empty() == true && Arch.empty() == true)
 234       {
 235          // package descriptions
 236          if (MergeListGroup(List, PackageName) == false)
 237             return false;
 238          continue;
 239       }
 240
 241       // Get a pointer to the package structure
 242       pkgCache::PkgIterator Pkg;
 243       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 244       if (NewPackage(Pkg, PackageName, Arch) == false)
 245          // TRANSLATOR: The first placeholder is a package name,
 246          // the other two should be copied verbatim as they include debug info
 247          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 248                               PackageName.c_str(), "NewPackage", 1);
 249
 250
 251       if (Version.empty() == true)
 252       {
 253          if (MergeListPackage(List, Pkg) == false)
 254             return false;
 255       }
 256       else
 257       {
 258          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 259             return false;
 260       }
 261
 262       if (OutVer != 0)
 263          return true;
 264    }
 265
 266    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of package "
 268                              "names this APT is capable of."));
 269    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 270       return _error->Error(_("Wow, you exceeded the number of versions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 273       return _error->Error(_("Wow, you exceeded the number of descriptions "
 274                              "this APT is capable of."));
 275    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 276       return _error->Error(_("Wow, you exceeded the number of dependencies "
 277                              "this APT is capable of."));
 278
 279    return true;
 280 }
 281 // CacheGenerator::MergeListGroup                                       /*{{{*/
 282 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 283 {
 284    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 285    // a group has no data on it's own, only packages have it but these
 286    // stanzas like this come from Translation- files to add descriptions,
 287    // but without a version we don't need a description for it…
 288    if (Grp.end() == true)
 289       return true;
 290    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 291
 292    pkgCache::PkgIterator Pkg;
 293    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 294    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 295       if (MergeListPackage(List, Pkg) == false)
 296          return false;
 297
 298    return true;
 299 }
 300                                                                         /*}}}*/
 301 // CacheGenerator::MergeListPackage                                     /*{{{*/
 302 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 303 {
 304    // we first process the package, then the descriptions
 305    // (for deb this package processing is in fact a no-op)
 306    pkgCache::VerIterator Ver(Cache);
 307    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 308    if (List.UsePackage(Pkg, Ver) == false)
 309       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 310                            Pkg.Name(), "UsePackage", 1);
 311
 312    // Find the right version to write the description
 313    MD5SumValue CurMd5 = List.Description_md5();
 314    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 315    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 316    {
 317       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 318
 319       // a version can only have one md5 describing it
 320       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 321          continue;
 322
 323       map_stringitem_t md5idx = VerDesc->md5sum;
 324       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 325       {
 326          // don't add a new description if we have one for the given
 327          // md5 && language
 328          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 329             continue;
 330
 331          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 332       }
 333
 334       // we can stop here as all "same" versions will share the description
 335       break;
 336    }
 337
 338    return true;
 339 }
 340                                                                         /*}}}*/
 341 // CacheGenerator::MergeListVersion                                     /*{{{*/
 342 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 343                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 344 {
 345    pkgCache::VerIterator Ver = Pkg.VersionList();
 346    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 347    map_pointer_t *LastVer = &Pkg->VersionList;
 348    void const * oldMap = Map.Data();
 349
 350    unsigned short const Hash = List.VersionHash();
 351    if (Ver.end() == false)
 352    {
 353       /* We know the list is sorted so we use that fact in the search.
 354          Insertion of new versions is done with correct sorting */
 355       int Res = 1;
 356       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 357       {
 358          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 359          // Version is higher as current version - insert here
 360          if (Res > 0)
 361             break;
 362          // Versionstrings are equal - is hash also equal?
 363          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 364             break;
 365          // proceed with the next till we have either the right
 366          // or we found another version (which will be lower)
 367       }
 368
 369       /* We already have a version for this item, record that we saw it */
 370       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 371       {
 372          if (List.UsePackage(Pkg,Ver) == false)
 373             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 374                                  Pkg.Name(), "UsePackage", 2);
 375
 376          if (NewFileVer(Ver,List) == false)
 377             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 378                                  Pkg.Name(), "NewFileVer", 1);
 379
 380          // Read only a single record and return
 381          if (OutVer != 0)
 382          {
 383             *OutVer = Ver;
 384             return true;
 385          }
 386
 387          return true;
 388       }
 389    }
 390
 391    // Add a new version
 392    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 393    if (unlikely(verindex == 0))
 394       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 395                            Pkg.Name(), "NewVersion", 1);
 396
 397    if (oldMap != Map.Data())
 398          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 399    *LastVer = verindex;
 400
 401    if (unlikely(List.NewVersion(Ver) == false))
 402       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 403                            Pkg.Name(), "NewVersion", 2);
 404
 405    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 406       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 407                            Pkg.Name(), "UsePackage", 3);
 408
 409    if (unlikely(NewFileVer(Ver,List) == false))
 410       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 411                            Pkg.Name(), "NewFileVer", 2);
 412
 413    pkgCache::GrpIterator Grp = Pkg.Group();
 414    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 415
 416    /* If it is the first version of this package we need to add implicit
 417       Multi-Arch dependencies to all other package versions in the group now -
 418       otherwise we just add them for this new version */
 419    if (Pkg.VersionList()->NextVer == 0)
 420    {
 421       pkgCache::PkgIterator P = Grp.PackageList();
 422       Dynamic<pkgCache::PkgIterator> DynP(P);
 423       for (; P.end() != true; P = Grp.NextPkg(P))
 424       {
 425          if (P->ID == Pkg->ID)
 426             continue;
 427          pkgCache::VerIterator V = P.VersionList();
 428          Dynamic<pkgCache::VerIterator> DynV(V);
 429          for (; V.end() != true; ++V)
 430             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 431                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 432                                     Pkg.Name(), "AddImplicitDepends", 1);
 433       }
 434    }
 435    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 436       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 437                            Pkg.Name(), "AddImplicitDepends", 2);
 438
 439    // Read only a single record and return
 440    if (OutVer != 0)
 441    {
 442       *OutVer = Ver;
 443       return true;
 444    }
 445
 446    /* Record the Description(s) based on their master md5sum */
 447    MD5SumValue CurMd5 = List.Description_md5();
 448
 449    /* Before we add a new description we first search in the group for
 450       a version with a description of the same MD5 - if so we reuse this
 451       description group instead of creating our own for this version */
 452    for (pkgCache::PkgIterator P = Grp.PackageList();
 453         P.end() == false; P = Grp.NextPkg(P))
 454    {
 455       for (pkgCache::VerIterator V = P.VersionList();
 456            V.end() == false; ++V)
 457       {
 458          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 459             continue;
 460          Ver->DescriptionList = V->DescriptionList;
 461       }
 462    }
 463
 464    // We haven't found reusable descriptions, so add the first description(s)
 465    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 466    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 467    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 468       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 469          return false;
 470    return true;
 471 }
 472                                                                         /*}}}*/
 473 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 474 {
 475    pkgCache::DescIterator Desc;
 476    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 477
 478    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 479    if (unlikely(descindex == 0))
 480       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 481             Ver.ParentPkg().Name(), "NewDescription", 1);
 482
 483    md5idx = Desc->md5sum;
 484    Desc->ParentPkg = Ver.ParentPkg().Index();
 485
 486    // we add at the end, so that the start is constant as we need
 487    // that to be able to efficiently share these lists
 488    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 489    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 490    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 491    *LastNextDesc = descindex;
 492
 493    if (NewFileDesc(Desc,List) == false)
 494       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 495             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 496
 497    return true;
 498 }
 499                                                                         /*}}}*/
 500                                                                         /*}}}*/
 501 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 502 // ---------------------------------------------------------------------
 503 /* This creates a new group structure and adds it to the hash table */
 504 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 505 {
 506    Grp = Cache.FindGrp(Name);
 507    if (Grp.end() == false)
 508       return true;
 509
 510    // Get a structure
 511    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 512    if (unlikely(Group == 0))
 513       return false;
 514
 515    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 516    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 517    if (unlikely(idxName == 0))
 518       return false;
 519    Grp->Name = idxName;
 520
 521    // Insert it into the hash table
 522    unsigned long const Hash = Cache.Hash(Name);
 523    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 524
 525    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 526       insertAt = &(Cache.GrpP + *insertAt)->Next;
 527    Grp->Next = *insertAt;
 528    *insertAt = Group;
 529
 530    Grp->ID = Cache.HeaderP->GroupCount++;
 531    return true;
 532 }
 533                                                                         /*}}}*/
 534 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 535 // ---------------------------------------------------------------------
 536 /* This creates a new package structure and adds it to the hash table */
 537 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 538                                         StringView Arch) {
 539    pkgCache::GrpIterator Grp;
 540    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 541    if (unlikely(NewGroup(Grp, Name) == false))
 542       return false;
 543
 544    Pkg = Grp.FindPkg(Arch);
 545       if (Pkg.end() == false)
 546          return true;
 547
 548    // Get a structure
 549    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 550    if (unlikely(Package == 0))
 551       return false;
 552    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 553
 554    // Set the name, arch and the ID
 555    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 556    Pkg->Group = Grp.Index();
 557    // all is mapped to the native architecture
 558    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 559    if (unlikely(idxArch == 0))
 560       return false;
 561    Pkg->Arch = idxArch;
 562    Pkg->ID = Cache.HeaderP->PackageCount++;
 563
 564    // Insert the package into our package list
 565    if (Grp->FirstPackage == 0) // the group is new
 566    {
 567       Grp->FirstPackage = Package;
 568       // Insert it into the hash table
 569       map_id_t const Hash = Cache.Hash(Name);
 570       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 571       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 572          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 573       Pkg->NextPackage = *insertAt;
 574       *insertAt = Package;
 575    }
 576    else // Group the Packages together
 577    {
 578       // if sibling is provided by another package, this one is too
 579       {
 580          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 581          if (M.end() == false) {
 582             pkgCache::PrvIterator Prv;
 583             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 584             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 585             {
 586                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 587                   continue;
 588                pkgCache::VerIterator Ver = Prv.OwnerVer();
 589                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 590                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 591                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 592                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 593                {
 594                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 595                      continue;
 596                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 597                      return false;
 598                }
 599             }
 600          }
 601       }
 602       // let M-A:foreign package siblings provide this package
 603       {
 604          pkgCache::PkgIterator P;
 605          pkgCache::VerIterator Ver;
 606          Dynamic<pkgCache::PkgIterator> DynP(P);
 607          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 608
 609          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 610          {
 611             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 612                continue;
 613             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 614                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 615                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 616                      return false;
 617          }
 618       }
 619       // and negative dependencies, don't forget negative dependencies
 620       {
 621          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 622          if (M.end() == false) {
 623             pkgCache::DepIterator Dep;
 624             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 625             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 626             {
 627                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 628                   continue;
 629                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 630                      Dep->Type != pkgCache::Dep::Replaces)
 631                   continue;
 632                pkgCache::VerIterator Ver = Dep.ParentVer();
 633                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 634                map_pointer_t * unused = NULL;
 635                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 636                   return false;
 637             }
 638          }
 639       }
 640
 641       // this package is the new last package
 642       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 643       Pkg->NextPackage = LastPkg->NextPackage;
 644       LastPkg->NextPackage = Package;
 645    }
 646    Grp->LastPackage = Package;
 647
 648    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 649    if (Arch == "any")
 650    {
 651       size_t const found = Name.find(':');
 652       StringView const NameA = Name.substr(0, found);
 653       StringView const ArchA = Name.substr(found + 1);
 654       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 655       if (PkgA.end() == false)
 656       {
 657          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 658          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 659          for (; Prv.end() == false; ++Prv)
 660          {
 661             if (Prv.IsMultiArchImplicit())
 662                continue;
 663             pkgCache::VerIterator V = Prv.OwnerVer();
 664             if (ArchA != V.ParentPkg().Arch())
 665                continue;
 666             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 667                return false;
 668          }
 669          pkgCache::VerIterator V = PkgA.VersionList();
 670          Dynamic<pkgCache::VerIterator> DynV(V);
 671          for (; V.end() == false; ++V)
 672          {
 673             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 674                return false;
 675          }
 676       }
 677    }
 678    return true;
 679 }
 680                                                                         /*}}}*/
 681 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 682 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 683                                            pkgCache::PkgIterator &P,
 684                                            pkgCache::VerIterator &V)
 685 {
 686    // copy P.Arch() into a string here as a cache remap
 687    // in NewDepends() later may alter the pointer location
 688    string Arch = P.Arch() == NULL ? "" : P.Arch();
 689    map_pointer_t *OldDepLast = NULL;
 690    /* MultiArch handling introduces a lot of implicit Dependencies:
 691       - MultiArch: same → Co-Installable if they have the same version
 692       - All others conflict with all other group members */
 693    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 694    pkgCache::PkgIterator D = G.PackageList();
 695    Dynamic<pkgCache::PkgIterator> DynD(D);
 696    map_stringitem_t const VerStrIdx = V->VerStr;
 697    for (; D.end() != true; D = G.NextPkg(D))
 698    {
 699       if (Arch == D.Arch() || D->VersionList == 0)
 700          continue;
 701       /* We allow only one installed arch at the time
 702          per group, therefore each group member conflicts
 703          with all other group members */
 704       if (coInstall == true)
 705       {
 706          // Replaces: ${self}:other ( << ${binary:Version})
 707          NewDepends(D, V, VerStrIdx,
 708                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 709                     OldDepLast);
 710          // Breaks: ${self}:other (!= ${binary:Version})
 711          NewDepends(D, V, VerStrIdx,
 712                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 713                     OldDepLast);
 714       } else {
 715          // Conflicts: ${self}:other
 716          NewDepends(D, V, 0,
 717                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 718                     OldDepLast);
 719       }
 720    }
 721    return true;
 722 }
 723 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 724                                            pkgCache::PkgIterator &D)
 725 {
 726    /* MultiArch handling introduces a lot of implicit Dependencies:
 727       - MultiArch: same → Co-Installable if they have the same version
 728       - All others conflict with all other group members */
 729    map_pointer_t *OldDepLast = NULL;
 730    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 731    if (coInstall == true)
 732    {
 733       map_stringitem_t const VerStrIdx = V->VerStr;
 734       // Replaces: ${self}:other ( << ${binary:Version})
 735       NewDepends(D, V, VerStrIdx,
 736                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 737                  OldDepLast);
 738       // Breaks: ${self}:other (!= ${binary:Version})
 739       NewDepends(D, V, VerStrIdx,
 740                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 741                  OldDepLast);
 742    } else {
 743       // Conflicts: ${self}:other
 744       NewDepends(D, V, 0,
 745                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 746                  OldDepLast);
 747    }
 748    return true;
 749 }
 750
 751                                                                         /*}}}*/
 752 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 753 // ---------------------------------------------------------------------
 754 /* */
 755 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 756                                    ListParser &List)
 757 {
 758    if (CurrentFile == 0)
 759       return true;
 760
 761    // Get a structure
 762    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 763    if (VerFile == 0)
 764       return false;
 765
 766    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 767    VF->File = CurrentFile - Cache.PkgFileP;
 768
 769    // Link it to the end of the list
 770    map_pointer_t *Last = &Ver->FileList;
 771    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 772       Last = &V->NextFile;
 773    VF->NextFile = *Last;
 774    *Last = VF.Index();
 775
 776    VF->Offset = List.Offset();
 777    VF->Size = List.Size();
 778    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 779       Cache.HeaderP->MaxVerFileSize = VF->Size;
 780    Cache.HeaderP->VerFileCount++;
 781
 782    return true;
 783 }
 784                                                                         /*}}}*/
 785 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* This puts a version structure in the linked list */
 788 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 789                                             const string &VerStr,
 790                                             map_pointer_t const ParentPkg,
 791                                             unsigned short const Hash,
 792                                             map_pointer_t const Next)
 793 {
 794    // Get a structure
 795    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 796    if (Version == 0)
 797       return 0;
 798
 799    // Fill it in
 800    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 801    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 802    Ver->NextVer = Next;
 803    Ver->ParentPkg = ParentPkg;
 804    Ver->Hash = Hash;
 805    Ver->ID = Cache.HeaderP->VersionCount++;
 806
 807    // try to find the version string in the group for reuse
 808    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 809    pkgCache::GrpIterator Grp = Pkg.Group();
 810    if (Pkg.end() == false && Grp.end() == false)
 811    {
 812       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 813       {
 814          if (Pkg == P)
 815             continue;
 816          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 817          {
 818             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 819             if (cmp == 0)
 820             {
 821                Ver->VerStr = V->VerStr;
 822                return Version;
 823             }
 824             else if (cmp < 0)
 825                break;
 826          }
 827       }
 828    }
 829    // haven't found the version string, so create
 830    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 831    if (unlikely(idxVerStr == 0))
 832       return 0;
 833    Ver->VerStr = idxVerStr;
 834    return Version;
 835 }
 836                                                                         /*}}}*/
 837 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 838 // ---------------------------------------------------------------------
 839 /* */
 840 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 841                                    ListParser &List)
 842 {
 843    if (CurrentFile == 0)
 844       return true;
 845
 846    // Get a structure
 847    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 848    if (DescFile == 0)
 849       return false;
 850
 851    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 852    DF->File = CurrentFile - Cache.PkgFileP;
 853
 854    // Link it to the end of the list
 855    map_pointer_t *Last = &Desc->FileList;
 856    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 857       Last = &D->NextFile;
 858
 859    DF->NextFile = *Last;
 860    *Last = DF.Index();
 861
 862    DF->Offset = List.Offset();
 863    DF->Size = List.Size();
 864    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 865       Cache.HeaderP->MaxDescFileSize = DF->Size;
 866    Cache.HeaderP->DescFileCount++;
 867
 868    return true;
 869 }
 870                                                                         /*}}}*/
 871 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 872 // ---------------------------------------------------------------------
 873 /* This puts a description structure in the linked list */
 874 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 875                                             const string &Lang,
 876                                             const MD5SumValue &md5sum,
 877                                             map_stringitem_t const idxmd5str)
 878 {
 879    // Get a structure
 880    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 881    if (Description == 0)
 882       return 0;
 883
 884    // Fill it in
 885    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 886    Desc->ID = Cache.HeaderP->DescriptionCount++;
 887    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 888    if (unlikely(idxlanguage_code == 0))
 889       return 0;
 890    Desc->language_code = idxlanguage_code;
 891
 892    if (idxmd5str != 0)
 893       Desc->md5sum = idxmd5str;
 894    else
 895    {
 896       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 897       if (unlikely(idxmd5sum == 0))
 898          return 0;
 899       Desc->md5sum = idxmd5sum;
 900    }
 901
 902    return Description;
 903 }
 904                                                                         /*}}}*/
 905 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 906 // ---------------------------------------------------------------------
 907 /* This creates a dependency element in the tree. It is linked to the
 908    version and to the package that it is pointing to. */
 909 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 910                                    pkgCache::VerIterator &Ver,
 911                                    map_pointer_t const Version,
 912                                    uint8_t const Op,
 913                                    uint8_t const Type,
 914                                    map_pointer_t* &OldDepLast)
 915 {
 916    void const * const oldMap = Map.Data();
 917    // Get a structure
 918    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 919    if (unlikely(Dependency == 0))
 920       return false;
 921
 922    bool isDuplicate = false;
 923    map_pointer_t DependencyData = 0;
 924    map_pointer_t PreviousData = 0;
 925    if (Pkg->RevDepends != 0)
 926    {
 927       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 928       DependencyData = L->DependencyData;
 929       do {
 930          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 931          if (Version > D->Version)
 932             break;
 933          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 934          {
 935             isDuplicate = true;
 936             break;
 937          }
 938          PreviousData = DependencyData;
 939          DependencyData = D->NextData;
 940       } while (DependencyData != 0);
 941    }
 942
 943    if (isDuplicate == false)
 944    {
 945       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 946       if (unlikely(DependencyData == 0))
 947         return false;
 948    }
 949
 950    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 951    Link->ParentVer = Ver.Index();
 952    Link->DependencyData = DependencyData;
 953    Link->ID = Cache.HeaderP->DependsCount++;
 954
 955    pkgCache::DepIterator Dep(Cache, Link);
 956    if (isDuplicate == false)
 957    {
 958       Dep->Type = Type;
 959       Dep->CompareOp = Op;
 960       Dep->Version = Version;
 961       Dep->Package = Pkg.Index();
 962       ++Cache.HeaderP->DependsDataCount;
 963       if (PreviousData != 0)
 964       {
 965          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 966          Dep->NextData = D->NextData;
 967          D->NextData = DependencyData;
 968       }
 969       else if (Pkg->RevDepends != 0)
 970       {
 971          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 972          Dep->NextData = D->DependencyData;
 973       }
 974    }
 975
 976    if (isDuplicate == true || PreviousData != 0)
 977    {
 978       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 979       Link->NextRevDepends = L->NextRevDepends;
 980       L->NextRevDepends = Dependency;
 981    }
 982    else
 983    {
 984       Link->NextRevDepends = Pkg->RevDepends;
 985       Pkg->RevDepends = Dependency;
 986    }
 987
 988
 989    // Do we know where to link the Dependency to?
 990    if (OldDepLast == NULL)
 991    {
 992       OldDepLast = &Ver->DependsList;
 993       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 994          OldDepLast = &D->NextDepends;
 995    } else if (oldMap != Map.Data())
 996       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 997
 998    Dep->NextDepends = *OldDepLast;
 999    *OldDepLast = Dependency;
1000    OldDepLast = &Dep->NextDepends;
1001    return true;
1002 }
1003                                                                         /*}}}*/
1004 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1005 // ---------------------------------------------------------------------
1006 /* This creates a Group and the Package to link this dependency to if
1007    needed and handles also the caching of the old endpoint */
1008 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1009                                                StringView PackageName,
1010                                                StringView Arch,
1011                                                StringView Version,
1012                                                uint8_t const Op,
1013                                                uint8_t const Type)
1014 {
1015    pkgCache::GrpIterator Grp;
1016    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1017    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1018       return false;
1019
1020    map_stringitem_t idxVersion = 0;
1021    if (Version.empty() == false)
1022    {
1023       int const CmpOp = Op & 0x0F;
1024       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1025       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1026          idxVersion = Ver->VerStr;
1027
1028       if (idxVersion == 0)
1029       {
1030          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1031          if (unlikely(idxVersion == 0))
1032             return false;
1033       }
1034    }
1035
1036    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1037          Type == pkgCache::Dep::Conflicts ||
1038          Type == pkgCache::Dep::Replaces);
1039
1040    pkgCache::PkgIterator Pkg;
1041    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1042    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1043    {
1044       // Locate the target package
1045       Pkg = Grp.FindPkg(Arch);
1046       if (Pkg.end() == true) {
1047          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1048             return false;
1049       }
1050
1051       /* Caching the old end point speeds up generation substantially */
1052       if (OldDepVer != Ver) {
1053          OldDepLast = NULL;
1054          OldDepVer = Ver;
1055       }
1056
1057       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1058    }
1059    else
1060    {
1061       /* Caching the old end point speeds up generation substantially */
1062       if (OldDepVer != Ver) {
1063          OldDepLast = NULL;
1064          OldDepVer = Ver;
1065       }
1066
1067       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1068       {
1069          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1070                return false;
1071       }
1072    }
1073    return true;
1074 }
1075                                                                         /*}}}*/
1076 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1077 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1078                                                 StringView PkgName,
1079                                                 StringView PkgArch,
1080                                                 StringView Version,
1081                                                 uint8_t const Flags)
1082 {
1083    pkgCache const &Cache = Owner->Cache;
1084
1085    // We do not add self referencing provides
1086    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1087         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1088          (Version.empty() || Version == Ver.VerStr()))
1089       return true;
1090
1091    // Locate the target package
1092    pkgCache::PkgIterator Pkg;
1093    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1094    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1095       return false;
1096
1097    map_stringitem_t idxProvideVersion = 0;
1098    if (Version.empty() == false) {
1099       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1100       if (unlikely(idxProvideVersion == 0))
1101          return false;
1102    }
1103    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1104 }
1105 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1106                                     pkgCache::PkgIterator &Pkg,
1107                                     map_pointer_t const ProvideVersion,
1108                                     uint8_t const Flags)
1109 {
1110    // Get a structure
1111    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1112    if (unlikely(Provides == 0))
1113       return false;
1114    ++Cache.HeaderP->ProvidesCount;
1115
1116    // Fill it in
1117    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1118    Prv->Version = Ver.Index();
1119    Prv->ProvideVersion = ProvideVersion;
1120    Prv->Flags = Flags;
1121    Prv->NextPkgProv = Ver->ProvidesList;
1122    Ver->ProvidesList = Prv.Index();
1123
1124    // Link it to the package
1125    Prv->ParentPkg = Pkg.Index();
1126    Prv->NextProvides = Pkg->ProvidesList;
1127    Pkg->ProvidesList = Prv.Index();
1128    return true;
1129 }
1130                                                                         /*}}}*/
1131 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1132 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1133                                 StringView Version, uint8_t const Flags) {
1134    pkgCache &Cache = Owner->Cache;
1135    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1136    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1137
1138    if (Grp.end() == true)
1139       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1140    else
1141    {
1142       map_stringitem_t idxProvideVersion = 0;
1143       if (Version.empty() == false) {
1144          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1145          if (unlikely(idxProvideVersion == 0))
1146             return false;
1147       }
1148
1149       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1150       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1151       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1152       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1153       pkgCache::PkgIterator Pkg;
1154       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1155       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1156       {
1157          if (isImplicit && OwnerPkg == Pkg)
1158             continue;
1159          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1160             continue;
1161          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1162             return false;
1163       }
1164    }
1165    return true;
1166 }
1167                                                                         /*}}}*/
1168 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1169       pkgCache::VerIterator const &Ver)
1170 {
1171    return Hash == Ver->Hash;
1172 }
1173                                                                         /*}}}*/
1174 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1175 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1176                                    unsigned long Flags)
1177 {
1178    if (File.empty() && Site.empty())
1179    {
1180       CurrentRlsFile = NULL;
1181       return true;
1182    }
1183
1184    // Get some space for the structure
1185    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1186    if (unlikely(idxFile == 0))
1187       return false;
1188    CurrentRlsFile = Cache.RlsFileP + idxFile;
1189
1190    // Fill it in
1191    map_stringitem_t const idxFileName = WriteStringInMap(File);
1192    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1193    if (unlikely(idxFileName == 0 || idxSite == 0))
1194       return false;
1195    CurrentRlsFile->FileName = idxFileName;
1196    CurrentRlsFile->Site = idxSite;
1197    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1198    CurrentRlsFile->Flags = Flags;
1199    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1200    RlsFileName = File;
1201    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1202    Cache.HeaderP->ReleaseFileCount++;
1203
1204    return true;
1205 }
1206                                                                         /*}}}*/
1207 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1208 // ---------------------------------------------------------------------
1209 /* This is used to select which file is to be associated with all newly
1210    added versions. The caller is responsible for setting the IMS fields. */
1211 bool pkgCacheGenerator::SelectFile(std::string const &File,
1212                                    pkgIndexFile const &Index,
1213                                    std::string const &Architecture,
1214                                    std::string const &Component,
1215                                    unsigned long const Flags)
1216 {
1217    // Get some space for the structure
1218    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1219    if (unlikely(idxFile == 0))
1220       return false;
1221    CurrentFile = Cache.PkgFileP + idxFile;
1222
1223    // Fill it in
1224    map_stringitem_t const idxFileName = WriteStringInMap(File);
1225    if (unlikely(idxFileName == 0))
1226       return false;
1227    CurrentFile->FileName = idxFileName;
1228    CurrentFile->NextFile = Cache.HeaderP->FileList;
1229    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1230    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1231    if (unlikely(idxIndexType == 0))
1232       return false;
1233    CurrentFile->IndexType = idxIndexType;
1234    if (Architecture.empty())
1235       CurrentFile->Architecture = 0;
1236    else
1237    {
1238       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1239       if (unlikely(arch == 0))
1240          return false;
1241       CurrentFile->Architecture = arch;
1242    }
1243    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1244    if (unlikely(component == 0))
1245       return false;
1246    CurrentFile->Component = component;
1247    CurrentFile->Flags = Flags;
1248    if (CurrentRlsFile != NULL)
1249       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1250    else
1251       CurrentFile->Release = 0;
1252    PkgFileName = File;
1253    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1254    Cache.HeaderP->PackageFileCount++;
1255
1256    if (Progress != 0)
1257       Progress->SubProgress(Index.Size());
1258    return true;
1259 }
1260                                                                         /*}}}*/
1261 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1262 // ---------------------------------------------------------------------
1263 /* This is used to create handles to strings. Given the same text it
1264    always returns the same number */
1265 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1266                                                  unsigned int Size)
1267 {
1268    auto strings = &strMixed;
1269    switch(type) {
1270       case MIXED: strings = &strMixed; break;
1271       case PKGNAME: strings = &strPkgNames; break;
1272       case VERSIONNUMBER: strings = &strVersions; break;
1273       case SECTION: strings = &strSections; break;
1274       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1275    }
1276
1277    auto const item = strings->find({S, Size, nullptr, 0});
1278    if (item != strings->end())
1279       return item->item;
1280
1281    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1282    strings->insert({nullptr, Size, this, idxString});
1283    return idxString;
1284 }
1285                                                                         /*}}}*/
1286 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1287 // ---------------------------------------------------------------------
1288 /* This just verifies that each file in the list of index files exists,
1289    has matching attributes with the cache and the cache does not have
1290    any extra files. */
1291 class APT_HIDDEN ScopedErrorRevert {
1292 public:
1293    ScopedErrorRevert() { _error->PushToStack(); }
1294    ~ScopedErrorRevert() { _error->RevertToStack(); }
1295 };
1296 static bool CheckValidity(const string &CacheFile,
1297                           pkgSourceList &List,
1298                           FileIterator const Start,
1299                           FileIterator const End,
1300                           MMap **OutMap = 0,
1301                           pkgCache **OutCache = 0)
1302 {
1303    ScopedErrorRevert ser;
1304    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1305    // No file, certainly invalid
1306    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1307    {
1308       if (Debug == true)
1309          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1310       return false;
1311    }
1312
1313    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1314    {
1315       if (Debug == true)
1316          std::clog << "sources.list is newer than the cache" << std::endl;
1317       return false;
1318    }
1319
1320    // Map it
1321    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1322    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1323    if (unlikely(Map->validData()) == false)
1324       return false;
1325    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1326    pkgCache &Cache = *CacheP.get();
1327    if (_error->PendingError() || Map->Size() == 0)
1328    {
1329       if (Debug == true)
1330          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1331       return false;
1332    }
1333
1334    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1335    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1336    std::vector<pkgIndexFile *> Files;
1337    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1338    {
1339       if (Debug == true)
1340          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1341       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1342       if (RlsFile.end() == true)
1343       {
1344          if (Debug == true)
1345             std::clog << "FindInCache returned end-Pointer" << std::endl;
1346          return false;
1347       }
1348
1349       RlsVisited[RlsFile->ID] = true;
1350       if (Debug == true)
1351          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1352
1353       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1354       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1355             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1356    }
1357    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1358       if (RlsVisited[I] == false)
1359       {
1360          if (Debug == true)
1361             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1362          return false;
1363       }
1364
1365    std::copy(Start, End, std::back_inserter(Files));
1366
1367    /* Now we check every index file, see if it is in the cache,
1368       verify the IMS data and check that it is on the disk too.. */
1369    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1370    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1371    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1372    {
1373       if (Debug == true)
1374          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1375       if ((*PkgFile)->Exists() == false)
1376       {
1377          if (Debug == true)
1378             std::clog << "file doesn't exist" << std::endl;
1379          continue;
1380       }
1381
1382       // FindInCache is also expected to do an IMS check.
1383       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1384       if (File.end() == true)
1385       {
1386          if (Debug == true)
1387             std::clog << "FindInCache returned end-Pointer" << std::endl;
1388          return false;
1389       }
1390
1391       Visited[File->ID] = true;
1392       if (Debug == true)
1393          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1394    }
1395
1396    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1397       if (Visited[I] == false)
1398       {
1399          if (Debug == true)
1400             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1401          return false;
1402       }
1403
1404    if (_error->PendingError() == true)
1405    {
1406       if (Debug == true)
1407       {
1408          std::clog << "Validity failed because of pending errors:" << std::endl;
1409          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1410       }
1411       return false;
1412    }
1413
1414    if (OutMap != 0)
1415       *OutMap = Map.release();
1416    if (OutCache != 0)
1417       *OutCache = CacheP.release();
1418    return true;
1419 }
1420                                                                         /*}}}*/
1421 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1422 // ---------------------------------------------------------------------
1423 /* Size is kind of an abstract notion that is only used for the progress
1424    meter */
1425 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1426 {
1427    map_filesize_t TotalSize = 0;
1428    if (List !=  NULL)
1429    {
1430       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1431       {
1432          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1433          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1434             if ((*j)->HasPackages() == true)
1435                TotalSize += (*j)->Size();
1436       }
1437    }
1438
1439    for (; Start < End; ++Start)
1440    {
1441       if ((*Start)->HasPackages() == false)
1442          continue;
1443       TotalSize += (*Start)->Size();
1444    }
1445    return TotalSize;
1446 }
1447                                                                         /*}}}*/
1448 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1449 static bool BuildCache(pkgCacheGenerator &Gen,
1450                        OpProgress * const Progress,
1451                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1452                        pkgSourceList const * const List,
1453                        FileIterator const Start, FileIterator const End)
1454 {
1455    bool mergeFailure = false;
1456
1457    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1458       if (I->HasPackages() == false || mergeFailure)
1459          return;
1460
1461       if (I->Exists() == false)
1462          return;
1463
1464       if (I->FindInCache(Gen.GetCache()).end() == false)
1465       {
1466          _error->Warning("Duplicate sources.list entry %s",
1467                I->Describe().c_str());
1468          return;
1469       }
1470
1471       map_filesize_t const Size = I->Size();
1472       if (Progress != NULL)
1473          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1474       CurrentSize += Size;
1475
1476       if (I->Merge(Gen,Progress) == false)
1477          mergeFailure = true;
1478    };
1479
1480    if (List !=  NULL)
1481    {
1482       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1483       {
1484          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1485          {
1486             _error->Warning("Duplicate sources.list entry %s",
1487                   (*i)->Describe().c_str());
1488             continue;
1489          }
1490
1491          if ((*i)->Merge(Gen, Progress) == false)
1492             return false;
1493
1494          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1495          if (Indexes != NULL)
1496             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1497          if (mergeFailure)
1498             return false;
1499       }
1500    }
1501
1502    if (Start != End)
1503    {
1504       Gen.SelectReleaseFile("", "");
1505       std::for_each(Start, End, indexFileMerge);
1506       if (mergeFailure)
1507          return false;
1508    }
1509    return true;
1510 }
1511                                                                         /*}}}*/
1512 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1513 // ---------------------------------------------------------------------
1514 /* This makes sure that the status cache (the cache that has all
1515    index files from the sources list and all local ones) is ready
1516    to be mmaped. If OutMap is not zero then a MMap object representing
1517    the cache will be stored there. This is pretty much mandetory if you
1518    are using AllowMem. AllowMem lets the function be run as non-root
1519    where it builds the cache 'fast' into a memory buffer. */
1520 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1521 {
1522    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1523    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1524    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1525    Flags |= MMap::Moveable;
1526    if (_config->FindB("APT::Cache-Fallback", false) == true)
1527       Flags |= MMap::Fallback;
1528    if (CacheF != NULL)
1529       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1530    else
1531       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1532 }
1533 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1534       std::string const &FileName)
1535 {
1536    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1537    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1538       return false;
1539
1540    fchmod(SCacheF.Fd(),0644);
1541
1542    // Write out the main data
1543    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1544       return _error->Error(_("IO Error saving source cache"));
1545
1546    // Write out the proper header
1547    Gen->GetCache().HeaderP->Dirty = false;
1548    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1549    if (SCacheF.Seek(0) == false ||
1550          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1551       return _error->Error(_("IO Error saving source cache"));
1552    Gen->GetCache().HeaderP->Dirty = true;
1553    return true;
1554 }
1555 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1556       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1557 {
1558    Map.reset(CreateDynamicMMap(NULL, 0));
1559    if (unlikely(Map->validData()) == false)
1560       return false;
1561    FileFd CacheF(FileName, FileFd::ReadOnly);
1562    if (CacheF.IsOpen() == false || CacheF.Failed())
1563       return false;
1564    _error->PushToStack();
1565    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1566    bool const newError = _error->PendingError();
1567    _error->MergeWithStack();
1568    if (alloc == 0 && newError)
1569       return false;
1570    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1571       return false;
1572    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1573    return true;
1574 }
1575 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1576                         MMap **OutMap, bool AllowMem)
1577    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1578 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1579                         MMap **OutMap,bool)
1580 {
1581    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1582 }
1583 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1584                         MMap **OutMap,pkgCache **OutCache, bool)
1585 {
1586    // FIXME: deprecate the ignored AllowMem parameter
1587    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1588
1589    std::vector<pkgIndexFile *> Files;
1590    if (_system->AddStatusFiles(Files) == false)
1591       return false;
1592
1593    // Decide if we can write to the files..
1594    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1595    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1596
1597    // ensure the cache directory exists
1598    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1599    {
1600       string dir = _config->FindDir("Dir::Cache");
1601       size_t const len = dir.size();
1602       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1603          dir = dir.substr(0, len - 5);
1604       if (CacheFile.empty() == false)
1605          CreateDirectory(dir, flNotFile(CacheFile));
1606       if (SrcCacheFile.empty() == false)
1607          CreateDirectory(dir, flNotFile(SrcCacheFile));
1608    }
1609
1610    if (Progress != NULL)
1611       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1612
1613    bool pkgcache_fine = false;
1614    bool srcpkgcache_fine = false;
1615    bool volatile_fine = List.GetVolatileFiles().empty();
1616
1617    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1618                      volatile_fine ? OutCache : NULL) == true)
1619    {
1620       if (Debug == true)
1621          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1622       pkgcache_fine = true;
1623       srcpkgcache_fine = true;
1624    }
1625    if (pkgcache_fine == false)
1626    {
1627       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1628       {
1629          if (Debug == true)
1630             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1631          srcpkgcache_fine = true;
1632       }
1633    }
1634
1635    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1636    {
1637       if (Progress != NULL)
1638          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1639       return true;
1640    }
1641
1642    bool Writeable = false;
1643    if (srcpkgcache_fine == false || pkgcache_fine == false)
1644    {
1645       if (CacheFile.empty() == false)
1646          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1647       else if (SrcCacheFile.empty() == false)
1648          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1649
1650       if (Debug == true)
1651          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1652    }
1653
1654    // At this point we know we need to construct something, so get storage ready
1655    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1656    if (unlikely(Map->validData()) == false)
1657       return false;
1658    if (Debug == true)
1659       std::clog << "Open memory Map (not filebased)" << std::endl;
1660
1661    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1662    map_filesize_t CurrentSize = 0;
1663    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1664    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1665    if (srcpkgcache_fine == true && pkgcache_fine == false)
1666    {
1667       if (Debug == true)
1668          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1669       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1670          return false;
1671       srcpkgcache_fine = true;
1672       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1673    }
1674    else if (srcpkgcache_fine == false)
1675    {
1676       if (Debug == true)
1677          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1678       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1679
1680       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1681       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1682                Files.end(),Files.end()) == false)
1683          return false;
1684
1685       if (Writeable == true && SrcCacheFile.empty() == false)
1686          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1687             return false;
1688    }
1689
1690    if (pkgcache_fine == false)
1691    {
1692       if (Debug == true)
1693          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1694       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1695                Files.begin(), Files.end()) == false)
1696          return false;
1697
1698       if (Writeable == true && CacheFile.empty() == false)
1699          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1700             return false;
1701    }
1702
1703    if (Debug == true)
1704       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1705
1706    if (volatile_fine == false)
1707    {
1708       if (Gen == nullptr)
1709       {
1710          if (Debug == true)
1711             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1712          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1713             return false;
1714       }
1715
1716       Files = List.GetVolatileFiles();
1717       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1718                Files.begin(), Files.end()) == false)
1719          return false;
1720    }
1721
1722    if (OutMap != nullptr)
1723       *OutMap = Map.release();
1724
1725    if (Debug == true)
1726       std::clog << "Everything is ready for shipping" << std::endl;
1727    return true;
1728 }
1729                                                                         /*}}}*/
1730 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1731 class APT_HIDDEN ScopedErrorMerge {
1732 public:
1733    ScopedErrorMerge() { _error->PushToStack(); }
1734    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1735 };
1736 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1737    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1738 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1739 {
1740    std::vector<pkgIndexFile *> Files;
1741    if (_system->AddStatusFiles(Files) == false)
1742       return false;
1743
1744    ScopedErrorMerge sem;
1745    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1746    if (unlikely(Map->validData()) == false)
1747       return false;
1748    map_filesize_t CurrentSize = 0;
1749    map_filesize_t TotalSize = 0;
1750    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1751
1752    // Build the status cache
1753    if (Progress != NULL)
1754       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1755    pkgCacheGenerator Gen(Map.get(),Progress);
1756    if (_error->PendingError() == true)
1757       return false;
1758    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1759                   Files.begin(), Files.end()) == false)
1760       return false;
1761
1762    if (_error->PendingError() == true)
1763       return false;
1764    *OutMap = Map.release();
1765
1766    return true;
1767 }
1768                                                                         /*}}}*/
1769 // IsDuplicateDescription                                               /*{{{*/
1770 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1771                             MD5SumValue const &CurMd5, std::string const &CurLang)
1772 {
1773    // Descriptions in the same link-list have all the same md5
1774    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1775       return false;
1776    for (; Desc.end() == false; ++Desc)
1777       if (Desc.LanguageCode() == CurLang)
1778          return true;
1779    return false;
1780 }
1781                                                                         /*}}}*/
1782
1783 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1784 pkgCacheListParser::~pkgCacheListParser() {}