apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 181       // Ignore views outside of the cache.
 182       if (ViewP->data() < static_cast<const char*>(oldMap)
 183          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 184          continue;
 185       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 186       *ViewP = StringView(data , ViewP->size());
 187    }
 188 }                                                                       /*}}}*/
 189 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 190 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 191                                         const unsigned long &Len) {
 192    size_t oldSize = Map.Size();
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String, Len);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data(), oldSize);
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 201 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 202    size_t oldSize = Map.Size();
 203    void const * const oldMap = Map.Data();
 204    map_stringitem_t const index = Map.WriteString(String);
 205    if (index != 0)
 206       ReMap(oldMap, Map.Data(), oldSize);
 207    return index;
 208 }
 209                                                                         /*}}}*/
 210 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 211    size_t oldSize = Map.Size();
 212    void const * const oldMap = Map.Data();
 213    map_pointer_t const index = Map.Allocate(size);
 214    if (index != 0)
 215       ReMap(oldMap, Map.Data(), oldSize);
 216    return index;
 217 }
 218                                                                         /*}}}*/
 219 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 220 // ---------------------------------------------------------------------
 221 /* This provides the generation of the entries in the cache. Each loop
 222    goes through a single package record from the underlying parse engine. */
 223 bool pkgCacheGenerator::MergeList(ListParser &List,
 224                                   pkgCache::VerIterator *OutVer)
 225 {
 226    List.Owner = this;
 227
 228    unsigned int Counter = 0;
 229    while (List.Step() == true)
 230    {
 231       string const PackageName = List.Package();
 232       if (PackageName.empty() == true)
 233          return false;
 234
 235       Counter++;
 236       if (Counter % 100 == 0 && Progress != 0)
 237          Progress->Progress(List.Offset());
 238
 239       string Arch = List.Architecture();
 240       string const Version = List.Version();
 241       if (Version.empty() == true && Arch.empty() == true)
 242       {
 243          // package descriptions
 244          if (MergeListGroup(List, PackageName) == false)
 245             return false;
 246          continue;
 247       }
 248
 249       // Get a pointer to the package structure
 250       pkgCache::PkgIterator Pkg;
 251       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 252       if (NewPackage(Pkg, PackageName, Arch) == false)
 253          // TRANSLATOR: The first placeholder is a package name,
 254          // the other two should be copied verbatim as they include debug info
 255          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 256                               PackageName.c_str(), "NewPackage", 1);
 257
 258
 259       if (Version.empty() == true)
 260       {
 261          if (MergeListPackage(List, Pkg) == false)
 262             return false;
 263       }
 264       else
 265       {
 266          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 267             return false;
 268       }
 269
 270       if (OutVer != 0)
 271          return true;
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    return true;
 288 }
 289 // CacheGenerator::MergeListGroup                                       /*{{{*/
 290 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 291 {
 292    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 293    // a group has no data on it's own, only packages have it but these
 294    // stanzas like this come from Translation- files to add descriptions,
 295    // but without a version we don't need a description for it…
 296    if (Grp.end() == true)
 297       return true;
 298    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 299
 300    pkgCache::PkgIterator Pkg;
 301    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 302    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 303       if (MergeListPackage(List, Pkg) == false)
 304          return false;
 305
 306    return true;
 307 }
 308                                                                         /*}}}*/
 309 // CacheGenerator::MergeListPackage                                     /*{{{*/
 310 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 311 {
 312    // we first process the package, then the descriptions
 313    // (for deb this package processing is in fact a no-op)
 314    pkgCache::VerIterator Ver(Cache);
 315    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 316    if (List.UsePackage(Pkg, Ver) == false)
 317       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 318                            Pkg.Name(), "UsePackage", 1);
 319
 320    // Find the right version to write the description
 321    MD5SumValue CurMd5 = List.Description_md5();
 322    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 323    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 324    {
 325       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 326
 327       // a version can only have one md5 describing it
 328       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 329          continue;
 330
 331       map_stringitem_t md5idx = VerDesc->md5sum;
 332       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 333       {
 334          // don't add a new description if we have one for the given
 335          // md5 && language
 336          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 337             continue;
 338
 339          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 340       }
 341
 342       // we can stop here as all "same" versions will share the description
 343       break;
 344    }
 345
 346    return true;
 347 }
 348                                                                         /*}}}*/
 349 // CacheGenerator::MergeListVersion                                     /*{{{*/
 350 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 351                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 352 {
 353    pkgCache::VerIterator Ver = Pkg.VersionList();
 354    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 355    map_pointer_t *LastVer = &Pkg->VersionList;
 356    void const * oldMap = Map.Data();
 357
 358    unsigned short const Hash = List.VersionHash();
 359    if (Ver.end() == false)
 360    {
 361       /* We know the list is sorted so we use that fact in the search.
 362          Insertion of new versions is done with correct sorting */
 363       int Res = 1;
 364       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 365       {
 366          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 367          // Version is higher as current version - insert here
 368          if (Res > 0)
 369             break;
 370          // Versionstrings are equal - is hash also equal?
 371          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 372             break;
 373          // proceed with the next till we have either the right
 374          // or we found another version (which will be lower)
 375       }
 376
 377       /* We already have a version for this item, record that we saw it */
 378       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 379       {
 380          if (List.UsePackage(Pkg,Ver) == false)
 381             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 382                                  Pkg.Name(), "UsePackage", 2);
 383
 384          if (NewFileVer(Ver,List) == false)
 385             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 386                                  Pkg.Name(), "NewFileVer", 1);
 387
 388          // Read only a single record and return
 389          if (OutVer != 0)
 390          {
 391             *OutVer = Ver;
 392             return true;
 393          }
 394
 395          return true;
 396       }
 397    }
 398
 399    // Add a new version
 400    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 401    if (unlikely(verindex == 0))
 402       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 403                            Pkg.Name(), "NewVersion", 1);
 404
 405    if (oldMap != Map.Data())
 406          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 407    *LastVer = verindex;
 408
 409    if (unlikely(List.NewVersion(Ver) == false))
 410       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 411                            Pkg.Name(), "NewVersion", 2);
 412
 413    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 414       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 415                            Pkg.Name(), "UsePackage", 3);
 416
 417    if (unlikely(NewFileVer(Ver,List) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "NewFileVer", 2);
 420
 421    pkgCache::GrpIterator Grp = Pkg.Group();
 422    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 423
 424    /* If it is the first version of this package we need to add implicit
 425       Multi-Arch dependencies to all other package versions in the group now -
 426       otherwise we just add them for this new version */
 427    if (Pkg.VersionList()->NextVer == 0)
 428    {
 429       pkgCache::PkgIterator P = Grp.PackageList();
 430       Dynamic<pkgCache::PkgIterator> DynP(P);
 431       for (; P.end() != true; P = Grp.NextPkg(P))
 432       {
 433          if (P->ID == Pkg->ID)
 434             continue;
 435          pkgCache::VerIterator V = P.VersionList();
 436          Dynamic<pkgCache::VerIterator> DynV(V);
 437          for (; V.end() != true; ++V)
 438             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 439                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 440                                     Pkg.Name(), "AddImplicitDepends", 1);
 441       }
 442    }
 443    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 444       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 445                            Pkg.Name(), "AddImplicitDepends", 2);
 446
 447    // Read only a single record and return
 448    if (OutVer != 0)
 449    {
 450       *OutVer = Ver;
 451       return true;
 452    }
 453
 454    /* Record the Description(s) based on their master md5sum */
 455    MD5SumValue CurMd5 = List.Description_md5();
 456
 457    /* Before we add a new description we first search in the group for
 458       a version with a description of the same MD5 - if so we reuse this
 459       description group instead of creating our own for this version */
 460    for (pkgCache::PkgIterator P = Grp.PackageList();
 461         P.end() == false; P = Grp.NextPkg(P))
 462    {
 463       for (pkgCache::VerIterator V = P.VersionList();
 464            V.end() == false; ++V)
 465       {
 466          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 467             continue;
 468          Ver->DescriptionList = V->DescriptionList;
 469       }
 470    }
 471
 472    // We haven't found reusable descriptions, so add the first description(s)
 473    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 474    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 475    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 476       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 477          return false;
 478    return true;
 479 }
 480                                                                         /*}}}*/
 481 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 482 {
 483    pkgCache::DescIterator Desc;
 484    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 485
 486    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 487    if (unlikely(descindex == 0))
 488       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 489             Ver.ParentPkg().Name(), "NewDescription", 1);
 490
 491    md5idx = Desc->md5sum;
 492    Desc->ParentPkg = Ver.ParentPkg().Index();
 493
 494    // we add at the end, so that the start is constant as we need
 495    // that to be able to efficiently share these lists
 496    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 497    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 498    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 499    *LastNextDesc = descindex;
 500
 501    if (NewFileDesc(Desc,List) == false)
 502       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 503             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 504
 505    return true;
 506 }
 507                                                                         /*}}}*/
 508                                                                         /*}}}*/
 509 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* This creates a new group structure and adds it to the hash table */
 512 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 513 {
 514    Dynamic<StringView> DName(Name);
 515    Grp = Cache.FindGrp(Name);
 516    if (Grp.end() == false)
 517       return true;
 518
 519    // Get a structure
 520    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 521    if (unlikely(Group == 0))
 522       return false;
 523
 524    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 525    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 526    if (unlikely(idxName == 0))
 527       return false;
 528    Grp->Name = idxName;
 529
 530    // Insert it into the hash table
 531    unsigned long const Hash = Cache.Hash(Name);
 532    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 533
 534    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 535       insertAt = &(Cache.GrpP + *insertAt)->Next;
 536    Grp->Next = *insertAt;
 537    *insertAt = Group;
 538
 539    Grp->ID = Cache.HeaderP->GroupCount++;
 540    return true;
 541 }
 542                                                                         /*}}}*/
 543 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* This creates a new package structure and adds it to the hash table */
 546 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 547                                         StringView Arch) {
 548    pkgCache::GrpIterator Grp;
 549    Dynamic<StringView> DName(Name);
 550    Dynamic<StringView> DArch(Arch);
 551    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 552    if (unlikely(NewGroup(Grp, Name) == false))
 553       return false;
 554
 555    Pkg = Grp.FindPkg(Arch);
 556       if (Pkg.end() == false)
 557          return true;
 558
 559    // Get a structure
 560    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 561    if (unlikely(Package == 0))
 562       return false;
 563    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 564
 565    // Set the name, arch and the ID
 566    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 567    Pkg->Group = Grp.Index();
 568    // all is mapped to the native architecture
 569    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 570    if (unlikely(idxArch == 0))
 571       return false;
 572    Pkg->Arch = idxArch;
 573    Pkg->ID = Cache.HeaderP->PackageCount++;
 574
 575    // Insert the package into our package list
 576    if (Grp->FirstPackage == 0) // the group is new
 577    {
 578       Grp->FirstPackage = Package;
 579       // Insert it into the hash table
 580       map_id_t const Hash = Cache.Hash(Name);
 581       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 582       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 583          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 584       Pkg->NextPackage = *insertAt;
 585       *insertAt = Package;
 586    }
 587    else // Group the Packages together
 588    {
 589       // if sibling is provided by another package, this one is too
 590       {
 591          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 592          if (M.end() == false) {
 593             pkgCache::PrvIterator Prv;
 594             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 595             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 596             {
 597                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 598                   continue;
 599                pkgCache::VerIterator Ver = Prv.OwnerVer();
 600                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 601                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 602                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 603                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 604                {
 605                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 606                      continue;
 607                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 608                      return false;
 609                }
 610             }
 611          }
 612       }
 613       // let M-A:foreign package siblings provide this package
 614       {
 615          pkgCache::PkgIterator P;
 616          pkgCache::VerIterator Ver;
 617          Dynamic<pkgCache::PkgIterator> DynP(P);
 618          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 619
 620          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 621          {
 622             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 623                continue;
 624             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 625                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 626                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 627                      return false;
 628          }
 629       }
 630       // and negative dependencies, don't forget negative dependencies
 631       {
 632          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 633          if (M.end() == false) {
 634             pkgCache::DepIterator Dep;
 635             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 636             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 637             {
 638                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 639                   continue;
 640                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 641                      Dep->Type != pkgCache::Dep::Replaces)
 642                   continue;
 643                pkgCache::VerIterator Ver = Dep.ParentVer();
 644                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 645                map_pointer_t * unused = NULL;
 646                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 647                   return false;
 648             }
 649          }
 650       }
 651
 652       // this package is the new last package
 653       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 654       Pkg->NextPackage = LastPkg->NextPackage;
 655       LastPkg->NextPackage = Package;
 656    }
 657    Grp->LastPackage = Package;
 658
 659    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 660    if (Arch == "any")
 661    {
 662       size_t const found = Name.find(':');
 663       StringView const NameA = Name.substr(0, found);
 664       StringView const ArchA = Name.substr(found + 1);
 665       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 666       if (PkgA.end() == false)
 667       {
 668          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 669          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 670          for (; Prv.end() == false; ++Prv)
 671          {
 672             if (Prv.IsMultiArchImplicit())
 673                continue;
 674             pkgCache::VerIterator V = Prv.OwnerVer();
 675             if (ArchA != V.ParentPkg().Arch())
 676                continue;
 677             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 678                return false;
 679          }
 680          pkgCache::VerIterator V = PkgA.VersionList();
 681          Dynamic<pkgCache::VerIterator> DynV(V);
 682          for (; V.end() == false; ++V)
 683          {
 684             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 685                return false;
 686          }
 687       }
 688    }
 689    return true;
 690 }
 691                                                                         /*}}}*/
 692 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 693 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 694                                            pkgCache::PkgIterator &P,
 695                                            pkgCache::VerIterator &V)
 696 {
 697    // copy P.Arch() into a string here as a cache remap
 698    // in NewDepends() later may alter the pointer location
 699    string Arch = P.Arch() == NULL ? "" : P.Arch();
 700    map_pointer_t *OldDepLast = NULL;
 701    /* MultiArch handling introduces a lot of implicit Dependencies:
 702       - MultiArch: same → Co-Installable if they have the same version
 703       - All others conflict with all other group members */
 704    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 705    pkgCache::PkgIterator D = G.PackageList();
 706    Dynamic<pkgCache::PkgIterator> DynD(D);
 707    map_stringitem_t const VerStrIdx = V->VerStr;
 708    for (; D.end() != true; D = G.NextPkg(D))
 709    {
 710       if (Arch == D.Arch() || D->VersionList == 0)
 711          continue;
 712       /* We allow only one installed arch at the time
 713          per group, therefore each group member conflicts
 714          with all other group members */
 715       if (coInstall == true)
 716       {
 717          // Replaces: ${self}:other ( << ${binary:Version})
 718          NewDepends(D, V, VerStrIdx,
 719                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 720                     OldDepLast);
 721          // Breaks: ${self}:other (!= ${binary:Version})
 722          NewDepends(D, V, VerStrIdx,
 723                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 724                     OldDepLast);
 725       } else {
 726          // Conflicts: ${self}:other
 727          NewDepends(D, V, 0,
 728                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 729                     OldDepLast);
 730       }
 731    }
 732    return true;
 733 }
 734 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 735                                            pkgCache::PkgIterator &D)
 736 {
 737    /* MultiArch handling introduces a lot of implicit Dependencies:
 738       - MultiArch: same → Co-Installable if they have the same version
 739       - All others conflict with all other group members */
 740    map_pointer_t *OldDepLast = NULL;
 741    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 742    if (coInstall == true)
 743    {
 744       map_stringitem_t const VerStrIdx = V->VerStr;
 745       // Replaces: ${self}:other ( << ${binary:Version})
 746       NewDepends(D, V, VerStrIdx,
 747                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 748                  OldDepLast);
 749       // Breaks: ${self}:other (!= ${binary:Version})
 750       NewDepends(D, V, VerStrIdx,
 751                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 752                  OldDepLast);
 753    } else {
 754       // Conflicts: ${self}:other
 755       NewDepends(D, V, 0,
 756                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 757                  OldDepLast);
 758    }
 759    return true;
 760 }
 761
 762                                                                         /*}}}*/
 763 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 764 // ---------------------------------------------------------------------
 765 /* */
 766 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 767                                    ListParser &List)
 768 {
 769    if (CurrentFile == 0)
 770       return true;
 771
 772    // Get a structure
 773    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 774    if (VerFile == 0)
 775       return false;
 776
 777    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 778    VF->File = CurrentFile - Cache.PkgFileP;
 779
 780    // Link it to the end of the list
 781    map_pointer_t *Last = &Ver->FileList;
 782    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 783       Last = &V->NextFile;
 784    VF->NextFile = *Last;
 785    *Last = VF.Index();
 786
 787    VF->Offset = List.Offset();
 788    VF->Size = List.Size();
 789    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 790       Cache.HeaderP->MaxVerFileSize = VF->Size;
 791    Cache.HeaderP->VerFileCount++;
 792
 793    return true;
 794 }
 795                                                                         /*}}}*/
 796 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 797 // ---------------------------------------------------------------------
 798 /* This puts a version structure in the linked list */
 799 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 800                                             const string &VerStr,
 801                                             map_pointer_t const ParentPkg,
 802                                             unsigned short const Hash,
 803                                             map_pointer_t const Next)
 804 {
 805    // Get a structure
 806    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 807    if (Version == 0)
 808       return 0;
 809
 810    // Fill it in
 811    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 812    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 813    Ver->NextVer = Next;
 814    Ver->ParentPkg = ParentPkg;
 815    Ver->Hash = Hash;
 816    Ver->ID = Cache.HeaderP->VersionCount++;
 817
 818    // try to find the version string in the group for reuse
 819    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 820    pkgCache::GrpIterator Grp = Pkg.Group();
 821    if (Pkg.end() == false && Grp.end() == false)
 822    {
 823       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 824       {
 825          if (Pkg == P)
 826             continue;
 827          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 828          {
 829             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 830             if (cmp == 0)
 831             {
 832                Ver->VerStr = V->VerStr;
 833                return Version;
 834             }
 835             else if (cmp < 0)
 836                break;
 837          }
 838       }
 839    }
 840    // haven't found the version string, so create
 841    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 842    if (unlikely(idxVerStr == 0))
 843       return 0;
 844    Ver->VerStr = idxVerStr;
 845    return Version;
 846 }
 847                                                                         /*}}}*/
 848 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* */
 851 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 852                                    ListParser &List)
 853 {
 854    if (CurrentFile == 0)
 855       return true;
 856
 857    // Get a structure
 858    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 859    if (DescFile == 0)
 860       return false;
 861
 862    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 863    DF->File = CurrentFile - Cache.PkgFileP;
 864
 865    // Link it to the end of the list
 866    map_pointer_t *Last = &Desc->FileList;
 867    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 868       Last = &D->NextFile;
 869
 870    DF->NextFile = *Last;
 871    *Last = DF.Index();
 872
 873    DF->Offset = List.Offset();
 874    DF->Size = List.Size();
 875    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 876       Cache.HeaderP->MaxDescFileSize = DF->Size;
 877    Cache.HeaderP->DescFileCount++;
 878
 879    return true;
 880 }
 881                                                                         /*}}}*/
 882 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 883 // ---------------------------------------------------------------------
 884 /* This puts a description structure in the linked list */
 885 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 886                                             const string &Lang,
 887                                             const MD5SumValue &md5sum,
 888                                             map_stringitem_t const idxmd5str)
 889 {
 890    // Get a structure
 891    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 892    if (Description == 0)
 893       return 0;
 894
 895    // Fill it in
 896    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 897    Desc->ID = Cache.HeaderP->DescriptionCount++;
 898    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 899    if (unlikely(idxlanguage_code == 0))
 900       return 0;
 901    Desc->language_code = idxlanguage_code;
 902
 903    if (idxmd5str != 0)
 904       Desc->md5sum = idxmd5str;
 905    else
 906    {
 907       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 908       if (unlikely(idxmd5sum == 0))
 909          return 0;
 910       Desc->md5sum = idxmd5sum;
 911    }
 912
 913    return Description;
 914 }
 915                                                                         /*}}}*/
 916 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* This creates a dependency element in the tree. It is linked to the
 919    version and to the package that it is pointing to. */
 920 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 921                                    pkgCache::VerIterator &Ver,
 922                                    map_pointer_t const Version,
 923                                    uint8_t const Op,
 924                                    uint8_t const Type,
 925                                    map_pointer_t* &OldDepLast)
 926 {
 927    void const * const oldMap = Map.Data();
 928    // Get a structure
 929    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 930    if (unlikely(Dependency == 0))
 931       return false;
 932
 933    bool isDuplicate = false;
 934    map_pointer_t DependencyData = 0;
 935    map_pointer_t PreviousData = 0;
 936    if (Pkg->RevDepends != 0)
 937    {
 938       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 939       DependencyData = L->DependencyData;
 940       do {
 941          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 942          if (Version > D->Version)
 943             break;
 944          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 945          {
 946             isDuplicate = true;
 947             break;
 948          }
 949          PreviousData = DependencyData;
 950          DependencyData = D->NextData;
 951       } while (DependencyData != 0);
 952    }
 953
 954    if (isDuplicate == false)
 955    {
 956       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 957       if (unlikely(DependencyData == 0))
 958         return false;
 959    }
 960
 961    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 962    Link->ParentVer = Ver.Index();
 963    Link->DependencyData = DependencyData;
 964    Link->ID = Cache.HeaderP->DependsCount++;
 965
 966    pkgCache::DepIterator Dep(Cache, Link);
 967    if (isDuplicate == false)
 968    {
 969       Dep->Type = Type;
 970       Dep->CompareOp = Op;
 971       Dep->Version = Version;
 972       Dep->Package = Pkg.Index();
 973       ++Cache.HeaderP->DependsDataCount;
 974       if (PreviousData != 0)
 975       {
 976          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 977          Dep->NextData = D->NextData;
 978          D->NextData = DependencyData;
 979       }
 980       else if (Pkg->RevDepends != 0)
 981       {
 982          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 983          Dep->NextData = D->DependencyData;
 984       }
 985    }
 986
 987    if (isDuplicate == true || PreviousData != 0)
 988    {
 989       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 990       Link->NextRevDepends = L->NextRevDepends;
 991       L->NextRevDepends = Dependency;
 992    }
 993    else
 994    {
 995       Link->NextRevDepends = Pkg->RevDepends;
 996       Pkg->RevDepends = Dependency;
 997    }
 998
 999
1000    // Do we know where to link the Dependency to?
1001    if (OldDepLast == NULL)
1002    {
1003       OldDepLast = &Ver->DependsList;
1004       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1005          OldDepLast = &D->NextDepends;
1006    } else if (oldMap != Map.Data())
1007       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1008
1009    Dep->NextDepends = *OldDepLast;
1010    *OldDepLast = Dependency;
1011    OldDepLast = &Dep->NextDepends;
1012    return true;
1013 }
1014                                                                         /*}}}*/
1015 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1016 // ---------------------------------------------------------------------
1017 /* This creates a Group and the Package to link this dependency to if
1018    needed and handles also the caching of the old endpoint */
1019 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1020                                                StringView PackageName,
1021                                                StringView Arch,
1022                                                StringView Version,
1023                                                uint8_t const Op,
1024                                                uint8_t const Type)
1025 {
1026    pkgCache::GrpIterator Grp;
1027    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1028    Dynamic<StringView> DynPackageName(PackageName);
1029    Dynamic<StringView> DynArch(Arch);
1030    Dynamic<StringView> DynVersion(Version);
1031    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1032       return false;
1033
1034    map_stringitem_t idxVersion = 0;
1035    if (Version.empty() == false)
1036    {
1037       int const CmpOp = Op & 0x0F;
1038       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1039       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1040          idxVersion = Ver->VerStr;
1041
1042       if (idxVersion == 0)
1043       {
1044          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1045          if (unlikely(idxVersion == 0))
1046             return false;
1047       }
1048    }
1049
1050    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1051          Type == pkgCache::Dep::Conflicts ||
1052          Type == pkgCache::Dep::Replaces);
1053
1054    pkgCache::PkgIterator Pkg;
1055    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1056    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1057    {
1058       // Locate the target package
1059       Pkg = Grp.FindPkg(Arch);
1060       if (Pkg.end() == true) {
1061          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1062             return false;
1063       }
1064
1065       /* Caching the old end point speeds up generation substantially */
1066       if (OldDepVer != Ver) {
1067          OldDepLast = NULL;
1068          OldDepVer = Ver;
1069       }
1070
1071       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1072    }
1073    else
1074    {
1075       /* Caching the old end point speeds up generation substantially */
1076       if (OldDepVer != Ver) {
1077          OldDepLast = NULL;
1078          OldDepVer = Ver;
1079       }
1080
1081       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1082       {
1083          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1084                return false;
1085       }
1086    }
1087    return true;
1088 }
1089                                                                         /*}}}*/
1090 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1091 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1092                                                 StringView PkgName,
1093                                                 StringView PkgArch,
1094                                                 StringView Version,
1095                                                 uint8_t const Flags)
1096 {
1097    pkgCache const &Cache = Owner->Cache;
1098    Dynamic<StringView> DynPkgName(PkgName);
1099    Dynamic<StringView> DynArch(PkgArch);
1100    Dynamic<StringView> DynVersion(Version);
1101
1102    // We do not add self referencing provides
1103    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1104         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1105          (Version.empty() || Version == Ver.VerStr()))
1106       return true;
1107
1108    // Locate the target package
1109    pkgCache::PkgIterator Pkg;
1110    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1111    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1112       return false;
1113
1114    map_stringitem_t idxProvideVersion = 0;
1115    if (Version.empty() == false) {
1116       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1117       if (unlikely(idxProvideVersion == 0))
1118          return false;
1119    }
1120    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1121 }
1122 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1123                                     pkgCache::PkgIterator &Pkg,
1124                                     map_pointer_t const ProvideVersion,
1125                                     uint8_t const Flags)
1126 {
1127    // Get a structure
1128    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1129    if (unlikely(Provides == 0))
1130       return false;
1131    ++Cache.HeaderP->ProvidesCount;
1132
1133    // Fill it in
1134    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1135    Prv->Version = Ver.Index();
1136    Prv->ProvideVersion = ProvideVersion;
1137    Prv->Flags = Flags;
1138    Prv->NextPkgProv = Ver->ProvidesList;
1139    Ver->ProvidesList = Prv.Index();
1140
1141    // Link it to the package
1142    Prv->ParentPkg = Pkg.Index();
1143    Prv->NextProvides = Pkg->ProvidesList;
1144    Pkg->ProvidesList = Prv.Index();
1145    return true;
1146 }
1147                                                                         /*}}}*/
1148 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1149 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1150                                 StringView Version, uint8_t const Flags) {
1151    pkgCache &Cache = Owner->Cache;
1152    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1153    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1154    Dynamic<StringView> DynPackage(Package);
1155    Dynamic<StringView> DynVersion(Version);
1156
1157    if (Grp.end() == true)
1158       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1159    else
1160    {
1161       map_stringitem_t idxProvideVersion = 0;
1162       if (Version.empty() == false) {
1163          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1164          if (unlikely(idxProvideVersion == 0))
1165             return false;
1166       }
1167
1168       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1169       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1170       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1171       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1172       pkgCache::PkgIterator Pkg;
1173       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1174       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1175       {
1176          if (isImplicit && OwnerPkg == Pkg)
1177             continue;
1178          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1179             continue;
1180          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1181             return false;
1182       }
1183    }
1184    return true;
1185 }
1186                                                                         /*}}}*/
1187 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1188       pkgCache::VerIterator const &Ver)
1189 {
1190    return Hash == Ver->Hash;
1191 }
1192                                                                         /*}}}*/
1193 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1194 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1195                                    unsigned long Flags)
1196 {
1197    if (File.empty() && Site.empty())
1198    {
1199       CurrentRlsFile = NULL;
1200       return true;
1201    }
1202
1203    // Get some space for the structure
1204    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1205    if (unlikely(idxFile == 0))
1206       return false;
1207    CurrentRlsFile = Cache.RlsFileP + idxFile;
1208
1209    // Fill it in
1210    map_stringitem_t const idxFileName = WriteStringInMap(File);
1211    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1212    if (unlikely(idxFileName == 0 || idxSite == 0))
1213       return false;
1214    CurrentRlsFile->FileName = idxFileName;
1215    CurrentRlsFile->Site = idxSite;
1216    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1217    CurrentRlsFile->Flags = Flags;
1218    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1219    RlsFileName = File;
1220    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1221    Cache.HeaderP->ReleaseFileCount++;
1222
1223    return true;
1224 }
1225                                                                         /*}}}*/
1226 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1227 // ---------------------------------------------------------------------
1228 /* This is used to select which file is to be associated with all newly
1229    added versions. The caller is responsible for setting the IMS fields. */
1230 bool pkgCacheGenerator::SelectFile(std::string const &File,
1231                                    pkgIndexFile const &Index,
1232                                    std::string const &Architecture,
1233                                    std::string const &Component,
1234                                    unsigned long const Flags)
1235 {
1236    // Get some space for the structure
1237    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1238    if (unlikely(idxFile == 0))
1239       return false;
1240    CurrentFile = Cache.PkgFileP + idxFile;
1241
1242    // Fill it in
1243    map_stringitem_t const idxFileName = WriteStringInMap(File);
1244    if (unlikely(idxFileName == 0))
1245       return false;
1246    CurrentFile->FileName = idxFileName;
1247    CurrentFile->NextFile = Cache.HeaderP->FileList;
1248    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1249    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1250    if (unlikely(idxIndexType == 0))
1251       return false;
1252    CurrentFile->IndexType = idxIndexType;
1253    if (Architecture.empty())
1254       CurrentFile->Architecture = 0;
1255    else
1256    {
1257       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1258       if (unlikely(arch == 0))
1259          return false;
1260       CurrentFile->Architecture = arch;
1261    }
1262    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1263    if (unlikely(component == 0))
1264       return false;
1265    CurrentFile->Component = component;
1266    CurrentFile->Flags = Flags;
1267    if (CurrentRlsFile != NULL)
1268       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1269    else
1270       CurrentFile->Release = 0;
1271    PkgFileName = File;
1272    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1273    Cache.HeaderP->PackageFileCount++;
1274
1275    if (Progress != 0)
1276       Progress->SubProgress(Index.Size());
1277    return true;
1278 }
1279                                                                         /*}}}*/
1280 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1281 // ---------------------------------------------------------------------
1282 /* This is used to create handles to strings. Given the same text it
1283    always returns the same number */
1284 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1285                                                  unsigned int Size)
1286 {
1287    auto strings = &strMixed;
1288    switch(type) {
1289       case MIXED: strings = &strMixed; break;
1290       case PKGNAME: strings = &strPkgNames; break;
1291       case VERSIONNUMBER: strings = &strVersions; break;
1292       case SECTION: strings = &strSections; break;
1293       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1294    }
1295
1296    auto const item = strings->find({S, Size, nullptr, 0});
1297    if (item != strings->end())
1298       return item->item;
1299
1300    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1301    strings->insert({nullptr, Size, this, idxString});
1302    return idxString;
1303 }
1304                                                                         /*}}}*/
1305 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1306 // ---------------------------------------------------------------------
1307 /* This just verifies that each file in the list of index files exists,
1308    has matching attributes with the cache and the cache does not have
1309    any extra files. */
1310 class APT_HIDDEN ScopedErrorRevert {
1311 public:
1312    ScopedErrorRevert() { _error->PushToStack(); }
1313    ~ScopedErrorRevert() { _error->RevertToStack(); }
1314 };
1315 static bool CheckValidity(const string &CacheFile,
1316                           pkgSourceList &List,
1317                           FileIterator const Start,
1318                           FileIterator const End,
1319                           MMap **OutMap = 0,
1320                           pkgCache **OutCache = 0)
1321 {
1322    ScopedErrorRevert ser;
1323    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1324    // No file, certainly invalid
1325    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1326    {
1327       if (Debug == true)
1328          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1329       return false;
1330    }
1331
1332    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1333    {
1334       if (Debug == true)
1335          std::clog << "sources.list is newer than the cache" << std::endl;
1336       return false;
1337    }
1338
1339    // Map it
1340    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1341    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1342    if (unlikely(Map->validData()) == false)
1343       return false;
1344    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1345    pkgCache &Cache = *CacheP.get();
1346    if (_error->PendingError() || Map->Size() == 0)
1347    {
1348       if (Debug == true)
1349          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1350       return false;
1351    }
1352
1353    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1354    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1355    std::vector<pkgIndexFile *> Files;
1356    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1357    {
1358       if (Debug == true)
1359          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1360       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1361       if (RlsFile.end() == true)
1362       {
1363          if (Debug == true)
1364             std::clog << "FindInCache returned end-Pointer" << std::endl;
1365          return false;
1366       }
1367
1368       RlsVisited[RlsFile->ID] = true;
1369       if (Debug == true)
1370          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1371
1372       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1373       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1374             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1375    }
1376    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1377       if (RlsVisited[I] == false)
1378       {
1379          if (Debug == true)
1380             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1381          return false;
1382       }
1383
1384    std::copy(Start, End, std::back_inserter(Files));
1385
1386    /* Now we check every index file, see if it is in the cache,
1387       verify the IMS data and check that it is on the disk too.. */
1388    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1389    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1390    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1391    {
1392       if (Debug == true)
1393          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1394       if ((*PkgFile)->Exists() == false)
1395       {
1396          if (Debug == true)
1397             std::clog << "file doesn't exist" << std::endl;
1398          continue;
1399       }
1400
1401       // FindInCache is also expected to do an IMS check.
1402       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1403       if (File.end() == true)
1404       {
1405          if (Debug == true)
1406             std::clog << "FindInCache returned end-Pointer" << std::endl;
1407          return false;
1408       }
1409
1410       Visited[File->ID] = true;
1411       if (Debug == true)
1412          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1413    }
1414
1415    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1416       if (Visited[I] == false)
1417       {
1418          if (Debug == true)
1419             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1420          return false;
1421       }
1422
1423    if (_error->PendingError() == true)
1424    {
1425       if (Debug == true)
1426       {
1427          std::clog << "Validity failed because of pending errors:" << std::endl;
1428          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1429       }
1430       return false;
1431    }
1432
1433    if (OutMap != 0)
1434       *OutMap = Map.release();
1435    if (OutCache != 0)
1436       *OutCache = CacheP.release();
1437    return true;
1438 }
1439                                                                         /*}}}*/
1440 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1441 // ---------------------------------------------------------------------
1442 /* Size is kind of an abstract notion that is only used for the progress
1443    meter */
1444 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1445 {
1446    map_filesize_t TotalSize = 0;
1447    if (List !=  NULL)
1448    {
1449       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1450       {
1451          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1452          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1453             if ((*j)->HasPackages() == true)
1454                TotalSize += (*j)->Size();
1455       }
1456    }
1457
1458    for (; Start < End; ++Start)
1459    {
1460       if ((*Start)->HasPackages() == false)
1461          continue;
1462       TotalSize += (*Start)->Size();
1463    }
1464    return TotalSize;
1465 }
1466                                                                         /*}}}*/
1467 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1468 static bool BuildCache(pkgCacheGenerator &Gen,
1469                        OpProgress * const Progress,
1470                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1471                        pkgSourceList const * const List,
1472                        FileIterator const Start, FileIterator const End)
1473 {
1474    bool mergeFailure = false;
1475
1476    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1477       if (I->HasPackages() == false || mergeFailure)
1478          return;
1479
1480       if (I->Exists() == false)
1481          return;
1482
1483       if (I->FindInCache(Gen.GetCache()).end() == false)
1484       {
1485          _error->Warning("Duplicate sources.list entry %s",
1486                I->Describe().c_str());
1487          return;
1488       }
1489
1490       map_filesize_t const Size = I->Size();
1491       if (Progress != NULL)
1492          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1493       CurrentSize += Size;
1494
1495       if (I->Merge(Gen,Progress) == false)
1496          mergeFailure = true;
1497    };
1498
1499    if (List !=  NULL)
1500    {
1501       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1502       {
1503          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1504          {
1505             _error->Warning("Duplicate sources.list entry %s",
1506                   (*i)->Describe().c_str());
1507             continue;
1508          }
1509
1510          if ((*i)->Merge(Gen, Progress) == false)
1511             return false;
1512
1513          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1514          if (Indexes != NULL)
1515             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1516          if (mergeFailure)
1517             return false;
1518       }
1519    }
1520
1521    if (Start != End)
1522    {
1523       Gen.SelectReleaseFile("", "");
1524       std::for_each(Start, End, indexFileMerge);
1525       if (mergeFailure)
1526          return false;
1527    }
1528    return true;
1529 }
1530                                                                         /*}}}*/
1531 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1532 // ---------------------------------------------------------------------
1533 /* This makes sure that the status cache (the cache that has all
1534    index files from the sources list and all local ones) is ready
1535    to be mmaped. If OutMap is not zero then a MMap object representing
1536    the cache will be stored there. This is pretty much mandetory if you
1537    are using AllowMem. AllowMem lets the function be run as non-root
1538    where it builds the cache 'fast' into a memory buffer. */
1539 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1540 {
1541    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1542    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1543    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1544    Flags |= MMap::Moveable;
1545    if (_config->FindB("APT::Cache-Fallback", false) == true)
1546       Flags |= MMap::Fallback;
1547    if (CacheF != NULL)
1548       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1549    else
1550       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1551 }
1552 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1553       std::string const &FileName)
1554 {
1555    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1556    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1557       return false;
1558
1559    fchmod(SCacheF.Fd(),0644);
1560
1561    // Write out the main data
1562    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1563       return _error->Error(_("IO Error saving source cache"));
1564
1565    // Write out the proper header
1566    Gen->GetCache().HeaderP->Dirty = false;
1567    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1568    if (SCacheF.Seek(0) == false ||
1569          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1570       return _error->Error(_("IO Error saving source cache"));
1571    Gen->GetCache().HeaderP->Dirty = true;
1572    return true;
1573 }
1574 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1575       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1576 {
1577    Map.reset(CreateDynamicMMap(NULL, 0));
1578    if (unlikely(Map->validData()) == false)
1579       return false;
1580    FileFd CacheF(FileName, FileFd::ReadOnly);
1581    if (CacheF.IsOpen() == false || CacheF.Failed())
1582       return false;
1583    _error->PushToStack();
1584    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1585    bool const newError = _error->PendingError();
1586    _error->MergeWithStack();
1587    if (alloc == 0 && newError)
1588       return false;
1589    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1590       return false;
1591    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1592    return true;
1593 }
1594 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1595                         MMap **OutMap, bool AllowMem)
1596    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1597 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1598                         MMap **OutMap,bool)
1599 {
1600    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1601 }
1602 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1603                         MMap **OutMap,pkgCache **OutCache, bool)
1604 {
1605    // FIXME: deprecate the ignored AllowMem parameter
1606    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1607
1608    std::vector<pkgIndexFile *> Files;
1609    if (_system->AddStatusFiles(Files) == false)
1610       return false;
1611
1612    // Decide if we can write to the files..
1613    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1614    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1615
1616    // ensure the cache directory exists
1617    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1618    {
1619       string dir = _config->FindDir("Dir::Cache");
1620       size_t const len = dir.size();
1621       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1622          dir = dir.substr(0, len - 5);
1623       if (CacheFile.empty() == false)
1624          CreateDirectory(dir, flNotFile(CacheFile));
1625       if (SrcCacheFile.empty() == false)
1626          CreateDirectory(dir, flNotFile(SrcCacheFile));
1627    }
1628
1629    if (Progress != NULL)
1630       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1631
1632    bool pkgcache_fine = false;
1633    bool srcpkgcache_fine = false;
1634    bool volatile_fine = List.GetVolatileFiles().empty();
1635
1636    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1637                      volatile_fine ? OutCache : NULL) == true)
1638    {
1639       if (Debug == true)
1640          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1641       pkgcache_fine = true;
1642       srcpkgcache_fine = true;
1643    }
1644    if (pkgcache_fine == false)
1645    {
1646       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1647       {
1648          if (Debug == true)
1649             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1650          srcpkgcache_fine = true;
1651       }
1652    }
1653
1654    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1655    {
1656       if (Progress != NULL)
1657          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1658       return true;
1659    }
1660
1661    bool Writeable = false;
1662    if (srcpkgcache_fine == false || pkgcache_fine == false)
1663    {
1664       if (CacheFile.empty() == false)
1665          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1666       else if (SrcCacheFile.empty() == false)
1667          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1668
1669       if (Debug == true)
1670          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1671    }
1672
1673    // At this point we know we need to construct something, so get storage ready
1674    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1675    if (unlikely(Map->validData()) == false)
1676       return false;
1677    if (Debug == true)
1678       std::clog << "Open memory Map (not filebased)" << std::endl;
1679
1680    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1681    map_filesize_t CurrentSize = 0;
1682    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1683    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1684    if (srcpkgcache_fine == true && pkgcache_fine == false)
1685    {
1686       if (Debug == true)
1687          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1688       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1689          return false;
1690       srcpkgcache_fine = true;
1691       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1692    }
1693    else if (srcpkgcache_fine == false)
1694    {
1695       if (Debug == true)
1696          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1697       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1698
1699       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1700       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1701                Files.end(),Files.end()) == false)
1702          return false;
1703
1704       if (Writeable == true && SrcCacheFile.empty() == false)
1705          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1706             return false;
1707    }
1708
1709    if (pkgcache_fine == false)
1710    {
1711       if (Debug == true)
1712          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1713       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1714                Files.begin(), Files.end()) == false)
1715          return false;
1716
1717       if (Writeable == true && CacheFile.empty() == false)
1718          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1719             return false;
1720    }
1721
1722    if (Debug == true)
1723       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1724
1725    if (volatile_fine == false)
1726    {
1727       if (Gen == nullptr)
1728       {
1729          if (Debug == true)
1730             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1731          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1732             return false;
1733       }
1734
1735       Files = List.GetVolatileFiles();
1736       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1737                Files.begin(), Files.end()) == false)
1738          return false;
1739    }
1740
1741    if (OutMap != nullptr)
1742       *OutMap = Map.release();
1743
1744    if (Debug == true)
1745       std::clog << "Everything is ready for shipping" << std::endl;
1746    return true;
1747 }
1748                                                                         /*}}}*/
1749 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1750 class APT_HIDDEN ScopedErrorMerge {
1751 public:
1752    ScopedErrorMerge() { _error->PushToStack(); }
1753    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1754 };
1755 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1756    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1757 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1758 {
1759    std::vector<pkgIndexFile *> Files;
1760    if (_system->AddStatusFiles(Files) == false)
1761       return false;
1762
1763    ScopedErrorMerge sem;
1764    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1765    if (unlikely(Map->validData()) == false)
1766       return false;
1767    map_filesize_t CurrentSize = 0;
1768    map_filesize_t TotalSize = 0;
1769    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1770
1771    // Build the status cache
1772    if (Progress != NULL)
1773       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1774    pkgCacheGenerator Gen(Map.get(),Progress);
1775    if (_error->PendingError() == true)
1776       return false;
1777    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1778                   Files.begin(), Files.end()) == false)
1779       return false;
1780
1781    if (_error->PendingError() == true)
1782       return false;
1783    *OutMap = Map.release();
1784
1785    return true;
1786 }
1787                                                                         /*}}}*/
1788 // IsDuplicateDescription                                               /*{{{*/
1789 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1790                             MD5SumValue const &CurMd5, std::string const &CurLang)
1791 {
1792    // Descriptions in the same link-list have all the same md5
1793    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1794       return false;
1795    for (; Desc.end() == false; ++Desc)
1796       if (Desc.LanguageCode() == CurLang)
1797          return true;
1798    return false;
1799 }
1800                                                                         /*}}}*/
1801
1802 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1803 pkgCacheListParser::~pkgCacheListParser() {}