apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61 }
  62 bool pkgCacheGenerator::Start()
  63 {
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       _error->PushToStack();
  69       Map.RawAllocate(sizeof(pkgCache::Header));
  70       bool const newError = _error->PendingError();
  71       _error->MergeWithStack();
  72       if (newError)
  73          return false;
  74       if (Map.Size() <= 0)
  75          return false;
  76
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78
  79       // Starting header
  80       *Cache.HeaderP = pkgCache::Header();
  81
  82       // make room for the hashtables for packages and groups
  83       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  84          return false;
  85
  86       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  87       if (unlikely(idxVerSysName == 0))
  88          return false;
  89       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  90       if (unlikely(idxArchitecture == 0))
  91          return false;
  92       map_stringitem_t idxArchitectures;
  93
  94       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  95       if (archs.size() > 1)
  96       {
  97          std::vector<std::string>::const_iterator a = archs.begin();
  98          std::string list = *a;
  99          for (++a; a != archs.end(); ++a)
 100             list.append(",").append(*a);
 101          idxArchitectures = WriteStringInMap(list);
 102          if (unlikely(idxArchitectures == 0))
 103             return false;
 104       }
 105       else
 106          idxArchitectures = idxArchitecture;
 107
 108       Cache.HeaderP = (pkgCache::Header *)Map.Data();
 109       Cache.HeaderP->VerSysName = idxVerSysName;
 110       Cache.HeaderP->Architecture = idxArchitecture;
 111       Cache.HeaderP->SetArchitectures(idxArchitectures);
 112
 113       // Calculate the hash for the empty map, so ReMap does not fail
 114       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 115       Cache.ReMap();
 116    }
 117    else
 118    {
 119       // Map directly from the existing file
 120       Cache.ReMap();
 121       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 122       if (Cache.VS != _system->VS)
 123          return _error->Error(_("Cache has an incompatible versioning system"));
 124    }
 125
 126    Cache.HeaderP->Dirty = true;
 127    Map.Sync(0,sizeof(pkgCache::Header));
 128    return true;
 129 }
 130                                                                         /*}}}*/
 131 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* We sync the data then unset the dirty flag in two steps so as to
 134    advoid a problem during a crash */
 135 pkgCacheGenerator::~pkgCacheGenerator()
 136 {
 137    if (_error->PendingError() == true || Map.validData() == false)
 138       return;
 139    if (Map.Sync() == false)
 140       return;
 141
 142    Cache.HeaderP->Dirty = false;
 143    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 147    Map.Sync(0,sizeof(pkgCache::Header));
 148 }
 149                                                                         /*}}}*/
 150 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 151    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
 152    // returns (something, true) the iterator was not yet seen and we can
 153    // remap it.
 154    std::unordered_set<void *> seen;
 155    if (oldMap == newMap)
 156       return;
 157
 158    if (_config->FindB("Debug::pkgCacheGen", false))
 159       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 160
 161    Cache.ReMap(false);
 162
 163    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 164    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 165
 166    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 168       if (std::get<1>(seen.insert(*i)) == true)
 169          (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 172       if (std::get<1>(seen.insert(*i)) == true)
 173          (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 176       if (std::get<1>(seen.insert(*i)) == true)
 177          (*i)->ReMap(oldMap, newMap);
 178    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 179         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 180         if (std::get<1>(seen.insert(*i)) == true)
 181          (*i)->ReMap(oldMap, newMap);
 182    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 183         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 184       if (std::get<1>(seen.insert(*i)) == true)
 185          (*i)->ReMap(oldMap, newMap);
 186    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 187         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 188       if (std::get<1>(seen.insert(*i)) == true)
 189          (*i)->ReMap(oldMap, newMap);
 190    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 191         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 192       if (std::get<1>(seen.insert(*i)) == true)
 193          (*i)->ReMap(oldMap, newMap);
 194    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 195         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 196       if (std::get<1>(seen.insert(*i)) == true)
 197          (*i)->ReMap(oldMap, newMap);
 198    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
 199       if (std::get<1>(seen.insert(ViewP)) == false)
 200          continue;
 201       // Ignore views outside of the cache.
 202       if (ViewP->data() < static_cast<const char*>(oldMap)
 203          || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
 204          continue;
 205       const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
 206       *ViewP = StringView(data , ViewP->size());
 207    }
 208 }                                                                       /*}}}*/
 209 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 210 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 211                                         const unsigned long &Len) {
 212    size_t oldSize = Map.Size();
 213    void const * const oldMap = Map.Data();
 214    map_stringitem_t const index = Map.WriteString(String, Len);
 215    if (index != 0)
 216       ReMap(oldMap, Map.Data(), oldSize);
 217    return index;
 218 }
 219                                                                         /*}}}*/
 220 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 221 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 222    size_t oldSize = Map.Size();
 223    void const * const oldMap = Map.Data();
 224    map_stringitem_t const index = Map.WriteString(String);
 225    if (index != 0)
 226       ReMap(oldMap, Map.Data(), oldSize);
 227    return index;
 228 }
 229                                                                         /*}}}*/
 230 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 231    size_t oldSize = Map.Size();
 232    void const * const oldMap = Map.Data();
 233    map_pointer_t const index = Map.Allocate(size);
 234    if (index != 0)
 235       ReMap(oldMap, Map.Data(), oldSize);
 236    return index;
 237 }
 238                                                                         /*}}}*/
 239 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This provides the generation of the entries in the cache. Each loop
 242    goes through a single package record from the underlying parse engine. */
 243 bool pkgCacheGenerator::MergeList(ListParser &List,
 244                                   pkgCache::VerIterator *OutVer)
 245 {
 246    List.Owner = this;
 247
 248    unsigned int Counter = 0;
 249    while (List.Step() == true)
 250    {
 251       string const PackageName = List.Package();
 252       if (PackageName.empty() == true)
 253          return false;
 254
 255       Counter++;
 256       if (Counter % 100 == 0 && Progress != 0)
 257          Progress->Progress(List.Offset());
 258
 259       APT::StringView Arch = List.Architecture();
 260       Dynamic<APT::StringView> DynArch(Arch);
 261       APT::StringView Version = List.Version();
 262       Dynamic<APT::StringView> DynVersion(Version);
 263       if (Version.empty() == true && Arch.empty() == true)
 264       {
 265          // package descriptions
 266          if (MergeListGroup(List, PackageName) == false)
 267             return false;
 268          continue;
 269       }
 270
 271       // Get a pointer to the package structure
 272       pkgCache::PkgIterator Pkg;
 273       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 274       if (NewPackage(Pkg, PackageName, Arch) == false)
 275          // TRANSLATOR: The first placeholder is a package name,
 276          // the other two should be copied verbatim as they include debug info
 277          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 278                               PackageName.c_str(), "NewPackage", 1);
 279
 280
 281       if (Version.empty() == true)
 282       {
 283          if (MergeListPackage(List, Pkg) == false)
 284             return false;
 285       }
 286       else
 287       {
 288          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 289             return false;
 290       }
 291
 292       if (OutVer != 0)
 293          return true;
 294    }
 295
 296    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 297       return _error->Error(_("Wow, you exceeded the number of package "
 298                              "names this APT is capable of."));
 299    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 300       return _error->Error(_("Wow, you exceeded the number of versions "
 301                              "this APT is capable of."));
 302    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 303       return _error->Error(_("Wow, you exceeded the number of descriptions "
 304                              "this APT is capable of."));
 305    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 306       return _error->Error(_("Wow, you exceeded the number of dependencies "
 307                              "this APT is capable of."));
 308
 309    return true;
 310 }
 311 // CacheGenerator::MergeListGroup                                       /*{{{*/
 312 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 313 {
 314    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 315    // a group has no data on it's own, only packages have it but these
 316    // stanzas like this come from Translation- files to add descriptions,
 317    // but without a version we don't need a description for it…
 318    if (Grp.end() == true)
 319       return true;
 320    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 321
 322    pkgCache::PkgIterator Pkg;
 323    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 324    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 325       if (MergeListPackage(List, Pkg) == false)
 326          return false;
 327
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // CacheGenerator::MergeListPackage                                     /*{{{*/
 332 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 333 {
 334    // we first process the package, then the descriptions
 335    // (for deb this package processing is in fact a no-op)
 336    pkgCache::VerIterator Ver(Cache);
 337    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 338    if (List.UsePackage(Pkg, Ver) == false)
 339       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 340                            Pkg.Name(), "UsePackage", 1);
 341
 342    // Find the right version to write the description
 343    MD5SumValue CurMd5 = List.Description_md5();
 344    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 345    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 346    {
 347       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 348
 349       // a version can only have one md5 describing it
 350       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 351          continue;
 352
 353       map_stringitem_t md5idx = VerDesc->md5sum;
 354       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 355       {
 356          // don't add a new description if we have one for the given
 357          // md5 && language
 358          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 359             continue;
 360
 361          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 362       }
 363
 364       // we can stop here as all "same" versions will share the description
 365       break;
 366    }
 367
 368    return true;
 369 }
 370                                                                         /*}}}*/
 371 // CacheGenerator::MergeListVersion                                     /*{{{*/
 372 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 373                                          APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 374 {
 375    pkgCache::VerIterator Ver = Pkg.VersionList();
 376    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 377    map_pointer_t *LastVer = &Pkg->VersionList;
 378    void const * oldMap = Map.Data();
 379
 380    unsigned short const Hash = List.VersionHash();
 381    if (Ver.end() == false)
 382    {
 383       /* We know the list is sorted so we use that fact in the search.
 384          Insertion of new versions is done with correct sorting */
 385       int Res = 1;
 386       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 387       {
 388          char const * const VerStr = Ver.VerStr();
 389          Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
 390                VerStr, VerStr + strlen(VerStr));
 391          // Version is higher as current version - insert here
 392          if (Res > 0)
 393             break;
 394          // Versionstrings are equal - is hash also equal?
 395          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 396             break;
 397          // proceed with the next till we have either the right
 398          // or we found another version (which will be lower)
 399       }
 400
 401       /* We already have a version for this item, record that we saw it */
 402       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 403       {
 404          if (List.UsePackage(Pkg,Ver) == false)
 405             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 406                                  Pkg.Name(), "UsePackage", 2);
 407
 408          if (NewFileVer(Ver,List) == false)
 409             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 410                                  Pkg.Name(), "NewFileVer", 1);
 411
 412          // Read only a single record and return
 413          if (OutVer != 0)
 414          {
 415             *OutVer = Ver;
 416             return true;
 417          }
 418
 419          return true;
 420       }
 421    }
 422
 423    // Add a new version
 424    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 425    if (unlikely(verindex == 0))
 426       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 427                            Pkg.Name(), "NewVersion", 1);
 428
 429    if (oldMap != Map.Data())
 430          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 431    *LastVer = verindex;
 432
 433    if (unlikely(List.NewVersion(Ver) == false))
 434       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 435                            Pkg.Name(), "NewVersion", 2);
 436
 437    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 438       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 439                            Pkg.Name(), "UsePackage", 3);
 440
 441    if (unlikely(NewFileVer(Ver,List) == false))
 442       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 443                            Pkg.Name(), "NewFileVer", 2);
 444
 445    pkgCache::GrpIterator Grp = Pkg.Group();
 446    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 447
 448    /* If it is the first version of this package we need to add implicit
 449       Multi-Arch dependencies to all other package versions in the group now -
 450       otherwise we just add them for this new version */
 451    if (Pkg.VersionList()->NextVer == 0)
 452    {
 453       pkgCache::PkgIterator P = Grp.PackageList();
 454       Dynamic<pkgCache::PkgIterator> DynP(P);
 455       for (; P.end() != true; P = Grp.NextPkg(P))
 456       {
 457          if (P->ID == Pkg->ID)
 458             continue;
 459          pkgCache::VerIterator V = P.VersionList();
 460          Dynamic<pkgCache::VerIterator> DynV(V);
 461          for (; V.end() != true; ++V)
 462             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 463                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 464                                     Pkg.Name(), "AddImplicitDepends", 1);
 465       }
 466    }
 467    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 468       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 469                            Pkg.Name(), "AddImplicitDepends", 2);
 470
 471    // Read only a single record and return
 472    if (OutVer != 0)
 473    {
 474       *OutVer = Ver;
 475       return true;
 476    }
 477
 478    /* Record the Description(s) based on their master md5sum */
 479    MD5SumValue CurMd5 = List.Description_md5();
 480
 481    /* Before we add a new description we first search in the group for
 482       a version with a description of the same MD5 - if so we reuse this
 483       description group instead of creating our own for this version */
 484    for (pkgCache::PkgIterator P = Grp.PackageList();
 485         P.end() == false; P = Grp.NextPkg(P))
 486    {
 487       for (pkgCache::VerIterator V = P.VersionList();
 488            V.end() == false; ++V)
 489       {
 490          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 491             continue;
 492          Ver->DescriptionList = V->DescriptionList;
 493       }
 494    }
 495
 496    // We haven't found reusable descriptions, so add the first description(s)
 497    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 498    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 499    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 500       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 501          return false;
 502    return true;
 503 }
 504                                                                         /*}}}*/
 505 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 506 {
 507    pkgCache::DescIterator Desc;
 508    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 509
 510    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 511    if (unlikely(descindex == 0))
 512       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 513             Ver.ParentPkg().Name(), "NewDescription", 1);
 514
 515    md5idx = Desc->md5sum;
 516    Desc->ParentPkg = Ver.ParentPkg().Index();
 517
 518    // we add at the end, so that the start is constant as we need
 519    // that to be able to efficiently share these lists
 520    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 521    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 522    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 523    *LastNextDesc = descindex;
 524
 525    if (NewFileDesc(Desc,List) == false)
 526       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 527             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 528
 529    return true;
 530 }
 531                                                                         /*}}}*/
 532                                                                         /*}}}*/
 533 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 534 // ---------------------------------------------------------------------
 535 /* This creates a new group structure and adds it to the hash table */
 536 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 537 {
 538    Dynamic<StringView> DName(Name);
 539    Grp = Cache.FindGrp(Name);
 540    if (Grp.end() == false)
 541       return true;
 542
 543    // Get a structure
 544    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 545    if (unlikely(Group == 0))
 546       return false;
 547
 548    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 549    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 550    if (unlikely(idxName == 0))
 551       return false;
 552    Grp->Name = idxName;
 553
 554    // Insert it into the hash table
 555    unsigned long const Hash = Cache.Hash(Name);
 556    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 557
 558    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 559       insertAt = &(Cache.GrpP + *insertAt)->Next;
 560    Grp->Next = *insertAt;
 561    *insertAt = Group;
 562
 563    Grp->ID = Cache.HeaderP->GroupCount++;
 564    return true;
 565 }
 566                                                                         /*}}}*/
 567 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 568 // ---------------------------------------------------------------------
 569 /* This creates a new package structure and adds it to the hash table */
 570 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 571                                         StringView Arch) {
 572    pkgCache::GrpIterator Grp;
 573    Dynamic<StringView> DName(Name);
 574    Dynamic<StringView> DArch(Arch);
 575    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 576    if (unlikely(NewGroup(Grp, Name) == false))
 577       return false;
 578
 579    Pkg = Grp.FindPkg(Arch);
 580       if (Pkg.end() == false)
 581          return true;
 582
 583    // Get a structure
 584    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 585    if (unlikely(Package == 0))
 586       return false;
 587    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 588
 589    // Set the name, arch and the ID
 590    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 591    Pkg->Group = Grp.Index();
 592    // all is mapped to the native architecture
 593    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 594    if (unlikely(idxArch == 0))
 595       return false;
 596    Pkg->Arch = idxArch;
 597    Pkg->ID = Cache.HeaderP->PackageCount++;
 598
 599    // Insert the package into our package list
 600    if (Grp->FirstPackage == 0) // the group is new
 601    {
 602       Grp->FirstPackage = Package;
 603       // Insert it into the hash table
 604       map_id_t const Hash = Cache.Hash(Name);
 605       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 606       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 607          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 608       Pkg->NextPackage = *insertAt;
 609       *insertAt = Package;
 610    }
 611    else // Group the Packages together
 612    {
 613       // if sibling is provided by another package, this one is too
 614       {
 615          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 616          if (M.end() == false) {
 617             pkgCache::PrvIterator Prv;
 618             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 619             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 620             {
 621                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 622                   continue;
 623                pkgCache::VerIterator Ver = Prv.OwnerVer();
 624                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 625                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 626                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 627                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 628                {
 629                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 630                      continue;
 631                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 632                      return false;
 633                }
 634             }
 635          }
 636       }
 637       // let M-A:foreign package siblings provide this package
 638       {
 639          pkgCache::PkgIterator P;
 640          pkgCache::VerIterator Ver;
 641          Dynamic<pkgCache::PkgIterator> DynP(P);
 642          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 643
 644          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 645          {
 646             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 647                continue;
 648             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 649                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 650                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 651                      return false;
 652          }
 653       }
 654       // and negative dependencies, don't forget negative dependencies
 655       {
 656          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 657          if (M.end() == false) {
 658             pkgCache::DepIterator Dep;
 659             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 660             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 661             {
 662                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 663                   continue;
 664                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 665                      Dep->Type != pkgCache::Dep::Replaces)
 666                   continue;
 667                pkgCache::VerIterator Ver = Dep.ParentVer();
 668                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 669                map_pointer_t * unused = NULL;
 670                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 671                   return false;
 672             }
 673          }
 674       }
 675
 676       // this package is the new last package
 677       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 678       Pkg->NextPackage = LastPkg->NextPackage;
 679       LastPkg->NextPackage = Package;
 680    }
 681    Grp->LastPackage = Package;
 682
 683    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 684    if (Arch == "any")
 685    {
 686       size_t const found = Name.rfind(':');
 687       StringView ArchA = Name.substr(found + 1);
 688       if (ArchA != "any")
 689       {
 690          // ArchA is used inside the loop which might remap (NameA is not used)
 691          Dynamic<StringView> DynArchA(ArchA);
 692          StringView NameA = Name.substr(0, found);
 693          pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 694          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 695          if (PkgA.end())
 696          {
 697             Dynamic<StringView> DynNameA(NameA);
 698             if (NewPackage(PkgA, NameA, ArchA) == false)
 699                return false;
 700          }
 701          if (unlikely(PkgA.end()))
 702             return _error->Fatal("NewPackage was successful for %s:%s,"
 703                   "but the package doesn't exist anyhow!",
 704                   NameA.to_string().c_str(), ArchA.to_string().c_str());
 705          else
 706          {
 707             pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 708             for (; Prv.end() == false; ++Prv)
 709             {
 710                if (Prv.IsMultiArchImplicit())
 711                   continue;
 712                pkgCache::VerIterator V = Prv.OwnerVer();
 713                if (ArchA != V.ParentPkg().Arch())
 714                   continue;
 715                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 716                   return false;
 717             }
 718             pkgCache::VerIterator V = PkgA.VersionList();
 719             Dynamic<pkgCache::VerIterator> DynV(V);
 720             for (; V.end() == false; ++V)
 721             {
 722                if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 723                   return false;
 724             }
 725          }
 726       }
 727    }
 728    return true;
 729 }
 730                                                                         /*}}}*/
 731 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 732 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 733                                            pkgCache::PkgIterator &P,
 734                                            pkgCache::VerIterator &V)
 735 {
 736    APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
 737    Dynamic<APT::StringView> DynArch(Arch);
 738    map_pointer_t *OldDepLast = NULL;
 739    /* MultiArch handling introduces a lot of implicit Dependencies:
 740       - MultiArch: same → Co-Installable if they have the same version
 741       - All others conflict with all other group members */
 742    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 743    pkgCache::PkgIterator D = G.PackageList();
 744    Dynamic<pkgCache::PkgIterator> DynD(D);
 745    map_stringitem_t const VerStrIdx = V->VerStr;
 746    for (; D.end() != true; D = G.NextPkg(D))
 747    {
 748       if (Arch == D.Arch() || D->VersionList == 0)
 749          continue;
 750       /* We allow only one installed arch at the time
 751          per group, therefore each group member conflicts
 752          with all other group members */
 753       if (coInstall == true)
 754       {
 755          // Replaces: ${self}:other ( << ${binary:Version})
 756          NewDepends(D, V, VerStrIdx,
 757                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 758                     OldDepLast);
 759          // Breaks: ${self}:other (!= ${binary:Version})
 760          NewDepends(D, V, VerStrIdx,
 761                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 762                     OldDepLast);
 763       } else {
 764          // Conflicts: ${self}:other
 765          NewDepends(D, V, 0,
 766                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 767                     OldDepLast);
 768       }
 769    }
 770    return true;
 771 }
 772 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 773                                            pkgCache::PkgIterator &D)
 774 {
 775    /* MultiArch handling introduces a lot of implicit Dependencies:
 776       - MultiArch: same → Co-Installable if they have the same version
 777       - All others conflict with all other group members */
 778    map_pointer_t *OldDepLast = NULL;
 779    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 780    if (coInstall == true)
 781    {
 782       map_stringitem_t const VerStrIdx = V->VerStr;
 783       // Replaces: ${self}:other ( << ${binary:Version})
 784       NewDepends(D, V, VerStrIdx,
 785                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 786                  OldDepLast);
 787       // Breaks: ${self}:other (!= ${binary:Version})
 788       NewDepends(D, V, VerStrIdx,
 789                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 790                  OldDepLast);
 791    } else {
 792       // Conflicts: ${self}:other
 793       NewDepends(D, V, 0,
 794                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 795                  OldDepLast);
 796    }
 797    return true;
 798 }
 799
 800                                                                         /*}}}*/
 801 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 802 // ---------------------------------------------------------------------
 803 /* */
 804 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 805                                    ListParser &List)
 806 {
 807    if (CurrentFile == 0)
 808       return true;
 809
 810    // Get a structure
 811    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 812    if (VerFile == 0)
 813       return false;
 814
 815    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 816    VF->File = CurrentFile - Cache.PkgFileP;
 817
 818    // Link it to the end of the list
 819    map_pointer_t *Last = &Ver->FileList;
 820    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 821       Last = &V->NextFile;
 822    VF->NextFile = *Last;
 823    *Last = VF.Index();
 824
 825    VF->Offset = List.Offset();
 826    VF->Size = List.Size();
 827    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 828       Cache.HeaderP->MaxVerFileSize = VF->Size;
 829    Cache.HeaderP->VerFileCount++;
 830
 831    return true;
 832 }
 833                                                                         /*}}}*/
 834 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 835 // ---------------------------------------------------------------------
 836 /* This puts a version structure in the linked list */
 837 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 838                                             APT::StringView const &VerStr,
 839                                             map_pointer_t const ParentPkg,
 840                                             unsigned short const Hash,
 841                                             map_pointer_t const Next)
 842 {
 843    // Get a structure
 844    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 845    if (Version == 0)
 846       return 0;
 847
 848    // Fill it in
 849    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 850    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 851    Ver->NextVer = Next;
 852    Ver->ParentPkg = ParentPkg;
 853    Ver->Hash = Hash;
 854    Ver->ID = Cache.HeaderP->VersionCount++;
 855
 856    // try to find the version string in the group for reuse
 857    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 858    pkgCache::GrpIterator Grp = Pkg.Group();
 859    if (Pkg.end() == false && Grp.end() == false)
 860    {
 861       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 862       {
 863          if (Pkg == P)
 864             continue;
 865          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 866          {
 867             int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
 868             if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
 869             {
 870                Ver->VerStr = V->VerStr;
 871                return Version;
 872             }
 873             else if (cmp < 0)
 874                break;
 875          }
 876       }
 877    }
 878    // haven't found the version string, so create
 879    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 880    if (unlikely(idxVerStr == 0))
 881       return 0;
 882    Ver->VerStr = idxVerStr;
 883    return Version;
 884 }
 885                                                                         /*}}}*/
 886 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 887 // ---------------------------------------------------------------------
 888 /* */
 889 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 890                                    ListParser &List)
 891 {
 892    if (CurrentFile == 0)
 893       return true;
 894
 895    // Get a structure
 896    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 897    if (DescFile == 0)
 898       return false;
 899
 900    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 901    DF->File = CurrentFile - Cache.PkgFileP;
 902
 903    // Link it to the end of the list
 904    map_pointer_t *Last = &Desc->FileList;
 905    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 906       Last = &D->NextFile;
 907
 908    DF->NextFile = *Last;
 909    *Last = DF.Index();
 910
 911    DF->Offset = List.Offset();
 912    DF->Size = List.Size();
 913    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 914       Cache.HeaderP->MaxDescFileSize = DF->Size;
 915    Cache.HeaderP->DescFileCount++;
 916
 917    return true;
 918 }
 919                                                                         /*}}}*/
 920 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 921 // ---------------------------------------------------------------------
 922 /* This puts a description structure in the linked list */
 923 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 924                                             const string &Lang,
 925                                             const MD5SumValue &md5sum,
 926                                             map_stringitem_t const idxmd5str)
 927 {
 928    // Get a structure
 929    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 930    if (Description == 0)
 931       return 0;
 932
 933    // Fill it in
 934    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 935    Desc->ID = Cache.HeaderP->DescriptionCount++;
 936    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 937    if (unlikely(idxlanguage_code == 0))
 938       return 0;
 939    Desc->language_code = idxlanguage_code;
 940
 941    if (idxmd5str != 0)
 942       Desc->md5sum = idxmd5str;
 943    else
 944    {
 945       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 946       if (unlikely(idxmd5sum == 0))
 947          return 0;
 948       Desc->md5sum = idxmd5sum;
 949    }
 950
 951    return Description;
 952 }
 953                                                                         /*}}}*/
 954 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 955 // ---------------------------------------------------------------------
 956 /* This creates a dependency element in the tree. It is linked to the
 957    version and to the package that it is pointing to. */
 958 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 959                                    pkgCache::VerIterator &Ver,
 960                                    map_pointer_t const Version,
 961                                    uint8_t const Op,
 962                                    uint8_t const Type,
 963                                    map_pointer_t* &OldDepLast)
 964 {
 965    void const * const oldMap = Map.Data();
 966    // Get a structure
 967    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 968    if (unlikely(Dependency == 0))
 969       return false;
 970
 971    bool isDuplicate = false;
 972    map_pointer_t DependencyData = 0;
 973    map_pointer_t PreviousData = 0;
 974    if (Pkg->RevDepends != 0)
 975    {
 976       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 977       DependencyData = L->DependencyData;
 978       do {
 979          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 980          if (Version > D->Version)
 981             break;
 982          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 983          {
 984             isDuplicate = true;
 985             break;
 986          }
 987          PreviousData = DependencyData;
 988          DependencyData = D->NextData;
 989       } while (DependencyData != 0);
 990    }
 991
 992    if (isDuplicate == false)
 993    {
 994       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 995       if (unlikely(DependencyData == 0))
 996         return false;
 997    }
 998
 999    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1000    Link->ParentVer = Ver.Index();
1001    Link->DependencyData = DependencyData;
1002    Link->ID = Cache.HeaderP->DependsCount++;
1003
1004    pkgCache::DepIterator Dep(Cache, Link);
1005    if (isDuplicate == false)
1006    {
1007       Dep->Type = Type;
1008       Dep->CompareOp = Op;
1009       Dep->Version = Version;
1010       Dep->Package = Pkg.Index();
1011       ++Cache.HeaderP->DependsDataCount;
1012       if (PreviousData != 0)
1013       {
1014          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1015          Dep->NextData = D->NextData;
1016          D->NextData = DependencyData;
1017       }
1018       else if (Pkg->RevDepends != 0)
1019       {
1020          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1021          Dep->NextData = D->DependencyData;
1022       }
1023    }
1024
1025    if (isDuplicate == true || PreviousData != 0)
1026    {
1027       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1028       Link->NextRevDepends = L->NextRevDepends;
1029       L->NextRevDepends = Dependency;
1030    }
1031    else
1032    {
1033       Link->NextRevDepends = Pkg->RevDepends;
1034       Pkg->RevDepends = Dependency;
1035    }
1036
1037
1038    // Do we know where to link the Dependency to?
1039    if (OldDepLast == NULL)
1040    {
1041       OldDepLast = &Ver->DependsList;
1042       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1043          OldDepLast = &D->NextDepends;
1044    } else if (oldMap != Map.Data())
1045       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1046
1047    Dep->NextDepends = *OldDepLast;
1048    *OldDepLast = Dependency;
1049    OldDepLast = &Dep->NextDepends;
1050    return true;
1051 }
1052                                                                         /*}}}*/
1053 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1054 // ---------------------------------------------------------------------
1055 /* This creates a Group and the Package to link this dependency to if
1056    needed and handles also the caching of the old endpoint */
1057 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1058                                                StringView PackageName,
1059                                                StringView Arch,
1060                                                StringView Version,
1061                                                uint8_t const Op,
1062                                                uint8_t const Type)
1063 {
1064    pkgCache::GrpIterator Grp;
1065    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1066    Dynamic<StringView> DynPackageName(PackageName);
1067    Dynamic<StringView> DynArch(Arch);
1068    Dynamic<StringView> DynVersion(Version);
1069    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1070       return false;
1071
1072    map_stringitem_t idxVersion = 0;
1073    if (Version.empty() == false)
1074    {
1075       int const CmpOp = Op & 0x0F;
1076       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1077       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1078          idxVersion = Ver->VerStr;
1079
1080       if (idxVersion == 0)
1081       {
1082          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1083          if (unlikely(idxVersion == 0))
1084             return false;
1085       }
1086    }
1087
1088    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1089          Type == pkgCache::Dep::Conflicts ||
1090          Type == pkgCache::Dep::Replaces);
1091
1092    pkgCache::PkgIterator Pkg;
1093    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1094    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1095    {
1096       // Locate the target package
1097       Pkg = Grp.FindPkg(Arch);
1098       if (Pkg.end() == true) {
1099          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1100             return false;
1101       }
1102
1103       /* Caching the old end point speeds up generation substantially */
1104       if (OldDepVer != Ver) {
1105          OldDepLast = NULL;
1106          OldDepVer = Ver;
1107       }
1108
1109       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1110    }
1111    else
1112    {
1113       /* Caching the old end point speeds up generation substantially */
1114       if (OldDepVer != Ver) {
1115          OldDepLast = NULL;
1116          OldDepVer = Ver;
1117       }
1118
1119       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1120       {
1121          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1122                return false;
1123       }
1124    }
1125    return true;
1126 }
1127                                                                         /*}}}*/
1128 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1129 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1130                                                 StringView PkgName,
1131                                                 StringView PkgArch,
1132                                                 StringView Version,
1133                                                 uint8_t const Flags)
1134 {
1135    pkgCache const &Cache = Owner->Cache;
1136    Dynamic<StringView> DynPkgName(PkgName);
1137    Dynamic<StringView> DynArch(PkgArch);
1138    Dynamic<StringView> DynVersion(Version);
1139
1140    // We do not add self referencing provides
1141    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1142         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1143          (Version.empty() || Version == Ver.VerStr()))
1144       return true;
1145
1146    // Locate the target package
1147    pkgCache::PkgIterator Pkg;
1148    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1149    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1150       return false;
1151
1152    map_stringitem_t idxProvideVersion = 0;
1153    if (Version.empty() == false) {
1154       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1155       if (unlikely(idxProvideVersion == 0))
1156          return false;
1157    }
1158    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1159 }
1160 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1161                                     pkgCache::PkgIterator &Pkg,
1162                                     map_pointer_t const ProvideVersion,
1163                                     uint8_t const Flags)
1164 {
1165    // Get a structure
1166    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1167    if (unlikely(Provides == 0))
1168       return false;
1169    ++Cache.HeaderP->ProvidesCount;
1170
1171    // Fill it in
1172    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1173    Prv->Version = Ver.Index();
1174    Prv->ProvideVersion = ProvideVersion;
1175    Prv->Flags = Flags;
1176    Prv->NextPkgProv = Ver->ProvidesList;
1177    Ver->ProvidesList = Prv.Index();
1178
1179    // Link it to the package
1180    Prv->ParentPkg = Pkg.Index();
1181    Prv->NextProvides = Pkg->ProvidesList;
1182    Pkg->ProvidesList = Prv.Index();
1183    return true;
1184 }
1185                                                                         /*}}}*/
1186 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1187 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1188                                 StringView Version, uint8_t const Flags) {
1189    pkgCache &Cache = Owner->Cache;
1190    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1191    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1192    Dynamic<StringView> DynPackage(Package);
1193    Dynamic<StringView> DynVersion(Version);
1194
1195    if (Grp.end() == true)
1196       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1197    else
1198    {
1199       map_stringitem_t idxProvideVersion = 0;
1200       if (Version.empty() == false) {
1201          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1202          if (unlikely(idxProvideVersion == 0))
1203             return false;
1204       }
1205
1206       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1207       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1208       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1209       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1210       pkgCache::PkgIterator Pkg;
1211       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1212       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1213       {
1214          if (isImplicit && OwnerPkg == Pkg)
1215             continue;
1216          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1217             continue;
1218          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1219             return false;
1220       }
1221    }
1222    return true;
1223 }
1224                                                                         /*}}}*/
1225 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1226       pkgCache::VerIterator const &Ver)
1227 {
1228    return Hash == Ver->Hash;
1229 }
1230                                                                         /*}}}*/
1231 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1232 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1233                                    unsigned long Flags)
1234 {
1235    if (File.empty() && Site.empty())
1236    {
1237       CurrentRlsFile = NULL;
1238       return true;
1239    }
1240
1241    // Get some space for the structure
1242    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1243    if (unlikely(idxFile == 0))
1244       return false;
1245    CurrentRlsFile = Cache.RlsFileP + idxFile;
1246
1247    // Fill it in
1248    map_stringitem_t const idxFileName = WriteStringInMap(File);
1249    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1250    if (unlikely(idxFileName == 0 || idxSite == 0))
1251       return false;
1252    CurrentRlsFile->FileName = idxFileName;
1253    CurrentRlsFile->Site = idxSite;
1254    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1255    CurrentRlsFile->Flags = Flags;
1256    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1257    RlsFileName = File;
1258    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1259    Cache.HeaderP->ReleaseFileCount++;
1260
1261    return true;
1262 }
1263                                                                         /*}}}*/
1264 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1265 // ---------------------------------------------------------------------
1266 /* This is used to select which file is to be associated with all newly
1267    added versions. The caller is responsible for setting the IMS fields. */
1268 bool pkgCacheGenerator::SelectFile(std::string const &File,
1269                                    pkgIndexFile const &Index,
1270                                    std::string const &Architecture,
1271                                    std::string const &Component,
1272                                    unsigned long const Flags)
1273 {
1274    // Get some space for the structure
1275    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1276    if (unlikely(idxFile == 0))
1277       return false;
1278    CurrentFile = Cache.PkgFileP + idxFile;
1279
1280    // Fill it in
1281    map_stringitem_t const idxFileName = WriteStringInMap(File);
1282    if (unlikely(idxFileName == 0))
1283       return false;
1284    CurrentFile->FileName = idxFileName;
1285    CurrentFile->NextFile = Cache.HeaderP->FileList;
1286    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1287    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1288    if (unlikely(idxIndexType == 0))
1289       return false;
1290    CurrentFile->IndexType = idxIndexType;
1291    if (Architecture.empty())
1292       CurrentFile->Architecture = 0;
1293    else
1294    {
1295       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1296       if (unlikely(arch == 0))
1297          return false;
1298       CurrentFile->Architecture = arch;
1299    }
1300    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1301    if (unlikely(component == 0))
1302       return false;
1303    CurrentFile->Component = component;
1304    CurrentFile->Flags = Flags;
1305    if (CurrentRlsFile != NULL)
1306       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1307    else
1308       CurrentFile->Release = 0;
1309    PkgFileName = File;
1310    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1311    Cache.HeaderP->PackageFileCount++;
1312
1313    if (Progress != 0)
1314       Progress->SubProgress(Index.Size());
1315    return true;
1316 }
1317                                                                         /*}}}*/
1318 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1319 // ---------------------------------------------------------------------
1320 /* This is used to create handles to strings. Given the same text it
1321    always returns the same number */
1322 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1323                                                  unsigned int Size)
1324 {
1325    auto strings = &strMixed;
1326    switch(type) {
1327       case MIXED: strings = &strMixed; break;
1328       case PKGNAME: strings = &strPkgNames; break;
1329       case VERSIONNUMBER: strings = &strVersions; break;
1330       case SECTION: strings = &strSections; break;
1331       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1332    }
1333
1334    auto const item = strings->find({S, Size, nullptr, 0});
1335    if (item != strings->end())
1336       return item->item;
1337
1338    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1339    strings->insert({nullptr, Size, this, idxString});
1340    return idxString;
1341 }
1342                                                                         /*}}}*/
1343 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1344 // ---------------------------------------------------------------------
1345 /* This just verifies that each file in the list of index files exists,
1346    has matching attributes with the cache and the cache does not have
1347    any extra files. */
1348 class APT_HIDDEN ScopedErrorRevert {
1349 public:
1350    ScopedErrorRevert() { _error->PushToStack(); }
1351    ~ScopedErrorRevert() { _error->RevertToStack(); }
1352 };
1353 static bool CheckValidity(const string &CacheFile,
1354                           pkgSourceList &List,
1355                           FileIterator const Start,
1356                           FileIterator const End,
1357                           MMap **OutMap = 0,
1358                           pkgCache **OutCache = 0)
1359 {
1360    ScopedErrorRevert ser;
1361    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1362    // No file, certainly invalid
1363    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1364    {
1365       if (Debug == true)
1366          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1367       return false;
1368    }
1369
1370    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1371    {
1372       if (Debug == true)
1373          std::clog << "sources.list is newer than the cache" << std::endl;
1374       return false;
1375    }
1376
1377    // Map it
1378    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1379    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1380    if (unlikely(Map->validData()) == false)
1381       return false;
1382    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1383    pkgCache &Cache = *CacheP.get();
1384    if (_error->PendingError() || Map->Size() == 0)
1385    {
1386       if (Debug == true)
1387          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1388       return false;
1389    }
1390
1391    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1392    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1393    std::vector<pkgIndexFile *> Files;
1394    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1395    {
1396       if (Debug == true)
1397          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1398       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1399       if (RlsFile.end() == true)
1400       {
1401          if (Debug == true)
1402             std::clog << "FindInCache returned end-Pointer" << std::endl;
1403          return false;
1404       }
1405
1406       RlsVisited[RlsFile->ID] = true;
1407       if (Debug == true)
1408          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1409
1410       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1411       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1412             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1413    }
1414    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1415       if (RlsVisited[I] == false)
1416       {
1417          if (Debug == true)
1418             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1419          return false;
1420       }
1421
1422    std::copy(Start, End, std::back_inserter(Files));
1423
1424    /* Now we check every index file, see if it is in the cache,
1425       verify the IMS data and check that it is on the disk too.. */
1426    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1427    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1428    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1429    {
1430       if (Debug == true)
1431          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1432       if ((*PkgFile)->Exists() == false)
1433       {
1434          if (Debug == true)
1435             std::clog << "file doesn't exist" << std::endl;
1436          continue;
1437       }
1438
1439       // FindInCache is also expected to do an IMS check.
1440       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1441       if (File.end() == true)
1442       {
1443          if (Debug == true)
1444             std::clog << "FindInCache returned end-Pointer" << std::endl;
1445          return false;
1446       }
1447
1448       Visited[File->ID] = true;
1449       if (Debug == true)
1450          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1451    }
1452
1453    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1454       if (Visited[I] == false)
1455       {
1456          if (Debug == true)
1457             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1458          return false;
1459       }
1460
1461    if (_error->PendingError() == true)
1462    {
1463       if (Debug == true)
1464       {
1465          std::clog << "Validity failed because of pending errors:" << std::endl;
1466          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1467       }
1468       return false;
1469    }
1470
1471    if (OutMap != 0)
1472       *OutMap = Map.release();
1473    if (OutCache != 0)
1474       *OutCache = CacheP.release();
1475    return true;
1476 }
1477                                                                         /*}}}*/
1478 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1479 // ---------------------------------------------------------------------
1480 /* Size is kind of an abstract notion that is only used for the progress
1481    meter */
1482 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1483 {
1484    map_filesize_t TotalSize = 0;
1485    if (List !=  NULL)
1486    {
1487       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1488       {
1489          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1490          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1491             if ((*j)->HasPackages() == true)
1492                TotalSize += (*j)->Size();
1493       }
1494    }
1495
1496    for (; Start < End; ++Start)
1497    {
1498       if ((*Start)->HasPackages() == false)
1499          continue;
1500       TotalSize += (*Start)->Size();
1501    }
1502    return TotalSize;
1503 }
1504                                                                         /*}}}*/
1505 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1506 static bool BuildCache(pkgCacheGenerator &Gen,
1507                        OpProgress * const Progress,
1508                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1509                        pkgSourceList const * const List,
1510                        FileIterator const Start, FileIterator const End)
1511 {
1512    bool mergeFailure = false;
1513
1514    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1515       if (I->HasPackages() == false || mergeFailure)
1516          return;
1517
1518       if (I->Exists() == false)
1519          return;
1520
1521       if (I->FindInCache(Gen.GetCache()).end() == false)
1522       {
1523          _error->Warning("Duplicate sources.list entry %s",
1524                I->Describe().c_str());
1525          return;
1526       }
1527
1528       map_filesize_t const Size = I->Size();
1529       if (Progress != NULL)
1530          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1531       CurrentSize += Size;
1532
1533       if (I->Merge(Gen,Progress) == false)
1534          mergeFailure = true;
1535    };
1536
1537    if (List !=  NULL)
1538    {
1539       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1540       {
1541          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1542          {
1543             _error->Warning("Duplicate sources.list entry %s",
1544                   (*i)->Describe().c_str());
1545             continue;
1546          }
1547
1548          if ((*i)->Merge(Gen, Progress) == false)
1549             return false;
1550
1551          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1552          if (Indexes != NULL)
1553             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1554          if (mergeFailure)
1555             return false;
1556       }
1557    }
1558
1559    if (Start != End)
1560    {
1561       Gen.SelectReleaseFile("", "");
1562       std::for_each(Start, End, indexFileMerge);
1563       if (mergeFailure)
1564          return false;
1565    }
1566    return true;
1567 }
1568                                                                         /*}}}*/
1569 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1570 // ---------------------------------------------------------------------
1571 /* This makes sure that the status cache (the cache that has all
1572    index files from the sources list and all local ones) is ready
1573    to be mmaped. If OutMap is not zero then a MMap object representing
1574    the cache will be stored there. This is pretty much mandetory if you
1575    are using AllowMem. AllowMem lets the function be run as non-root
1576    where it builds the cache 'fast' into a memory buffer. */
1577 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1578 {
1579    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1580    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1581    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1582    Flags |= MMap::Moveable;
1583    if (_config->FindB("APT::Cache-Fallback", false) == true)
1584       Flags |= MMap::Fallback;
1585    if (CacheF != NULL)
1586       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1587    else
1588       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1589 }
1590 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1591       std::string const &FileName)
1592 {
1593    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1594    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1595       return false;
1596
1597    fchmod(SCacheF.Fd(),0644);
1598
1599    // Write out the main data
1600    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1601       return _error->Error(_("IO Error saving source cache"));
1602
1603    // Write out the proper header
1604    Gen->GetCache().HeaderP->Dirty = false;
1605    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1606    if (SCacheF.Seek(0) == false ||
1607          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1608       return _error->Error(_("IO Error saving source cache"));
1609    Gen->GetCache().HeaderP->Dirty = true;
1610    return true;
1611 }
1612 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1613       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1614 {
1615    Map.reset(CreateDynamicMMap(NULL, 0));
1616    if (unlikely(Map->validData()) == false)
1617       return false;
1618    FileFd CacheF(FileName, FileFd::ReadOnly);
1619    if (CacheF.IsOpen() == false || CacheF.Failed())
1620       return false;
1621    _error->PushToStack();
1622    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1623    bool const newError = _error->PendingError();
1624    _error->MergeWithStack();
1625    if (alloc == 0 && newError)
1626       return false;
1627    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1628       return false;
1629    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1630    return Gen->Start();
1631 }
1632 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1633                         MMap **OutMap, bool AllowMem)
1634    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1635 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1636                         MMap **OutMap,bool)
1637 {
1638    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1639 }
1640 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1641                         MMap **OutMap,pkgCache **OutCache, bool)
1642 {
1643    // FIXME: deprecate the ignored AllowMem parameter
1644    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1645
1646    std::vector<pkgIndexFile *> Files;
1647    if (_system->AddStatusFiles(Files) == false)
1648       return false;
1649
1650    // Decide if we can write to the files..
1651    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1652    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1653
1654    // ensure the cache directory exists
1655    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1656    {
1657       string dir = _config->FindDir("Dir::Cache");
1658       size_t const len = dir.size();
1659       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1660          dir = dir.substr(0, len - 5);
1661       if (CacheFile.empty() == false)
1662          CreateDirectory(dir, flNotFile(CacheFile));
1663       if (SrcCacheFile.empty() == false)
1664          CreateDirectory(dir, flNotFile(SrcCacheFile));
1665    }
1666
1667    if (Progress != NULL)
1668       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1669
1670    bool pkgcache_fine = false;
1671    bool srcpkgcache_fine = false;
1672    bool volatile_fine = List.GetVolatileFiles().empty();
1673
1674    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1675                      volatile_fine ? OutCache : NULL) == true)
1676    {
1677       if (Debug == true)
1678          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1679       pkgcache_fine = true;
1680       srcpkgcache_fine = true;
1681    }
1682    if (pkgcache_fine == false)
1683    {
1684       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1685       {
1686          if (Debug == true)
1687             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1688          srcpkgcache_fine = true;
1689       }
1690    }
1691
1692    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1693    {
1694       if (Progress != NULL)
1695          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1696       return true;
1697    }
1698
1699    bool Writeable = false;
1700    if (srcpkgcache_fine == false || pkgcache_fine == false)
1701    {
1702       if (CacheFile.empty() == false)
1703          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1704       else if (SrcCacheFile.empty() == false)
1705          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1706
1707       if (Debug == true)
1708          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1709    }
1710
1711    // At this point we know we need to construct something, so get storage ready
1712    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1713    if (unlikely(Map->validData()) == false)
1714       return false;
1715    if (Debug == true)
1716       std::clog << "Open memory Map (not filebased)" << std::endl;
1717
1718    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1719    map_filesize_t CurrentSize = 0;
1720    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1721    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1722    if (srcpkgcache_fine == true && pkgcache_fine == false)
1723    {
1724       if (Debug == true)
1725          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1726       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1727          return false;
1728       srcpkgcache_fine = true;
1729       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1730    }
1731    else if (srcpkgcache_fine == false)
1732    {
1733       if (Debug == true)
1734          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1735       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1736       if (Gen->Start() == false)
1737          return false;
1738
1739       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1740       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1741                Files.end(),Files.end()) == false)
1742          return false;
1743
1744       if (Writeable == true && SrcCacheFile.empty() == false)
1745          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1746             return false;
1747    }
1748
1749    if (pkgcache_fine == false)
1750    {
1751       if (Debug == true)
1752          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1753       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1754                Files.begin(), Files.end()) == false)
1755          return false;
1756
1757       if (Writeable == true && CacheFile.empty() == false)
1758          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1759             return false;
1760    }
1761
1762    if (Debug == true)
1763       std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
1764
1765    if (volatile_fine == false)
1766    {
1767       if (Gen == nullptr)
1768       {
1769          if (Debug == true)
1770             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1771          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1772             return false;
1773       }
1774
1775       Files = List.GetVolatileFiles();
1776       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1777                Files.begin(), Files.end()) == false)
1778          return false;
1779    }
1780
1781    if (OutMap != nullptr)
1782       *OutMap = Map.release();
1783
1784    if (Debug == true)
1785       std::clog << "Everything is ready for shipping" << std::endl;
1786    return true;
1787 }
1788                                                                         /*}}}*/
1789 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1790 class APT_HIDDEN ScopedErrorMerge {
1791 public:
1792    ScopedErrorMerge() { _error->PushToStack(); }
1793    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1794 };
1795 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1796    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1797 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1798 {
1799    std::vector<pkgIndexFile *> Files;
1800    if (_system->AddStatusFiles(Files) == false)
1801       return false;
1802
1803    ScopedErrorMerge sem;
1804    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1805    if (unlikely(Map->validData()) == false)
1806       return false;
1807    map_filesize_t CurrentSize = 0;
1808    map_filesize_t TotalSize = 0;
1809    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1810
1811    // Build the status cache
1812    if (Progress != NULL)
1813       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1814    pkgCacheGenerator Gen(Map.get(),Progress);
1815    if (Gen.Start() == false || _error->PendingError() == true)
1816       return false;
1817    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1818                   Files.begin(), Files.end()) == false)
1819       return false;
1820
1821    if (_error->PendingError() == true)
1822       return false;
1823    *OutMap = Map.release();
1824
1825    return true;
1826 }
1827                                                                         /*}}}*/
1828 // IsDuplicateDescription                                               /*{{{*/
1829 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1830                             MD5SumValue const &CurMd5, std::string const &CurLang)
1831 {
1832    // Descriptions in the same link-list have all the same md5
1833    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1834       return false;
1835    for (; Desc.end() == false; ++Desc)
1836       if (Desc.LanguageCode() == CurLang)
1837          return true;
1838    return false;
1839 }
1840                                                                         /*}}}*/
1841
1842 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1843 pkgCacheListParser::~pkgCacheListParser() {}