apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52
  53 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* We set the dirty flag and make sure that is written to the disk */
  56 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  57                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  58                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  59 {
  60    if (Map.Size() == 0)
  61    {
  62       // Setup the map interface..
  63       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  64       _error->PushToStack();
  65       Map.RawAllocate(sizeof(pkgCache::Header));
  66       bool const newError = _error->PendingError();
  67       _error->MergeWithStack();
  68       if (newError)
  69          return;
  70
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72
  73       // Starting header
  74       *Cache.HeaderP = pkgCache::Header();
  75
  76       // make room for the hashtables for packages and groups
  77       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  78          return;
  79
  80       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  81       if (unlikely(idxVerSysName == 0))
  82          return;
  83       Cache.HeaderP->VerSysName = idxVerSysName;
  84       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  85       if (unlikely(idxArchitecture == 0))
  86          return;
  87       Cache.HeaderP->Architecture = idxArchitecture;
  88
  89       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  90       if (archs.size() > 1)
  91       {
  92          std::vector<std::string>::const_iterator a = archs.begin();
  93          std::string list = *a;
  94          for (++a; a != archs.end(); ++a)
  95             list.append(",").append(*a);
  96          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  97          if (unlikely(idxArchitectures == 0))
  98             return;
  99          Cache.HeaderP->SetArchitectures(idxArchitectures);
 100       }
 101       else
 102          Cache.HeaderP->SetArchitectures(idxArchitecture);
 103
 104       // Calculate the hash for the empty map, so ReMap does not fail
 105       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 106       Cache.ReMap();
 107    }
 108    else
 109    {
 110       // Map directly from the existing file
 111       Cache.ReMap();
 112       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 113       if (Cache.VS != _system->VS)
 114       {
 115          _error->Error(_("Cache has an incompatible versioning system"));
 116          return;
 117       }
 118    }
 119
 120    Cache.HeaderP->Dirty = true;
 121    Map.Sync(0,sizeof(pkgCache::Header));
 122 }
 123                                                                         /*}}}*/
 124 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* We sync the data then unset the dirty flag in two steps so as to
 127    advoid a problem during a crash */
 128 pkgCacheGenerator::~pkgCacheGenerator()
 129 {
 130    if (_error->PendingError() == true || Map.validData() == false)
 131       return;
 132    if (Map.Sync() == false)
 133       return;
 134
 135    Cache.HeaderP->Dirty = false;
 136    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 137
 138    if (_config->FindB("Debug::pkgCacheGen", false))
 139       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 140    Map.Sync(0,sizeof(pkgCache::Header));
 141 }
 142                                                                         /*}}}*/
 143 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 144    if (oldMap == newMap)
 145       return;
 146
 147    if (_config->FindB("Debug::pkgCacheGen", false))
 148       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 149
 150    Cache.ReMap(false);
 151
 152    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 153    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 154
 155    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 156         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 157       (*i)->ReMap(oldMap, newMap);
 158    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 159         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 160       (*i)->ReMap(oldMap, newMap);
 161    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 162         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 163       (*i)->ReMap(oldMap, newMap);
 164    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 165         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 166       (*i)->ReMap(oldMap, newMap);
 167    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 168         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 169       (*i)->ReMap(oldMap, newMap);
 170    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 171         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 172       (*i)->ReMap(oldMap, newMap);
 173    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 174         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 175       (*i)->ReMap(oldMap, newMap);
 176    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 177         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 178       (*i)->ReMap(oldMap, newMap);
 179 }                                                                       /*}}}*/
 180 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 181 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 182                                         const unsigned long &Len) {
 183    void const * const oldMap = Map.Data();
 184    map_stringitem_t const index = Map.WriteString(String, Len);
 185    if (index != 0)
 186       ReMap(oldMap, Map.Data());
 187    return index;
 188 }
 189                                                                         /*}}}*/
 190 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 191 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 192    void const * const oldMap = Map.Data();
 193    map_stringitem_t const index = Map.WriteString(String);
 194    if (index != 0)
 195       ReMap(oldMap, Map.Data());
 196    return index;
 197 }
 198                                                                         /*}}}*/
 199 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 200    void const * const oldMap = Map.Data();
 201    map_pointer_t const index = Map.Allocate(size);
 202    if (index != 0)
 203       ReMap(oldMap, Map.Data());
 204    return index;
 205 }
 206                                                                         /*}}}*/
 207 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 208 // ---------------------------------------------------------------------
 209 /* This provides the generation of the entries in the cache. Each loop
 210    goes through a single package record from the underlying parse engine. */
 211 bool pkgCacheGenerator::MergeList(ListParser &List,
 212                                   pkgCache::VerIterator *OutVer)
 213 {
 214    List.Owner = this;
 215
 216    unsigned int Counter = 0;
 217    while (List.Step() == true)
 218    {
 219       string const PackageName = List.Package();
 220       if (PackageName.empty() == true)
 221          return false;
 222
 223       Counter++;
 224       if (Counter % 100 == 0 && Progress != 0)
 225          Progress->Progress(List.Offset());
 226
 227       string Arch = List.Architecture();
 228       string const Version = List.Version();
 229       if (Version.empty() == true && Arch.empty() == true)
 230       {
 231          // package descriptions
 232          if (MergeListGroup(List, PackageName) == false)
 233             return false;
 234          continue;
 235       }
 236
 237       // Get a pointer to the package structure
 238       pkgCache::PkgIterator Pkg;
 239       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 240       if (NewPackage(Pkg, PackageName, Arch) == false)
 241          // TRANSLATOR: The first placeholder is a package name,
 242          // the other two should be copied verbatim as they include debug info
 243          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 244                               PackageName.c_str(), "NewPackage", 1);
 245
 246
 247       if (Version.empty() == true)
 248       {
 249          if (MergeListPackage(List, Pkg) == false)
 250             return false;
 251       }
 252       else
 253       {
 254          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 255             return false;
 256       }
 257
 258       if (OutVer != 0)
 259          return true;
 260    }
 261
 262    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 263       return _error->Error(_("Wow, you exceeded the number of package "
 264                              "names this APT is capable of."));
 265    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 266       return _error->Error(_("Wow, you exceeded the number of versions "
 267                              "this APT is capable of."));
 268    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 269       return _error->Error(_("Wow, you exceeded the number of descriptions "
 270                              "this APT is capable of."));
 271    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 272       return _error->Error(_("Wow, you exceeded the number of dependencies "
 273                              "this APT is capable of."));
 274
 275    return true;
 276 }
 277 // CacheGenerator::MergeListGroup                                       /*{{{*/
 278 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 279 {
 280    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 281    // a group has no data on it's own, only packages have it but these
 282    // stanzas like this come from Translation- files to add descriptions,
 283    // but without a version we don't need a description for it…
 284    if (Grp.end() == true)
 285       return true;
 286    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 287
 288    pkgCache::PkgIterator Pkg;
 289    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 290    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 291       if (MergeListPackage(List, Pkg) == false)
 292          return false;
 293
 294    return true;
 295 }
 296                                                                         /*}}}*/
 297 // CacheGenerator::MergeListPackage                                     /*{{{*/
 298 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 299 {
 300    // we first process the package, then the descriptions
 301    // (for deb this package processing is in fact a no-op)
 302    pkgCache::VerIterator Ver(Cache);
 303    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 304    if (List.UsePackage(Pkg, Ver) == false)
 305       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 306                            Pkg.Name(), "UsePackage", 1);
 307
 308    // Find the right version to write the description
 309    MD5SumValue CurMd5 = List.Description_md5();
 310    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 311       return true;
 312    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 313    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 314    {
 315       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 316
 317       // a version can only have one md5 describing it
 318       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 319          continue;
 320
 321       map_stringitem_t md5idx = VerDesc->md5sum;
 322       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 323       {
 324          // don't add a new description if we have one for the given
 325          // md5 && language
 326          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 327             continue;
 328
 329          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 330       }
 331
 332       // we can stop here as all "same" versions will share the description
 333       break;
 334    }
 335
 336    return true;
 337 }
 338                                                                         /*}}}*/
 339 // CacheGenerator::MergeListVersion                                     /*{{{*/
 340 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 341                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 342 {
 343    pkgCache::VerIterator Ver = Pkg.VersionList();
 344    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 345    map_pointer_t *LastVer = &Pkg->VersionList;
 346    void const * oldMap = Map.Data();
 347
 348    unsigned short const Hash = List.VersionHash();
 349    if (Ver.end() == false)
 350    {
 351       /* We know the list is sorted so we use that fact in the search.
 352          Insertion of new versions is done with correct sorting */
 353       int Res = 1;
 354       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 355       {
 356          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 357          // Version is higher as current version - insert here
 358          if (Res > 0)
 359             break;
 360          // Versionstrings are equal - is hash also equal?
 361          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 362             break;
 363          // proceed with the next till we have either the right
 364          // or we found another version (which will be lower)
 365       }
 366
 367       /* We already have a version for this item, record that we saw it */
 368       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 369       {
 370          if (List.UsePackage(Pkg,Ver) == false)
 371             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 372                                  Pkg.Name(), "UsePackage", 2);
 373
 374          if (NewFileVer(Ver,List) == false)
 375             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 376                                  Pkg.Name(), "NewFileVer", 1);
 377
 378          // Read only a single record and return
 379          if (OutVer != 0)
 380          {
 381             *OutVer = Ver;
 382             return true;
 383          }
 384
 385          return true;
 386       }
 387    }
 388
 389    // Add a new version
 390    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 391    if (unlikely(verindex == 0))
 392       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 393                            Pkg.Name(), "NewVersion", 1);
 394
 395    if (oldMap != Map.Data())
 396          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 397    *LastVer = verindex;
 398
 399    if (unlikely(List.NewVersion(Ver) == false))
 400       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 401                            Pkg.Name(), "NewVersion", 2);
 402
 403    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 404       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 405                            Pkg.Name(), "UsePackage", 3);
 406
 407    if (unlikely(NewFileVer(Ver,List) == false))
 408       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 409                            Pkg.Name(), "NewFileVer", 2);
 410
 411    pkgCache::GrpIterator Grp = Pkg.Group();
 412    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 413
 414    /* If it is the first version of this package we need to add implicit
 415       Multi-Arch dependencies to all other package versions in the group now -
 416       otherwise we just add them for this new version */
 417    if (Pkg.VersionList()->NextVer == 0)
 418    {
 419       pkgCache::PkgIterator P = Grp.PackageList();
 420       Dynamic<pkgCache::PkgIterator> DynP(P);
 421       for (; P.end() != true; P = Grp.NextPkg(P))
 422       {
 423          if (P->ID == Pkg->ID)
 424             continue;
 425          pkgCache::VerIterator V = P.VersionList();
 426          Dynamic<pkgCache::VerIterator> DynV(V);
 427          for (; V.end() != true; ++V)
 428             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 429                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 430                                     Pkg.Name(), "AddImplicitDepends", 1);
 431       }
 432    }
 433    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 434       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 435                            Pkg.Name(), "AddImplicitDepends", 2);
 436
 437    // Read only a single record and return
 438    if (OutVer != 0)
 439    {
 440       *OutVer = Ver;
 441       return true;
 442    }
 443
 444    /* Record the Description(s) based on their master md5sum */
 445    MD5SumValue CurMd5 = List.Description_md5();
 446    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 447       return true;
 448
 449    /* Before we add a new description we first search in the group for
 450       a version with a description of the same MD5 - if so we reuse this
 451       description group instead of creating our own for this version */
 452    for (pkgCache::PkgIterator P = Grp.PackageList();
 453         P.end() == false; P = Grp.NextPkg(P))
 454    {
 455       for (pkgCache::VerIterator V = P.VersionList();
 456            V.end() == false; ++V)
 457       {
 458          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 459             continue;
 460          Ver->DescriptionList = V->DescriptionList;
 461       }
 462    }
 463
 464    // We haven't found reusable descriptions, so add the first description(s)
 465    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 466    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 467    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 468       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 469          return false;
 470    return true;
 471 }
 472                                                                         /*}}}*/
 473 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 474 {
 475    pkgCache::DescIterator Desc;
 476    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 477
 478    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 479    if (unlikely(descindex == 0))
 480       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 481             Ver.ParentPkg().Name(), "NewDescription", 1);
 482
 483    md5idx = Desc->md5sum;
 484    Desc->ParentPkg = Ver.ParentPkg().Index();
 485
 486    // we add at the end, so that the start is constant as we need
 487    // that to be able to efficiently share these lists
 488    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 489    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 490    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 491    *LastNextDesc = descindex;
 492
 493    if (NewFileDesc(Desc,List) == false)
 494       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 495             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 496
 497    return true;
 498 }
 499                                                                         /*}}}*/
 500                                                                         /*}}}*/
 501 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 502 // ---------------------------------------------------------------------
 503 /* This creates a new group structure and adds it to the hash table */
 504 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 505 {
 506    Grp = Cache.FindGrp(Name);
 507    if (Grp.end() == false)
 508       return true;
 509
 510    // Get a structure
 511    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 512    if (unlikely(Group == 0))
 513       return false;
 514
 515    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 516    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 517    if (unlikely(idxName == 0))
 518       return false;
 519    Grp->Name = idxName;
 520
 521    // Insert it into the hash table
 522    unsigned long const Hash = Cache.Hash(Name);
 523    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 524    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 525       insertAt = &(Cache.GrpP + *insertAt)->Next;
 526    Grp->Next = *insertAt;
 527    *insertAt = Group;
 528
 529    Grp->ID = Cache.HeaderP->GroupCount++;
 530    return true;
 531 }
 532                                                                         /*}}}*/
 533 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 534 // ---------------------------------------------------------------------
 535 /* This creates a new package structure and adds it to the hash table */
 536 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 537                                         const string &Arch) {
 538    pkgCache::GrpIterator Grp;
 539    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 540    if (unlikely(NewGroup(Grp, Name) == false))
 541       return false;
 542
 543    Pkg = Grp.FindPkg(Arch);
 544       if (Pkg.end() == false)
 545          return true;
 546
 547    // Get a structure
 548    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 549    if (unlikely(Package == 0))
 550       return false;
 551    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 552
 553    // Set the name, arch and the ID
 554    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 555    Pkg->Group = Grp.Index();
 556    // all is mapped to the native architecture
 557    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 558    if (unlikely(idxArch == 0))
 559       return false;
 560    Pkg->Arch = idxArch;
 561    Pkg->ID = Cache.HeaderP->PackageCount++;
 562
 563    // Insert the package into our package list
 564    if (Grp->FirstPackage == 0) // the group is new
 565    {
 566       Grp->FirstPackage = Package;
 567       // Insert it into the hash table
 568       map_id_t const Hash = Cache.Hash(Name);
 569       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 570       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 571          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 572       Pkg->NextPackage = *insertAt;
 573       *insertAt = Package;
 574    }
 575    else // Group the Packages together
 576    {
 577       // but first get implicit provides done
 578       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 579       {
 580          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 581          if (M.end() == false) {
 582             pkgCache::PrvIterator Prv;
 583             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 584             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 585             {
 586                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 587                   continue;
 588                pkgCache::VerIterator Ver = Prv.OwnerVer();
 589                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 590                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 591                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 592                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 593                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 594                      return false;
 595             }
 596          }
 597
 598
 599          pkgCache::PkgIterator P;
 600          pkgCache::VerIterator Ver;
 601          Dynamic<pkgCache::PkgIterator> DynP(P);
 602          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 603
 604          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 605             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 606                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 607                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 608                      return false;
 609       }
 610       // and negative dependencies, don't forget negative dependencies
 611       {
 612          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 613          if (M.end() == false) {
 614             pkgCache::DepIterator Dep;
 615             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 616             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 617             {
 618                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 619                   continue;
 620                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 621                      Dep->Type != pkgCache::Dep::Replaces)
 622                   continue;
 623                pkgCache::VerIterator Ver = Dep.ParentVer();
 624                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 625                map_pointer_t * unused = NULL;
 626                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 627                   return false;
 628             }
 629          }
 630       }
 631
 632       // this package is the new last package
 633       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 634       Pkg->NextPackage = LastPkg->NextPackage;
 635       LastPkg->NextPackage = Package;
 636    }
 637    Grp->LastPackage = Package;
 638
 639    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 640    if (Arch == "any")
 641    {
 642       size_t const found = Name.find(':');
 643       std::string const NameA = Name.substr(0, found);
 644       std::string const ArchA = Name.substr(found + 1);
 645       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 646       if (PkgA.end() == false)
 647       {
 648          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 649          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 650          for (; Prv.end() == false; ++Prv)
 651          {
 652             if (Prv.IsMultiArchImplicit())
 653                continue;
 654             pkgCache::VerIterator V = Prv.OwnerVer();
 655             if (ArchA != V.ParentPkg().Arch())
 656                continue;
 657             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 658                return false;
 659          }
 660          pkgCache::VerIterator V = PkgA.VersionList();
 661          Dynamic<pkgCache::VerIterator> DynV(V);
 662          for (; V.end() == false; ++V)
 663          {
 664             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 665                return false;
 666          }
 667       }
 668    }
 669    return true;
 670 }
 671                                                                         /*}}}*/
 672 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 673 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 674                                            pkgCache::PkgIterator &P,
 675                                            pkgCache::VerIterator &V)
 676 {
 677    // copy P.Arch() into a string here as a cache remap
 678    // in NewDepends() later may alter the pointer location
 679    string Arch = P.Arch() == NULL ? "" : P.Arch();
 680    map_pointer_t *OldDepLast = NULL;
 681    /* MultiArch handling introduces a lot of implicit Dependencies:
 682       - MultiArch: same → Co-Installable if they have the same version
 683       - All others conflict with all other group members */
 684    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 685    pkgCache::PkgIterator D = G.PackageList();
 686    Dynamic<pkgCache::PkgIterator> DynD(D);
 687    map_stringitem_t const VerStrIdx = V->VerStr;
 688    for (; D.end() != true; D = G.NextPkg(D))
 689    {
 690       if (Arch == D.Arch() || D->VersionList == 0)
 691          continue;
 692       /* We allow only one installed arch at the time
 693          per group, therefore each group member conflicts
 694          with all other group members */
 695       if (coInstall == true)
 696       {
 697          // Replaces: ${self}:other ( << ${binary:Version})
 698          NewDepends(D, V, VerStrIdx,
 699                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 700                     OldDepLast);
 701          // Breaks: ${self}:other (!= ${binary:Version})
 702          NewDepends(D, V, VerStrIdx,
 703                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 704                     OldDepLast);
 705       } else {
 706          // Conflicts: ${self}:other
 707          NewDepends(D, V, 0,
 708                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 709                     OldDepLast);
 710       }
 711    }
 712    return true;
 713 }
 714 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 715                                            pkgCache::PkgIterator &D)
 716 {
 717    /* MultiArch handling introduces a lot of implicit Dependencies:
 718       - MultiArch: same → Co-Installable if they have the same version
 719       - All others conflict with all other group members */
 720    map_pointer_t *OldDepLast = NULL;
 721    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 722    if (coInstall == true)
 723    {
 724       map_stringitem_t const VerStrIdx = V->VerStr;
 725       // Replaces: ${self}:other ( << ${binary:Version})
 726       NewDepends(D, V, VerStrIdx,
 727                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 728                  OldDepLast);
 729       // Breaks: ${self}:other (!= ${binary:Version})
 730       NewDepends(D, V, VerStrIdx,
 731                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 732                  OldDepLast);
 733    } else {
 734       // Conflicts: ${self}:other
 735       NewDepends(D, V, 0,
 736                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 737                  OldDepLast);
 738    }
 739    return true;
 740 }
 741
 742                                                                         /*}}}*/
 743 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 744 // ---------------------------------------------------------------------
 745 /* */
 746 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 747                                    ListParser &List)
 748 {
 749    if (CurrentFile == 0)
 750       return true;
 751
 752    // Get a structure
 753    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 754    if (VerFile == 0)
 755       return false;
 756
 757    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 758    VF->File = CurrentFile - Cache.PkgFileP;
 759
 760    // Link it to the end of the list
 761    map_pointer_t *Last = &Ver->FileList;
 762    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 763       Last = &V->NextFile;
 764    VF->NextFile = *Last;
 765    *Last = VF.Index();
 766
 767    VF->Offset = List.Offset();
 768    VF->Size = List.Size();
 769    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 770       Cache.HeaderP->MaxVerFileSize = VF->Size;
 771    Cache.HeaderP->VerFileCount++;
 772
 773    return true;
 774 }
 775                                                                         /*}}}*/
 776 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 777 // ---------------------------------------------------------------------
 778 /* This puts a version structure in the linked list */
 779 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 780                                             const string &VerStr,
 781                                             map_pointer_t const ParentPkg,
 782                                             unsigned short const Hash,
 783                                             map_pointer_t const Next)
 784 {
 785    // Get a structure
 786    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 787    if (Version == 0)
 788       return 0;
 789
 790    // Fill it in
 791    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 792    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 793    Ver->NextVer = Next;
 794    Ver->ParentPkg = ParentPkg;
 795    Ver->Hash = Hash;
 796    Ver->ID = Cache.HeaderP->VersionCount++;
 797
 798    // try to find the version string in the group for reuse
 799    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 800    pkgCache::GrpIterator Grp = Pkg.Group();
 801    if (Pkg.end() == false && Grp.end() == false)
 802    {
 803       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 804       {
 805          if (Pkg == P)
 806             continue;
 807          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 808          {
 809             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 810             if (cmp == 0)
 811             {
 812                Ver->VerStr = V->VerStr;
 813                return Version;
 814             }
 815             else if (cmp < 0)
 816                break;
 817          }
 818       }
 819    }
 820    // haven't found the version string, so create
 821    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 822    if (unlikely(idxVerStr == 0))
 823       return 0;
 824    Ver->VerStr = idxVerStr;
 825    return Version;
 826 }
 827                                                                         /*}}}*/
 828 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 829 // ---------------------------------------------------------------------
 830 /* */
 831 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 832                                    ListParser &List)
 833 {
 834    if (CurrentFile == 0)
 835       return true;
 836
 837    // Get a structure
 838    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 839    if (DescFile == 0)
 840       return false;
 841
 842    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 843    DF->File = CurrentFile - Cache.PkgFileP;
 844
 845    // Link it to the end of the list
 846    map_pointer_t *Last = &Desc->FileList;
 847    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 848       Last = &D->NextFile;
 849
 850    DF->NextFile = *Last;
 851    *Last = DF.Index();
 852
 853    DF->Offset = List.Offset();
 854    DF->Size = List.Size();
 855    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 856       Cache.HeaderP->MaxDescFileSize = DF->Size;
 857    Cache.HeaderP->DescFileCount++;
 858
 859    return true;
 860 }
 861                                                                         /*}}}*/
 862 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 863 // ---------------------------------------------------------------------
 864 /* This puts a description structure in the linked list */
 865 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 866                                             const string &Lang,
 867                                             const MD5SumValue &md5sum,
 868                                             map_stringitem_t const idxmd5str)
 869 {
 870    // Get a structure
 871    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 872    if (Description == 0)
 873       return 0;
 874
 875    // Fill it in
 876    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 877    Desc->ID = Cache.HeaderP->DescriptionCount++;
 878    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 879    if (unlikely(idxlanguage_code == 0))
 880       return 0;
 881    Desc->language_code = idxlanguage_code;
 882
 883    if (idxmd5str != 0)
 884       Desc->md5sum = idxmd5str;
 885    else
 886    {
 887       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 888       if (unlikely(idxmd5sum == 0))
 889          return 0;
 890       Desc->md5sum = idxmd5sum;
 891    }
 892
 893    return Description;
 894 }
 895                                                                         /*}}}*/
 896 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 897 // ---------------------------------------------------------------------
 898 /* This creates a dependency element in the tree. It is linked to the
 899    version and to the package that it is pointing to. */
 900 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 901                                    pkgCache::VerIterator &Ver,
 902                                    map_pointer_t const Version,
 903                                    uint8_t const Op,
 904                                    uint8_t const Type,
 905                                    map_pointer_t* &OldDepLast)
 906 {
 907    void const * const oldMap = Map.Data();
 908    // Get a structure
 909    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 910    if (unlikely(Dependency == 0))
 911       return false;
 912
 913    bool isDuplicate = false;
 914    map_pointer_t DependencyData = 0;
 915    map_pointer_t PreviousData = 0;
 916    if (Pkg->RevDepends != 0)
 917    {
 918       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 919       DependencyData = L->DependencyData;
 920       do {
 921          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 922          if (Version > D->Version)
 923             break;
 924          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 925          {
 926             isDuplicate = true;
 927             break;
 928          }
 929          PreviousData = DependencyData;
 930          DependencyData = D->NextData;
 931       } while (DependencyData != 0);
 932    }
 933
 934    if (isDuplicate == false)
 935    {
 936       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 937       if (unlikely(DependencyData == 0))
 938         return false;
 939    }
 940
 941    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 942    Link->ParentVer = Ver.Index();
 943    Link->DependencyData = DependencyData;
 944    Link->ID = Cache.HeaderP->DependsCount++;
 945
 946    pkgCache::DepIterator Dep(Cache, Link);
 947    if (isDuplicate == false)
 948    {
 949       Dep->Type = Type;
 950       Dep->CompareOp = Op;
 951       Dep->Version = Version;
 952       Dep->Package = Pkg.Index();
 953       ++Cache.HeaderP->DependsDataCount;
 954       if (PreviousData != 0)
 955       {
 956          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 957          Dep->NextData = D->NextData;
 958          D->NextData = DependencyData;
 959       }
 960       else if (Pkg->RevDepends != 0)
 961       {
 962          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 963          Dep->NextData = D->DependencyData;
 964       }
 965    }
 966
 967    if (isDuplicate == true || PreviousData != 0)
 968    {
 969       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 970       Link->NextRevDepends = L->NextRevDepends;
 971       L->NextRevDepends = Dependency;
 972    }
 973    else
 974    {
 975       Link->NextRevDepends = Pkg->RevDepends;
 976       Pkg->RevDepends = Dependency;
 977    }
 978
 979
 980    // Do we know where to link the Dependency to?
 981    if (OldDepLast == NULL)
 982    {
 983       OldDepLast = &Ver->DependsList;
 984       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 985          OldDepLast = &D->NextDepends;
 986    } else if (oldMap != Map.Data())
 987       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 988
 989    Dep->NextDepends = *OldDepLast;
 990    *OldDepLast = Dependency;
 991    OldDepLast = &Dep->NextDepends;
 992    return true;
 993 }
 994                                                                         /*}}}*/
 995 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 996 // ---------------------------------------------------------------------
 997 /* This creates a Group and the Package to link this dependency to if
 998    needed and handles also the caching of the old endpoint */
 999 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1000                                                const string &PackageName,
1001                                                const string &Arch,
1002                                                const string &Version,
1003                                                uint8_t const Op,
1004                                                uint8_t const Type)
1005 {
1006    pkgCache::GrpIterator Grp;
1007    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1008    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1009       return false;
1010
1011    map_stringitem_t idxVersion = 0;
1012    if (Version.empty() == false)
1013    {
1014       int const CmpOp = Op & 0x0F;
1015       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1016       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1017          idxVersion = Ver->VerStr;
1018
1019       if (idxVersion == 0)
1020       {
1021          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1022          if (unlikely(idxVersion == 0))
1023             return false;
1024       }
1025    }
1026
1027    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1028          Type == pkgCache::Dep::Conflicts ||
1029          Type == pkgCache::Dep::Replaces);
1030
1031    pkgCache::PkgIterator Pkg;
1032    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1033    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1034    {
1035       // Locate the target package
1036       Pkg = Grp.FindPkg(Arch);
1037       if (Pkg.end() == true) {
1038          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1039             return false;
1040       }
1041
1042       /* Caching the old end point speeds up generation substantially */
1043       if (OldDepVer != Ver) {
1044          OldDepLast = NULL;
1045          OldDepVer = Ver;
1046       }
1047
1048       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1049    }
1050    else
1051    {
1052       /* Caching the old end point speeds up generation substantially */
1053       if (OldDepVer != Ver) {
1054          OldDepLast = NULL;
1055          OldDepVer = Ver;
1056       }
1057
1058       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1059       {
1060          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1061                return false;
1062       }
1063    }
1064    return true;
1065 }
1066                                                                         /*}}}*/
1067 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1068 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1069                                                 const string &PkgName,
1070                                                 const string &PkgArch,
1071                                                 const string &Version,
1072                                                 uint8_t const Flags)
1073 {
1074    pkgCache const &Cache = Owner->Cache;
1075
1076    // We do not add self referencing provides
1077    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1078         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1079          (Version.empty() || Version == Ver.VerStr()))
1080       return true;
1081
1082    // Locate the target package
1083    pkgCache::PkgIterator Pkg;
1084    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1085    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1086       return false;
1087
1088    map_stringitem_t idxProvideVersion = 0;
1089    if (Version.empty() == false) {
1090       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1091       if (unlikely(idxProvideVersion == 0))
1092          return false;
1093    }
1094    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1095 }
1096 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1097                                     pkgCache::PkgIterator &Pkg,
1098                                     map_pointer_t const ProvideVersion,
1099                                     uint8_t const Flags)
1100 {
1101    // Get a structure
1102    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1103    if (unlikely(Provides == 0))
1104       return false;
1105    ++Cache.HeaderP->ProvidesCount;
1106
1107    // Fill it in
1108    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1109    Prv->Version = Ver.Index();
1110    Prv->ProvideVersion = ProvideVersion;
1111    Prv->Flags = Flags;
1112    Prv->NextPkgProv = Ver->ProvidesList;
1113    Ver->ProvidesList = Prv.Index();
1114
1115    // Link it to the package
1116    Prv->ParentPkg = Pkg.Index();
1117    Prv->NextProvides = Pkg->ProvidesList;
1118    Pkg->ProvidesList = Prv.Index();
1119    return true;
1120 }
1121                                                                         /*}}}*/
1122 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1123 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1124                                 string const &Version, uint8_t const Flags) {
1125    pkgCache &Cache = Owner->Cache;
1126    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1127    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1128
1129    if (Grp.end() == true)
1130       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1131    else
1132    {
1133       map_stringitem_t idxProvideVersion = 0;
1134       if (Version.empty() == false) {
1135          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1136          if (unlikely(idxProvideVersion == 0))
1137             return false;
1138       }
1139
1140       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1141       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1142       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1143       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1144       pkgCache::PkgIterator Pkg;
1145       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1146       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1147       {
1148          if (isImplicit && OwnerPkg == Pkg)
1149             continue;
1150          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1151             continue;
1152          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1153             return false;
1154       }
1155    }
1156    return true;
1157 }
1158                                                                         /*}}}*/
1159 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1160       pkgCache::VerIterator const &Ver)
1161 {
1162    return Hash == Ver->Hash;
1163 }
1164                                                                         /*}}}*/
1165 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1166 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1167                                    unsigned long Flags)
1168 {
1169    if (File.empty() && Site.empty())
1170    {
1171       CurrentRlsFile = NULL;
1172       return true;
1173    }
1174
1175    // Get some space for the structure
1176    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1177    if (unlikely(idxFile == 0))
1178       return false;
1179    CurrentRlsFile = Cache.RlsFileP + idxFile;
1180
1181    // Fill it in
1182    map_stringitem_t const idxFileName = WriteStringInMap(File);
1183    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1184    if (unlikely(idxFileName == 0 || idxSite == 0))
1185       return false;
1186    CurrentRlsFile->FileName = idxFileName;
1187    CurrentRlsFile->Site = idxSite;
1188    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1189    CurrentRlsFile->Flags = Flags;
1190    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1191    RlsFileName = File;
1192    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1193    Cache.HeaderP->ReleaseFileCount++;
1194
1195    return true;
1196 }
1197                                                                         /*}}}*/
1198 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1199 // ---------------------------------------------------------------------
1200 /* This is used to select which file is to be associated with all newly
1201    added versions. The caller is responsible for setting the IMS fields. */
1202 bool pkgCacheGenerator::SelectFile(std::string const &File,
1203                                    pkgIndexFile const &Index,
1204                                    std::string const &Architecture,
1205                                    std::string const &Component,
1206                                    unsigned long const Flags)
1207 {
1208    // Get some space for the structure
1209    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1210    if (unlikely(idxFile == 0))
1211       return false;
1212    CurrentFile = Cache.PkgFileP + idxFile;
1213
1214    // Fill it in
1215    map_stringitem_t const idxFileName = WriteStringInMap(File);
1216    if (unlikely(idxFileName == 0))
1217       return false;
1218    CurrentFile->FileName = idxFileName;
1219    CurrentFile->NextFile = Cache.HeaderP->FileList;
1220    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1221    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1222    if (unlikely(idxIndexType == 0))
1223       return false;
1224    CurrentFile->IndexType = idxIndexType;
1225    if (Architecture.empty())
1226       CurrentFile->Architecture = 0;
1227    else
1228    {
1229       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1230       if (unlikely(arch == 0))
1231          return false;
1232       CurrentFile->Architecture = arch;
1233    }
1234    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1235    if (unlikely(component == 0))
1236       return false;
1237    CurrentFile->Component = component;
1238    CurrentFile->Flags = Flags;
1239    if (CurrentRlsFile != NULL)
1240       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1241    else
1242       CurrentFile->Release = 0;
1243    PkgFileName = File;
1244    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1245    Cache.HeaderP->PackageFileCount++;
1246
1247    if (Progress != 0)
1248       Progress->SubProgress(Index.Size());
1249    return true;
1250 }
1251                                                                         /*}}}*/
1252 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1253 // ---------------------------------------------------------------------
1254 /* This is used to create handles to strings. Given the same text it
1255    always returns the same number */
1256 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1257                                                  unsigned int Size)
1258 {
1259    std::string const key(S, Size);
1260
1261    std::unordered_map<std::string,map_stringitem_t> * strings;
1262    switch(type) {
1263       case MIXED: strings = &strMixed; break;
1264       case PKGNAME: strings = &strPkgNames; break;
1265       case VERSIONNUMBER: strings = &strVersions; break;
1266       case SECTION: strings = &strSections; break;
1267       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1268    }
1269
1270    std::unordered_map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1271    if (item != strings->end())
1272       return item->second;
1273
1274    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1275    strings->insert(std::make_pair(key, idxString));
1276    return idxString;
1277 }
1278                                                                         /*}}}*/
1279 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1280 // ---------------------------------------------------------------------
1281 /* This just verifies that each file in the list of index files exists,
1282    has matching attributes with the cache and the cache does not have
1283    any extra files. */
1284 class APT_HIDDEN ScopedErrorRevert {
1285 public:
1286    ScopedErrorRevert() { _error->PushToStack(); }
1287    ~ScopedErrorRevert() { _error->RevertToStack(); }
1288 };
1289 static bool CheckValidity(const string &CacheFile,
1290                           pkgSourceList &List,
1291                           FileIterator const Start,
1292                           FileIterator const End,
1293                           MMap **OutMap = 0,
1294                           pkgCache **OutCache = 0)
1295 {
1296    ScopedErrorRevert ser;
1297    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1298    // No file, certainly invalid
1299    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1300    {
1301       if (Debug == true)
1302          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1303       return false;
1304    }
1305
1306    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1307    {
1308       if (Debug == true)
1309          std::clog << "sources.list is newer than the cache" << std::endl;
1310       return false;
1311    }
1312
1313    // Map it
1314    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1315    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1316    if (unlikely(Map->validData()) == false)
1317       return false;
1318    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1319    pkgCache &Cache = *CacheP.get();
1320    if (_error->PendingError() || Map->Size() == 0)
1321    {
1322       if (Debug == true)
1323          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1324       return false;
1325    }
1326
1327    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1328    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1329    std::vector<pkgIndexFile *> Files;
1330    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1331    {
1332       if (Debug == true)
1333          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1334       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1335       if (RlsFile.end() == true)
1336       {
1337          if (Debug == true)
1338             std::clog << "FindInCache returned end-Pointer" << std::endl;
1339          return false;
1340       }
1341
1342       RlsVisited[RlsFile->ID] = true;
1343       if (Debug == true)
1344          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1345
1346       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1347       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1348             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1349    }
1350    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1351       if (RlsVisited[I] == false)
1352       {
1353          if (Debug == true)
1354             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1355          return false;
1356       }
1357
1358    std::copy(Start, End, std::back_inserter(Files));
1359
1360    /* Now we check every index file, see if it is in the cache,
1361       verify the IMS data and check that it is on the disk too.. */
1362    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1363    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1364    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1365    {
1366       if (Debug == true)
1367          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1368       if ((*PkgFile)->Exists() == false)
1369       {
1370          if (Debug == true)
1371             std::clog << "file doesn't exist" << std::endl;
1372          continue;
1373       }
1374
1375       // FindInCache is also expected to do an IMS check.
1376       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1377       if (File.end() == true)
1378       {
1379          if (Debug == true)
1380             std::clog << "FindInCache returned end-Pointer" << std::endl;
1381          return false;
1382       }
1383
1384       Visited[File->ID] = true;
1385       if (Debug == true)
1386          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1387    }
1388
1389    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1390       if (Visited[I] == false)
1391       {
1392          if (Debug == true)
1393             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1394          return false;
1395       }
1396
1397    if (_error->PendingError() == true)
1398    {
1399       if (Debug == true)
1400       {
1401          std::clog << "Validity failed because of pending errors:" << std::endl;
1402          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1403       }
1404       return false;
1405    }
1406
1407    if (OutMap != 0)
1408       *OutMap = Map.release();
1409    if (OutCache != 0)
1410       *OutCache = CacheP.release();
1411    return true;
1412 }
1413                                                                         /*}}}*/
1414 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1415 // ---------------------------------------------------------------------
1416 /* Size is kind of an abstract notion that is only used for the progress
1417    meter */
1418 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1419 {
1420    map_filesize_t TotalSize = 0;
1421    if (List !=  NULL)
1422    {
1423       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1424       {
1425          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1426          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1427             if ((*j)->HasPackages() == true)
1428                TotalSize += (*j)->Size();
1429       }
1430    }
1431
1432    for (; Start < End; ++Start)
1433    {
1434       if ((*Start)->HasPackages() == false)
1435          continue;
1436       TotalSize += (*Start)->Size();
1437    }
1438    return TotalSize;
1439 }
1440                                                                         /*}}}*/
1441 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1442 static bool BuildCache(pkgCacheGenerator &Gen,
1443                        OpProgress * const Progress,
1444                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1445                        pkgSourceList const * const List,
1446                        FileIterator const Start, FileIterator const End)
1447 {
1448    bool mergeFailure = false;
1449
1450    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1451       if (I->HasPackages() == false || mergeFailure)
1452          return;
1453
1454       if (I->Exists() == false)
1455          return;
1456
1457       if (I->FindInCache(Gen.GetCache()).end() == false)
1458       {
1459          _error->Warning("Duplicate sources.list entry %s",
1460                I->Describe().c_str());
1461          return;
1462       }
1463
1464       map_filesize_t const Size = I->Size();
1465       if (Progress != NULL)
1466          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1467       CurrentSize += Size;
1468
1469       if (I->Merge(Gen,Progress) == false)
1470          mergeFailure = true;
1471    };
1472
1473    if (List !=  NULL)
1474    {
1475       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1476       {
1477          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1478          {
1479             _error->Warning("Duplicate sources.list entry %s",
1480                   (*i)->Describe().c_str());
1481             continue;
1482          }
1483
1484          if ((*i)->Merge(Gen, Progress) == false)
1485             return false;
1486
1487          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1488          if (Indexes != NULL)
1489             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1490          if (mergeFailure)
1491             return false;
1492       }
1493    }
1494
1495    if (Start != End)
1496    {
1497       Gen.SelectReleaseFile("", "");
1498       std::for_each(Start, End, indexFileMerge);
1499       if (mergeFailure)
1500          return false;
1501    }
1502    return true;
1503 }
1504                                                                         /*}}}*/
1505 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1506 // ---------------------------------------------------------------------
1507 /* This makes sure that the status cache (the cache that has all
1508    index files from the sources list and all local ones) is ready
1509    to be mmaped. If OutMap is not zero then a MMap object representing
1510    the cache will be stored there. This is pretty much mandetory if you
1511    are using AllowMem. AllowMem lets the function be run as non-root
1512    where it builds the cache 'fast' into a memory buffer. */
1513 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1514 {
1515    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1516    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1517    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1518    Flags |= MMap::Moveable;
1519    if (_config->FindB("APT::Cache-Fallback", false) == true)
1520       Flags |= MMap::Fallback;
1521    if (CacheF != NULL)
1522       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1523    else
1524       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1525 }
1526 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1527       std::string const &FileName)
1528 {
1529    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1530    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1531       return false;
1532
1533    fchmod(SCacheF.Fd(),0644);
1534
1535    // Write out the main data
1536    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1537       return _error->Error(_("IO Error saving source cache"));
1538
1539    // Write out the proper header
1540    Gen->GetCache().HeaderP->Dirty = false;
1541    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1542    if (SCacheF.Seek(0) == false ||
1543          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1544       return _error->Error(_("IO Error saving source cache"));
1545    Gen->GetCache().HeaderP->Dirty = true;
1546    return true;
1547 }
1548 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1549       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1550 {
1551    Map.reset(CreateDynamicMMap(NULL, 0));
1552    if (unlikely(Map->validData()) == false)
1553       return false;
1554    FileFd CacheF(FileName, FileFd::ReadOnly);
1555    if (CacheF.IsOpen() == false || CacheF.Failed())
1556       return false;
1557    _error->PushToStack();
1558    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1559    bool const newError = _error->PendingError();
1560    _error->MergeWithStack();
1561    if (alloc == 0 && newError)
1562       return false;
1563    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1564       return false;
1565    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1566    return true;
1567 }
1568 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1569                         MMap **OutMap, bool AllowMem)
1570    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1571 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1572                         MMap **OutMap,bool)
1573 {
1574    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1575 }
1576 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1577                         MMap **OutMap,pkgCache **OutCache, bool)
1578 {
1579    // FIXME: deprecate the ignored AllowMem parameter
1580    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1581
1582    std::vector<pkgIndexFile *> Files;
1583    if (_system->AddStatusFiles(Files) == false)
1584       return false;
1585
1586    // Decide if we can write to the files..
1587    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1588    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1589
1590    // ensure the cache directory exists
1591    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1592    {
1593       string dir = _config->FindDir("Dir::Cache");
1594       size_t const len = dir.size();
1595       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1596          dir = dir.substr(0, len - 5);
1597       if (CacheFile.empty() == false)
1598          CreateDirectory(dir, flNotFile(CacheFile));
1599       if (SrcCacheFile.empty() == false)
1600          CreateDirectory(dir, flNotFile(SrcCacheFile));
1601    }
1602
1603    if (Progress != NULL)
1604       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1605
1606    bool pkgcache_fine = false;
1607    bool srcpkgcache_fine = false;
1608    bool volatile_fine = List.GetVolatileFiles().empty();
1609
1610    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1611                      volatile_fine ? OutCache : NULL) == true)
1612    {
1613       if (Debug == true)
1614          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1615       pkgcache_fine = true;
1616       srcpkgcache_fine = true;
1617    }
1618    if (pkgcache_fine == false)
1619    {
1620       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1621       {
1622          if (Debug == true)
1623             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1624          srcpkgcache_fine = true;
1625       }
1626    }
1627
1628    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1629    {
1630       if (Progress != NULL)
1631          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1632       return true;
1633    }
1634
1635    bool Writeable = false;
1636    if (srcpkgcache_fine == false || pkgcache_fine == false)
1637    {
1638       if (CacheFile.empty() == false)
1639          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1640       else if (SrcCacheFile.empty() == false)
1641          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1642
1643       if (Debug == true)
1644          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1645    }
1646
1647    // At this point we know we need to construct something, so get storage ready
1648    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1649    if (unlikely(Map->validData()) == false)
1650       return false;
1651    if (Debug == true)
1652       std::clog << "Open memory Map (not filebased)" << std::endl;
1653
1654    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1655    map_filesize_t CurrentSize = 0;
1656    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1657    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1658    if (srcpkgcache_fine == true && pkgcache_fine == false)
1659    {
1660       if (Debug == true)
1661          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1662       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1663          return false;
1664       srcpkgcache_fine = true;
1665       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1666    }
1667    else if (srcpkgcache_fine == false)
1668    {
1669       if (Debug == true)
1670          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1671       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1672
1673       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1674       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1675                Files.end(),Files.end()) == false)
1676          return false;
1677
1678       if (Writeable == true && SrcCacheFile.empty() == false)
1679          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1680             return false;
1681    }
1682
1683    if (pkgcache_fine == false)
1684    {
1685       if (Debug == true)
1686          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1687       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1688                Files.begin(), Files.end()) == false)
1689          return false;
1690
1691       if (Writeable == true && CacheFile.empty() == false)
1692          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1693             return false;
1694    }
1695
1696    if (Debug == true)
1697       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1698
1699    if (volatile_fine == false)
1700    {
1701       if (Gen == nullptr)
1702       {
1703          if (Debug == true)
1704             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1705          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1706             return false;
1707       }
1708
1709       Files = List.GetVolatileFiles();
1710       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1711                Files.begin(), Files.end()) == false)
1712          return false;
1713    }
1714
1715    if (OutMap != nullptr)
1716       *OutMap = Map.release();
1717
1718    if (Debug == true)
1719       std::clog << "Everything is ready for shipping" << std::endl;
1720    return true;
1721 }
1722                                                                         /*}}}*/
1723 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1724 class APT_HIDDEN ScopedErrorMerge {
1725 public:
1726    ScopedErrorMerge() { _error->PushToStack(); }
1727    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1728 };
1729 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1730    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1731 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1732 {
1733    std::vector<pkgIndexFile *> Files;
1734    if (_system->AddStatusFiles(Files) == false)
1735       return false;
1736
1737    ScopedErrorMerge sem;
1738    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1739    if (unlikely(Map->validData()) == false)
1740       return false;
1741    map_filesize_t CurrentSize = 0;
1742    map_filesize_t TotalSize = 0;
1743    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1744
1745    // Build the status cache
1746    if (Progress != NULL)
1747       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1748    pkgCacheGenerator Gen(Map.get(),Progress);
1749    if (_error->PendingError() == true)
1750       return false;
1751    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1752                   Files.begin(), Files.end()) == false)
1753       return false;
1754
1755    if (_error->PendingError() == true)
1756       return false;
1757    *OutMap = Map.release();
1758
1759    return true;
1760 }
1761                                                                         /*}}}*/
1762 // IsDuplicateDescription                                               /*{{{*/
1763 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1764                             MD5SumValue const &CurMd5, std::string const &CurLang)
1765 {
1766    // Descriptions in the same link-list have all the same md5
1767    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1768       return false;
1769    for (; Desc.end() == false; ++Desc)
1770       if (Desc.LanguageCode() == CurLang)
1771          return true;
1772    return false;
1773 }
1774                                                                         /*}}}*/
1775
1776 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1777 pkgCacheListParser::~pkgCacheListParser() {}