apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <memory>
  40 #include <algorithm>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  47 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  48
  49 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  50                             MD5SumValue const &CurMd5, std::string const &CurLang);
  51
  52 using std::string;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (_error->PendingError() == true)
  62       return;
  63
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  69          return;
  70
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72
  73       // Starting header
  74       *Cache.HeaderP = pkgCache::Header();
  75
  76       // make room for the hashtables for packages and groups
  77       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  78          return;
  79
  80       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  81       if (unlikely(idxVerSysName == 0))
  82          return;
  83       Cache.HeaderP->VerSysName = idxVerSysName;
  84       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  85       if (unlikely(idxArchitecture == 0))
  86          return;
  87       Cache.HeaderP->Architecture = idxArchitecture;
  88
  89       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  90       if (archs.size() > 1)
  91       {
  92          std::vector<std::string>::const_iterator a = archs.begin();
  93          std::string list = *a;
  94          for (++a; a != archs.end(); ++a)
  95             list.append(",").append(*a);
  96          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  97          if (unlikely(idxArchitectures == 0))
  98             return;
  99          Cache.HeaderP->SetArchitectures(idxArchitectures);
 100       }
 101       else
 102          Cache.HeaderP->SetArchitectures(idxArchitecture);
 103
 104       Cache.ReMap();
 105    }
 106    else
 107    {
 108       // Map directly from the existing file
 109       Cache.ReMap();
 110       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 111       if (Cache.VS != _system->VS)
 112       {
 113          _error->Error(_("Cache has an incompatible versioning system"));
 114          return;
 115       }
 116    }
 117
 118    Cache.HeaderP->Dirty = true;
 119    Map.Sync(0,sizeof(pkgCache::Header));
 120 }
 121                                                                         /*}}}*/
 122 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* We sync the data then unset the dirty flag in two steps so as to
 125    advoid a problem during a crash */
 126 pkgCacheGenerator::~pkgCacheGenerator()
 127 {
 128    if (_error->PendingError() == true)
 129       return;
 130    if (Map.Sync() == false)
 131       return;
 132
 133    Cache.HeaderP->Dirty = false;
 134    Cache.HeaderP->CacheFileSize = Map.Size();
 135    Map.Sync(0,sizeof(pkgCache::Header));
 136 }
 137                                                                         /*}}}*/
 138 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 139    if (oldMap == newMap)
 140       return;
 141
 142    if (_config->FindB("Debug::pkgCacheGen", false))
 143       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 144
 145    Cache.ReMap(false);
 146
 147    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 148    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 149
 150    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 151         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 152       (*i)->ReMap(oldMap, newMap);
 153    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 154         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 155       (*i)->ReMap(oldMap, newMap);
 156    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174 }                                                                       /*}}}*/
 175 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 176 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 177                                         const unsigned long &Len) {
 178    void const * const oldMap = Map.Data();
 179    map_stringitem_t const index = Map.WriteString(String, Len);
 180    if (index != 0)
 181       ReMap(oldMap, Map.Data());
 182    return index;
 183 }
 184                                                                         /*}}}*/
 185 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 186 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 187    void const * const oldMap = Map.Data();
 188    map_stringitem_t const index = Map.WriteString(String);
 189    if (index != 0)
 190       ReMap(oldMap, Map.Data());
 191    return index;
 192 }
 193                                                                         /*}}}*/
 194 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 195    void const * const oldMap = Map.Data();
 196    map_pointer_t const index = Map.Allocate(size);
 197    if (index != 0)
 198       ReMap(oldMap, Map.Data());
 199    return index;
 200 }
 201                                                                         /*}}}*/
 202 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 203 // ---------------------------------------------------------------------
 204 /* This provides the generation of the entries in the cache. Each loop
 205    goes through a single package record from the underlying parse engine. */
 206 bool pkgCacheGenerator::MergeList(ListParser &List,
 207                                   pkgCache::VerIterator *OutVer)
 208 {
 209    List.Owner = this;
 210
 211    unsigned int Counter = 0;
 212    while (List.Step() == true)
 213    {
 214       string const PackageName = List.Package();
 215       if (PackageName.empty() == true)
 216          return false;
 217
 218       Counter++;
 219       if (Counter % 100 == 0 && Progress != 0)
 220          Progress->Progress(List.Offset());
 221
 222       string Arch = List.Architecture();
 223       string const Version = List.Version();
 224       if (Version.empty() == true && Arch.empty() == true)
 225       {
 226          // package descriptions
 227          if (MergeListGroup(List, PackageName) == false)
 228             return false;
 229          continue;
 230       }
 231
 232       if (Arch.empty() == true)
 233       {
 234          // use the pseudo arch 'none' for arch-less packages
 235          Arch = "none";
 236          /* We might built a SingleArchCache here, which we don't want to blow up
 237             just for these :none packages to a proper MultiArchCache, so just ensure
 238             that we have always a native package structure first for SingleArch */
 239          pkgCache::PkgIterator NP;
 240          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 241          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 242          // TRANSLATOR: The first placeholder is a package name,
 243          // the other two should be copied verbatim as they include debug info
 244          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 245                               PackageName.c_str(), "NewPackage", 0);
 246       }
 247
 248       // Get a pointer to the package structure
 249       pkgCache::PkgIterator Pkg;
 250       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 251       if (NewPackage(Pkg, PackageName, Arch) == false)
 252          // TRANSLATOR: The first placeholder is a package name,
 253          // the other two should be copied verbatim as they include debug info
 254          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 255                               PackageName.c_str(), "NewPackage", 1);
 256
 257
 258       if (Version.empty() == true)
 259       {
 260          if (MergeListPackage(List, Pkg) == false)
 261             return false;
 262       }
 263       else
 264       {
 265          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 266             return false;
 267       }
 268
 269       if (OutVer != 0)
 270          return true;
 271    }
 272
 273    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 274       return _error->Error(_("Wow, you exceeded the number of package "
 275                              "names this APT is capable of."));
 276    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 277       return _error->Error(_("Wow, you exceeded the number of versions "
 278                              "this APT is capable of."));
 279    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 280       return _error->Error(_("Wow, you exceeded the number of descriptions "
 281                              "this APT is capable of."));
 282    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 283       return _error->Error(_("Wow, you exceeded the number of dependencies "
 284                              "this APT is capable of."));
 285
 286    return true;
 287 }
 288 // CacheGenerator::MergeListGroup                                       /*{{{*/
 289 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 290 {
 291    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 292    // a group has no data on it's own, only packages have it but these
 293    // stanzas like this come from Translation- files to add descriptions,
 294    // but without a version we don't need a description for it…
 295    if (Grp.end() == true)
 296       return true;
 297    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 298
 299    pkgCache::PkgIterator Pkg;
 300    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 301    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 302       if (MergeListPackage(List, Pkg) == false)
 303          return false;
 304
 305    return true;
 306 }
 307                                                                         /*}}}*/
 308 // CacheGenerator::MergeListPackage                                     /*{{{*/
 309 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 310 {
 311    // we first process the package, then the descriptions
 312    // (for deb this package processing is in fact a no-op)
 313    pkgCache::VerIterator Ver(Cache);
 314    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 315    if (List.UsePackage(Pkg, Ver) == false)
 316       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 317                            Pkg.Name(), "UsePackage", 1);
 318
 319    // Find the right version to write the description
 320    MD5SumValue CurMd5 = List.Description_md5();
 321    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 322       return true;
 323    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 324    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 325    {
 326       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 327
 328       // a version can only have one md5 describing it
 329       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 330          continue;
 331
 332       map_stringitem_t md5idx = VerDesc->md5sum;
 333       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 334       {
 335          // don't add a new description if we have one for the given
 336          // md5 && language
 337          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 338             continue;
 339
 340          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 341       }
 342
 343       // we can stop here as all "same" versions will share the description
 344       break;
 345    }
 346
 347    return true;
 348 }
 349                                                                         /*}}}*/
 350 // CacheGenerator::MergeListVersion                                     /*{{{*/
 351 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 352                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 353 {
 354    pkgCache::VerIterator Ver = Pkg.VersionList();
 355    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 356    map_pointer_t *LastVer = &Pkg->VersionList;
 357    void const * oldMap = Map.Data();
 358
 359    unsigned short const Hash = List.VersionHash();
 360    if (Ver.end() == false)
 361    {
 362       /* We know the list is sorted so we use that fact in the search.
 363          Insertion of new versions is done with correct sorting */
 364       int Res = 1;
 365       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 366       {
 367          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 368          // Version is higher as current version - insert here
 369          if (Res > 0)
 370             break;
 371          // Versionstrings are equal - is hash also equal?
 372          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 373             break;
 374          // proceed with the next till we have either the right
 375          // or we found another version (which will be lower)
 376       }
 377
 378       /* We already have a version for this item, record that we saw it */
 379       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 380       {
 381          if (List.UsePackage(Pkg,Ver) == false)
 382             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 383                                  Pkg.Name(), "UsePackage", 2);
 384
 385          if (NewFileVer(Ver,List) == false)
 386             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 387                                  Pkg.Name(), "NewFileVer", 1);
 388
 389          // Read only a single record and return
 390          if (OutVer != 0)
 391          {
 392             *OutVer = Ver;
 393             return true;
 394          }
 395
 396          return true;
 397       }
 398    }
 399
 400    // Add a new version
 401    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 402    if (verindex == 0 && _error->PendingError())
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "NewVersion", 1);
 405
 406    if (oldMap != Map.Data())
 407          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 408    *LastVer = verindex;
 409
 410    if (unlikely(List.NewVersion(Ver) == false))
 411       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 412                            Pkg.Name(), "NewVersion", 2);
 413
 414    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 415       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 416                            Pkg.Name(), "UsePackage", 3);
 417
 418    if (unlikely(NewFileVer(Ver,List) == false))
 419       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 420                            Pkg.Name(), "NewFileVer", 2);
 421
 422    pkgCache::GrpIterator Grp = Pkg.Group();
 423    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 424
 425    /* If it is the first version of this package we need to add implicit
 426       Multi-Arch dependencies to all other package versions in the group now -
 427       otherwise we just add them for this new version */
 428    if (Pkg.VersionList()->NextVer == 0)
 429    {
 430       pkgCache::PkgIterator P = Grp.PackageList();
 431       Dynamic<pkgCache::PkgIterator> DynP(P);
 432       for (; P.end() != true; P = Grp.NextPkg(P))
 433       {
 434          if (P->ID == Pkg->ID)
 435             continue;
 436          pkgCache::VerIterator V = P.VersionList();
 437          Dynamic<pkgCache::VerIterator> DynV(V);
 438          for (; V.end() != true; ++V)
 439             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 440                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 441                                     Pkg.Name(), "AddImplicitDepends", 1);
 442       }
 443       /* :none packages are packages without an architecture. They are forbidden by
 444          debian-policy, so usually they will only be in (old) dpkg status files -
 445          and dpkg will complain about them - and are pretty rare. We therefore do
 446          usually not create conflicts while the parent is created, but only if a :none
 447          package (= the target) appears. This creates incorrect dependencies on :none
 448          for architecture-specific dependencies on the package we copy from, but we
 449          will ignore this bug as architecture-specific dependencies are only allowed
 450          in jessie and until then the :none packages should be extinct (hopefully).
 451          In other words: This should work long enough to allow graceful removal of
 452          these packages, it is not supposed to allow users to keep using them … */
 453       if (strcmp(Pkg.Arch(), "none") == 0)
 454       {
 455          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 456          if (M.end() == false && Pkg != M)
 457          {
 458             pkgCache::DepIterator D = M.RevDependsList();
 459             Dynamic<pkgCache::DepIterator> DynD(D);
 460             for (; D.end() == false; ++D)
 461             {
 462                if ((D->Type != pkgCache::Dep::Conflicts &&
 463                     D->Type != pkgCache::Dep::DpkgBreaks &&
 464                     D->Type != pkgCache::Dep::Replaces) ||
 465                    D.ParentPkg().Group() == Grp)
 466                   continue;
 467
 468                map_pointer_t *OldDepLast = NULL;
 469                pkgCache::VerIterator ConVersion = D.ParentVer();
 470                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 471                // duplicate the Conflicts/Breaks/Replaces for :none arch
 472                NewDepends(Pkg, ConVersion, D->Version,
 473                      D->CompareOp, D->Type, OldDepLast);
 474             }
 475          }
 476       }
 477    }
 478    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 479       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 480                            Pkg.Name(), "AddImplicitDepends", 2);
 481
 482    // Read only a single record and return
 483    if (OutVer != 0)
 484    {
 485       *OutVer = Ver;
 486       return true;
 487    }
 488
 489    /* Record the Description(s) based on their master md5sum */
 490    MD5SumValue CurMd5 = List.Description_md5();
 491    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 492       return true;
 493
 494    /* Before we add a new description we first search in the group for
 495       a version with a description of the same MD5 - if so we reuse this
 496       description group instead of creating our own for this version */
 497    for (pkgCache::PkgIterator P = Grp.PackageList();
 498         P.end() == false; P = Grp.NextPkg(P))
 499    {
 500       for (pkgCache::VerIterator V = P.VersionList();
 501            V.end() == false; ++V)
 502       {
 503          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 504             continue;
 505          Ver->DescriptionList = V->DescriptionList;
 506       }
 507    }
 508
 509    // We haven't found reusable descriptions, so add the first description(s)
 510    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 511    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 512    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 513       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 514          return false;
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 519 {
 520    pkgCache::DescIterator Desc;
 521    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 522
 523    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 524    if (unlikely(descindex == 0 && _error->PendingError()))
 525       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 526             Ver.ParentPkg().Name(), "NewDescription", 1);
 527
 528    md5idx = Desc->md5sum;
 529    Desc->ParentPkg = Ver.ParentPkg().Index();
 530
 531    // we add at the end, so that the start is constant as we need
 532    // that to be able to efficiently share these lists
 533    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 534    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 535    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 536    *LastNextDesc = descindex;
 537
 538    if (NewFileDesc(Desc,List) == false)
 539       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 540             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 541
 542    return true;
 543 }
 544                                                                         /*}}}*/
 545                                                                         /*}}}*/
 546 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* This creates a new group structure and adds it to the hash table */
 549 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 550 {
 551    Grp = Cache.FindGrp(Name);
 552    if (Grp.end() == false)
 553       return true;
 554
 555    // Get a structure
 556    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 557    if (unlikely(Group == 0))
 558       return false;
 559
 560    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 561    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 562    if (unlikely(idxName == 0))
 563       return false;
 564    Grp->Name = idxName;
 565
 566    // Insert it into the hash table
 567    unsigned long const Hash = Cache.Hash(Name);
 568    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 569    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 570       insertAt = &(Cache.GrpP + *insertAt)->Next;
 571    Grp->Next = *insertAt;
 572    *insertAt = Group;
 573
 574    Grp->ID = Cache.HeaderP->GroupCount++;
 575    return true;
 576 }
 577                                                                         /*}}}*/
 578 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* This creates a new package structure and adds it to the hash table */
 581 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 582                                         const string &Arch) {
 583    pkgCache::GrpIterator Grp;
 584    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 585    if (unlikely(NewGroup(Grp, Name) == false))
 586       return false;
 587
 588    Pkg = Grp.FindPkg(Arch);
 589       if (Pkg.end() == false)
 590          return true;
 591
 592    // Get a structure
 593    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 594    if (unlikely(Package == 0))
 595       return false;
 596    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 597
 598    // Set the name, arch and the ID
 599    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 600    Pkg->Group = Grp.Index();
 601    // all is mapped to the native architecture
 602    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 603    if (unlikely(idxArch == 0))
 604       return false;
 605    Pkg->Arch = idxArch;
 606    Pkg->ID = Cache.HeaderP->PackageCount++;
 607
 608    // Insert the package into our package list
 609    if (Grp->FirstPackage == 0) // the group is new
 610    {
 611       Grp->FirstPackage = Package;
 612       // Insert it into the hash table
 613       map_id_t const Hash = Cache.Hash(Name);
 614       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 615       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 616          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 617       Pkg->NextPackage = *insertAt;
 618       *insertAt = Package;
 619    }
 620    else // Group the Packages together
 621    {
 622       // but first get implicit provides done
 623       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 624       {
 625          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 626          if (M.end() == false)
 627             for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 628             {
 629                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 630                   continue;
 631                pkgCache::VerIterator Ver = Prv.OwnerVer();
 632                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 633                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 634                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 635                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 636                      return false;
 637             }
 638
 639          for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 640             for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
 641                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 642                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 643                      return false;
 644       }
 645       // and negative dependencies, don't forget negative dependencies
 646       {
 647          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 648          if (M.end() == false)
 649             for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 650             {
 651                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 652                   continue;
 653                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 654                      Dep->Type != pkgCache::Dep::Replaces)
 655                   continue;
 656                pkgCache::VerIterator Ver = Dep.ParentVer();
 657                map_pointer_t * unused = NULL;
 658                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 659                   return false;
 660             }
 661       }
 662
 663       // this package is the new last package
 664       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 665       Pkg->NextPackage = LastPkg->NextPackage;
 666       LastPkg->NextPackage = Package;
 667    }
 668    Grp->LastPackage = Package;
 669    return true;
 670 }
 671                                                                         /*}}}*/
 672 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 673 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 674                                            pkgCache::PkgIterator &P,
 675                                            pkgCache::VerIterator &V)
 676 {
 677    // copy P.Arch() into a string here as a cache remap
 678    // in NewDepends() later may alter the pointer location
 679    string Arch = P.Arch() == NULL ? "" : P.Arch();
 680    map_pointer_t *OldDepLast = NULL;
 681    /* MultiArch handling introduces a lot of implicit Dependencies:
 682       - MultiArch: same → Co-Installable if they have the same version
 683       - All others conflict with all other group members */
 684    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 685    pkgCache::PkgIterator D = G.PackageList();
 686    Dynamic<pkgCache::PkgIterator> DynD(D);
 687    map_stringitem_t const VerStrIdx = V->VerStr;
 688    for (; D.end() != true; D = G.NextPkg(D))
 689    {
 690       if (Arch == D.Arch() || D->VersionList == 0)
 691          continue;
 692       /* We allow only one installed arch at the time
 693          per group, therefore each group member conflicts
 694          with all other group members */
 695       if (coInstall == true)
 696       {
 697          // Replaces: ${self}:other ( << ${binary:Version})
 698          NewDepends(D, V, VerStrIdx,
 699                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 700                     OldDepLast);
 701          // Breaks: ${self}:other (!= ${binary:Version})
 702          NewDepends(D, V, VerStrIdx,
 703                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 704                     OldDepLast);
 705       } else {
 706          // Conflicts: ${self}:other
 707          NewDepends(D, V, 0,
 708                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 709                     OldDepLast);
 710       }
 711    }
 712    return true;
 713 }
 714 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 715                                            pkgCache::PkgIterator &D)
 716 {
 717    /* MultiArch handling introduces a lot of implicit Dependencies:
 718       - MultiArch: same → Co-Installable if they have the same version
 719       - All others conflict with all other group members */
 720    map_pointer_t *OldDepLast = NULL;
 721    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 722    if (coInstall == true)
 723    {
 724       map_stringitem_t const VerStrIdx = V->VerStr;
 725       // Replaces: ${self}:other ( << ${binary:Version})
 726       NewDepends(D, V, VerStrIdx,
 727                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 728                  OldDepLast);
 729       // Breaks: ${self}:other (!= ${binary:Version})
 730       NewDepends(D, V, VerStrIdx,
 731                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 732                  OldDepLast);
 733    } else {
 734       // Conflicts: ${self}:other
 735       NewDepends(D, V, 0,
 736                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 737                  OldDepLast);
 738    }
 739    return true;
 740 }
 741
 742                                                                         /*}}}*/
 743 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 744 // ---------------------------------------------------------------------
 745 /* */
 746 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 747                                    ListParser &List)
 748 {
 749    if (CurrentFile == 0)
 750       return true;
 751
 752    // Get a structure
 753    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 754    if (VerFile == 0)
 755       return false;
 756
 757    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 758    VF->File = CurrentFile - Cache.PkgFileP;
 759
 760    // Link it to the end of the list
 761    map_pointer_t *Last = &Ver->FileList;
 762    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 763       Last = &V->NextFile;
 764    VF->NextFile = *Last;
 765    *Last = VF.Index();
 766
 767    VF->Offset = List.Offset();
 768    VF->Size = List.Size();
 769    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 770       Cache.HeaderP->MaxVerFileSize = VF->Size;
 771    Cache.HeaderP->VerFileCount++;
 772
 773    return true;
 774 }
 775                                                                         /*}}}*/
 776 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 777 // ---------------------------------------------------------------------
 778 /* This puts a version structure in the linked list */
 779 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 780                                             const string &VerStr,
 781                                             map_pointer_t const ParentPkg,
 782                                             unsigned short const Hash,
 783                                             map_pointer_t const Next)
 784 {
 785    // Get a structure
 786    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 787    if (Version == 0)
 788       return 0;
 789
 790    // Fill it in
 791    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 792    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 793    Ver->NextVer = Next;
 794    Ver->ParentPkg = ParentPkg;
 795    Ver->Hash = Hash;
 796    Ver->ID = Cache.HeaderP->VersionCount++;
 797
 798    // try to find the version string in the group for reuse
 799    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 800    pkgCache::GrpIterator Grp = Pkg.Group();
 801    if (Pkg.end() == false && Grp.end() == false)
 802    {
 803       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 804       {
 805          if (Pkg == P)
 806             continue;
 807          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 808          {
 809             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 810             if (cmp == 0)
 811             {
 812                Ver->VerStr = V->VerStr;
 813                return Version;
 814             }
 815             else if (cmp < 0)
 816                break;
 817          }
 818       }
 819    }
 820    // haven't found the version string, so create
 821    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 822    if (unlikely(idxVerStr == 0))
 823       return 0;
 824    Ver->VerStr = idxVerStr;
 825    return Version;
 826 }
 827                                                                         /*}}}*/
 828 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 829 // ---------------------------------------------------------------------
 830 /* */
 831 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 832                                    ListParser &List)
 833 {
 834    if (CurrentFile == 0)
 835       return true;
 836
 837    // Get a structure
 838    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 839    if (DescFile == 0)
 840       return false;
 841
 842    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 843    DF->File = CurrentFile - Cache.PkgFileP;
 844
 845    // Link it to the end of the list
 846    map_pointer_t *Last = &Desc->FileList;
 847    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 848       Last = &D->NextFile;
 849
 850    DF->NextFile = *Last;
 851    *Last = DF.Index();
 852
 853    DF->Offset = List.Offset();
 854    DF->Size = List.Size();
 855    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 856       Cache.HeaderP->MaxDescFileSize = DF->Size;
 857    Cache.HeaderP->DescFileCount++;
 858
 859    return true;
 860 }
 861                                                                         /*}}}*/
 862 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 863 // ---------------------------------------------------------------------
 864 /* This puts a description structure in the linked list */
 865 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 866                                             const string &Lang,
 867                                             const MD5SumValue &md5sum,
 868                                             map_stringitem_t const idxmd5str)
 869 {
 870    // Get a structure
 871    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 872    if (Description == 0)
 873       return 0;
 874
 875    // Fill it in
 876    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 877    Desc->ID = Cache.HeaderP->DescriptionCount++;
 878    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 879    if (unlikely(idxlanguage_code == 0))
 880       return 0;
 881    Desc->language_code = idxlanguage_code;
 882
 883    if (idxmd5str != 0)
 884       Desc->md5sum = idxmd5str;
 885    else
 886    {
 887       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 888       if (unlikely(idxmd5sum == 0))
 889          return 0;
 890       Desc->md5sum = idxmd5sum;
 891    }
 892
 893    return Description;
 894 }
 895                                                                         /*}}}*/
 896 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 897 // ---------------------------------------------------------------------
 898 /* This creates a dependency element in the tree. It is linked to the
 899    version and to the package that it is pointing to. */
 900 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 901                                    pkgCache::VerIterator &Ver,
 902                                    map_pointer_t const Version,
 903                                    uint8_t const Op,
 904                                    uint8_t const Type,
 905                                    map_pointer_t* &OldDepLast)
 906 {
 907    void const * const oldMap = Map.Data();
 908    // Get a structure
 909    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 910    if (unlikely(Dependency == 0))
 911       return false;
 912
 913    bool isDuplicate = false;
 914    map_pointer_t DependencyData = 0;
 915    map_pointer_t PreviousData = 0;
 916    if (Pkg->RevDepends != 0)
 917    {
 918       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 919       DependencyData = L->DependencyData;
 920       do {
 921          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 922          if (Version > D->Version)
 923             break;
 924          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 925          {
 926             isDuplicate = true;
 927             break;
 928          }
 929          PreviousData = DependencyData;
 930          DependencyData = D->NextData;
 931       } while (DependencyData != 0);
 932    }
 933
 934    if (isDuplicate == false)
 935    {
 936       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 937       if (unlikely(DependencyData == 0))
 938         return false;
 939    }
 940
 941    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 942    Link->ParentVer = Ver.Index();
 943    Link->DependencyData = DependencyData;
 944    Link->ID = Cache.HeaderP->DependsCount++;
 945
 946    pkgCache::DepIterator Dep(Cache, Link);
 947    if (isDuplicate == false)
 948    {
 949       Dep->Type = Type;
 950       Dep->CompareOp = Op;
 951       Dep->Version = Version;
 952       Dep->Package = Pkg.Index();
 953       ++Cache.HeaderP->DependsDataCount;
 954       if (PreviousData != 0)
 955       {
 956          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 957          Dep->NextData = D->NextData;
 958          D->NextData = DependencyData;
 959       }
 960       else if (Pkg->RevDepends != 0)
 961       {
 962          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 963          Dep->NextData = D->DependencyData;
 964       }
 965    }
 966
 967    if (isDuplicate == true || PreviousData != 0)
 968    {
 969       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 970       Link->NextRevDepends = L->NextRevDepends;
 971       L->NextRevDepends = Dependency;
 972    }
 973    else
 974    {
 975       Link->NextRevDepends = Pkg->RevDepends;
 976       Pkg->RevDepends = Dependency;
 977    }
 978
 979
 980    // Do we know where to link the Dependency to?
 981    if (OldDepLast == NULL)
 982    {
 983       OldDepLast = &Ver->DependsList;
 984       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 985          OldDepLast = &D->NextDepends;
 986    } else if (oldMap != Map.Data())
 987       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 988
 989    Dep->NextDepends = *OldDepLast;
 990    *OldDepLast = Dependency;
 991    OldDepLast = &Dep->NextDepends;
 992    return true;
 993 }
 994                                                                         /*}}}*/
 995 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 996 // ---------------------------------------------------------------------
 997 /* This creates a Group and the Package to link this dependency to if
 998    needed and handles also the caching of the old endpoint */
 999 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1000                                                const string &PackageName,
1001                                                const string &Arch,
1002                                                const string &Version,
1003                                                uint8_t const Op,
1004                                                uint8_t const Type)
1005 {
1006    pkgCache::GrpIterator Grp;
1007    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1008    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1009       return false;
1010
1011    map_stringitem_t idxVersion = 0;
1012    if (Version.empty() == false)
1013    {
1014       int const CmpOp = Op & 0x0F;
1015       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1016       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1017          idxVersion = Ver->VerStr;
1018
1019       if (idxVersion == 0)
1020       {
1021          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1022          if (unlikely(idxVersion == 0))
1023             return false;
1024       }
1025    }
1026
1027    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1028          Type == pkgCache::Dep::Conflicts ||
1029          Type == pkgCache::Dep::Replaces);
1030
1031    pkgCache::PkgIterator Pkg;
1032    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1033    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1034    {
1035       // Locate the target package
1036       Pkg = Grp.FindPkg(Arch);
1037       if (Pkg.end() == true) {
1038          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1039             return false;
1040       }
1041
1042       /* Caching the old end point speeds up generation substantially */
1043       if (OldDepVer != Ver) {
1044          OldDepLast = NULL;
1045          OldDepVer = Ver;
1046       }
1047
1048       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1049    }
1050    else
1051    {
1052       /* Caching the old end point speeds up generation substantially */
1053       if (OldDepVer != Ver) {
1054          OldDepLast = NULL;
1055          OldDepVer = Ver;
1056       }
1057
1058       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1059       {
1060          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1061                return false;
1062       }
1063    }
1064    return true;
1065 }
1066                                                                         /*}}}*/
1067 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1068 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1069                                                 const string &PkgName,
1070                                                 const string &PkgArch,
1071                                                 const string &Version,
1072                                                 uint8_t const Flags)
1073 {
1074    pkgCache const &Cache = Owner->Cache;
1075
1076    // We do not add self referencing provides
1077    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1078         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1079       return true;
1080
1081    // Locate the target package
1082    pkgCache::PkgIterator Pkg;
1083    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1084    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1085       return false;
1086
1087    map_stringitem_t idxProvideVersion = 0;
1088    if (Version.empty() == false) {
1089       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1090       if (unlikely(idxProvideVersion == 0))
1091          return false;
1092    }
1093    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1094 }
1095 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1096                                     pkgCache::PkgIterator &Pkg,
1097                                     map_pointer_t const ProvideVersion,
1098                                     uint8_t const Flags)
1099 {
1100    // Get a structure
1101    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1102    if (unlikely(Provides == 0))
1103       return false;
1104    ++Cache.HeaderP->ProvidesCount;
1105
1106    // Fill it in
1107    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1108    Prv->Version = Ver.Index();
1109    Prv->ProvideVersion = ProvideVersion;
1110    Prv->Flags = Flags;
1111    Prv->NextPkgProv = Ver->ProvidesList;
1112    Ver->ProvidesList = Prv.Index();
1113
1114    // Link it to the package
1115    Prv->ParentPkg = Pkg.Index();
1116    Prv->NextProvides = Pkg->ProvidesList;
1117    Pkg->ProvidesList = Prv.Index();
1118    return true;
1119 }
1120                                                                         /*}}}*/
1121 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1122 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1123                                 string const &Version, uint8_t const Flags) {
1124    pkgCache &Cache = Owner->Cache;
1125    pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1126    if (Grp.end() == true)
1127       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1128    else
1129    {
1130       map_stringitem_t idxProvideVersion = 0;
1131       if (Version.empty() == false) {
1132          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1133          if (unlikely(idxProvideVersion == 0))
1134             return false;
1135       }
1136
1137       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1138       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1139       pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1140       for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1141       {
1142          if (isImplicit && OwnerPkg == Pkg)
1143             continue;
1144          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1145             continue;
1146          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1147             return false;
1148       }
1149    }
1150    return true;
1151 }
1152                                                                         /*}}}*/
1153 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1154       pkgCache::VerIterator const &Ver)
1155 {
1156    return Hash == Ver->Hash;
1157 }
1158                                                                         /*}}}*/
1159 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1160 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1161                                    unsigned long Flags)
1162 {
1163    if (File.empty() && Site.empty())
1164    {
1165       CurrentRlsFile = NULL;
1166       return true;
1167    }
1168
1169    // Get some space for the structure
1170    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1171    if (unlikely(idxFile == 0))
1172       return false;
1173    CurrentRlsFile = Cache.RlsFileP + idxFile;
1174
1175    // Fill it in
1176    map_stringitem_t const idxFileName = WriteStringInMap(File);
1177    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1178    if (unlikely(idxFileName == 0 || idxSite == 0))
1179       return false;
1180    CurrentRlsFile->FileName = idxFileName;
1181    CurrentRlsFile->Site = idxSite;
1182    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1183    CurrentRlsFile->Flags = Flags;
1184    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1185    RlsFileName = File;
1186    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1187    Cache.HeaderP->ReleaseFileCount++;
1188
1189    return true;
1190 }
1191                                                                         /*}}}*/
1192 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1193 // ---------------------------------------------------------------------
1194 /* This is used to select which file is to be associated with all newly
1195    added versions. The caller is responsible for setting the IMS fields. */
1196 bool pkgCacheGenerator::SelectFile(std::string const &File,
1197                                    pkgIndexFile const &Index,
1198                                    std::string const &Architecture,
1199                                    std::string const &Component,
1200                                    unsigned long const Flags)
1201 {
1202    // Get some space for the structure
1203    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1204    if (unlikely(idxFile == 0))
1205       return false;
1206    CurrentFile = Cache.PkgFileP + idxFile;
1207
1208    // Fill it in
1209    map_stringitem_t const idxFileName = WriteStringInMap(File);
1210    if (unlikely(idxFileName == 0))
1211       return false;
1212    CurrentFile->FileName = idxFileName;
1213    CurrentFile->NextFile = Cache.HeaderP->FileList;
1214    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1215    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1216    if (unlikely(idxIndexType == 0))
1217       return false;
1218    CurrentFile->IndexType = idxIndexType;
1219    if (Architecture.empty())
1220       CurrentFile->Architecture = 0;
1221    else
1222    {
1223       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1224       if (unlikely(arch == 0))
1225          return false;
1226       CurrentFile->Architecture = arch;
1227    }
1228    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1229    if (unlikely(component == 0))
1230       return false;
1231    CurrentFile->Component = component;
1232    CurrentFile->Flags = Flags;
1233    if (CurrentRlsFile != NULL)
1234       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1235    else
1236       CurrentFile->Release = 0;
1237    PkgFileName = File;
1238    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1239    Cache.HeaderP->PackageFileCount++;
1240
1241    if (Progress != 0)
1242       Progress->SubProgress(Index.Size());
1243    return true;
1244 }
1245                                                                         /*}}}*/
1246 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1247 // ---------------------------------------------------------------------
1248 /* This is used to create handles to strings. Given the same text it
1249    always returns the same number */
1250 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1251                                                  unsigned int Size)
1252 {
1253    std::string const key(S, Size);
1254
1255    std::map<std::string,map_stringitem_t> * strings;
1256    switch(type) {
1257       case MIXED: strings = &strMixed; break;
1258       case PKGNAME: strings = &strPkgNames; break;
1259       case VERSIONNUMBER: strings = &strVersions; break;
1260       case SECTION: strings = &strSections; break;
1261       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1262    }
1263
1264    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1265    if (item != strings->end())
1266       return item->second;
1267
1268    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1269    strings->insert(std::make_pair(key, idxString));
1270    return idxString;
1271 }
1272                                                                         /*}}}*/
1273 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1274 // ---------------------------------------------------------------------
1275 /* This just verifies that each file in the list of index files exists,
1276    has matching attributes with the cache and the cache does not have
1277    any extra files. */
1278 static bool CheckValidity(const string &CacheFile,
1279                           pkgSourceList &List,
1280                           FileIterator const Start,
1281                           FileIterator const End,
1282                           MMap **OutMap = 0)
1283 {
1284    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1285    // No file, certainly invalid
1286    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1287    {
1288       if (Debug == true)
1289          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1290       return false;
1291    }
1292
1293    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1294    {
1295       if (Debug == true)
1296          std::clog << "sources.list is newer than the cache" << std::endl;
1297       return false;
1298    }
1299
1300    // Map it
1301    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1302    SPtr<MMap> Map = new MMap(CacheF,0);
1303    pkgCache Cache(Map);
1304    if (_error->PendingError() == true || Map->Size() == 0)
1305    {
1306       if (Debug == true)
1307          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1308       _error->Discard();
1309       return false;
1310    }
1311
1312    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1313    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1314    std::vector<pkgIndexFile *> Files;
1315    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1316    {
1317       if (Debug == true)
1318          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1319       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1320       if (RlsFile.end() == true)
1321       {
1322          if (Debug == true)
1323             std::clog << "FindInCache returned end-Pointer" << std::endl;
1324          return false;
1325       }
1326
1327       RlsVisited[RlsFile->ID] = true;
1328       if (Debug == true)
1329          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1330
1331       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1332       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1333             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1334    }
1335    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1336       if (RlsVisited[I] == false)
1337       {
1338          if (Debug == true)
1339             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1340          return false;
1341       }
1342
1343    std::copy(Start, End, std::back_inserter(Files));
1344
1345    /* Now we check every index file, see if it is in the cache,
1346       verify the IMS data and check that it is on the disk too.. */
1347    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1348    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1349    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1350    {
1351       if (Debug == true)
1352          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1353       if ((*PkgFile)->Exists() == false)
1354       {
1355          if (Debug == true)
1356             std::clog << "file doesn't exist" << std::endl;
1357          continue;
1358       }
1359
1360       // FindInCache is also expected to do an IMS check.
1361       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1362       if (File.end() == true)
1363       {
1364          if (Debug == true)
1365             std::clog << "FindInCache returned end-Pointer" << std::endl;
1366          return false;
1367       }
1368
1369       Visited[File->ID] = true;
1370       if (Debug == true)
1371          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1372    }
1373
1374    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1375       if (Visited[I] == false)
1376       {
1377          if (Debug == true)
1378             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1379          return false;
1380       }
1381
1382    if (_error->PendingError() == true)
1383    {
1384       if (Debug == true)
1385       {
1386          std::clog << "Validity failed because of pending errors:" << std::endl;
1387          _error->DumpErrors();
1388       }
1389       _error->Discard();
1390       return false;
1391    }
1392
1393    if (OutMap != 0)
1394       *OutMap = Map.UnGuard();
1395    return true;
1396 }
1397                                                                         /*}}}*/
1398 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1399 // ---------------------------------------------------------------------
1400 /* Size is kind of an abstract notion that is only used for the progress
1401    meter */
1402 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1403 {
1404    map_filesize_t TotalSize = 0;
1405    if (List !=  NULL)
1406    {
1407       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1408       {
1409          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1410          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1411             if ((*j)->HasPackages() == true)
1412                TotalSize += (*j)->Size();
1413       }
1414    }
1415
1416    for (; Start < End; ++Start)
1417    {
1418       if ((*Start)->HasPackages() == false)
1419          continue;
1420       TotalSize += (*Start)->Size();
1421    }
1422    return TotalSize;
1423 }
1424                                                                         /*}}}*/
1425 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1426 static bool BuildCache(pkgCacheGenerator &Gen,
1427                        OpProgress * const Progress,
1428                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1429                        pkgSourceList const * const List,
1430                        FileIterator const Start, FileIterator const End)
1431 {
1432    std::vector<pkgIndexFile *> Files;
1433    bool mergeFailure = false;
1434
1435    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1436       if (I->HasPackages() == false || mergeFailure)
1437          return;
1438
1439       if (I->Exists() == false)
1440          return;
1441
1442       if (I->FindInCache(Gen.GetCache()).end() == false)
1443       {
1444          _error->Warning("Duplicate sources.list entry %s",
1445                I->Describe().c_str());
1446          return;
1447       }
1448
1449       map_filesize_t const Size = I->Size();
1450       if (Progress != NULL)
1451          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1452       CurrentSize += Size;
1453
1454       if (I->Merge(Gen,Progress) == false)
1455          mergeFailure = true;
1456    };
1457
1458    if (List !=  NULL)
1459    {
1460       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1461       {
1462          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1463          {
1464             _error->Warning("Duplicate sources.list entry %s",
1465                   (*i)->Describe().c_str());
1466             continue;
1467          }
1468
1469          if ((*i)->Merge(Gen, Progress) == false)
1470             return false;
1471
1472          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1473          if (Indexes != NULL)
1474             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1475          if (mergeFailure)
1476             return false;
1477       }
1478    }
1479
1480    if (Start != End)
1481    {
1482       Gen.SelectReleaseFile("", "");
1483       std::for_each(Start, End, indexFileMerge);
1484       if (mergeFailure)
1485          return false;
1486    }
1487    return true;
1488 }
1489                                                                         /*}}}*/
1490 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1491 // ---------------------------------------------------------------------
1492 /* This makes sure that the status cache (the cache that has all
1493    index files from the sources list and all local ones) is ready
1494    to be mmaped. If OutMap is not zero then a MMap object representing
1495    the cache will be stored there. This is pretty much mandetory if you
1496    are using AllowMem. AllowMem lets the function be run as non-root
1497    where it builds the cache 'fast' into a memory buffer. */
1498 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1499 {
1500    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1501    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1502    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1503    Flags |= MMap::Moveable;
1504    if (_config->FindB("APT::Cache-Fallback", false) == true)
1505       Flags |= MMap::Fallback;
1506    if (CacheF != NULL)
1507       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1508    else
1509       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1510 }
1511 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1512       std::string const &FileName)
1513 {
1514    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1515    if (_error->PendingError() == true)
1516       return false;
1517
1518    fchmod(SCacheF.Fd(),0644);
1519
1520    // Write out the main data
1521    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1522       return _error->Error(_("IO Error saving source cache"));
1523    SCacheF.Sync();
1524
1525    // Write out the proper header
1526    Gen->GetCache().HeaderP->Dirty = false;
1527    if (SCacheF.Seek(0) == false ||
1528          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1529       return _error->Error(_("IO Error saving source cache"));
1530    Gen->GetCache().HeaderP->Dirty = true;
1531    SCacheF.Sync();
1532    return true;
1533 }
1534 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1535       SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1536 {
1537    Map = CreateDynamicMMap(NULL, 0);
1538    FileFd CacheF(FileName, FileFd::ReadOnly);
1539    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1540    if ((alloc == 0 && _error->PendingError())
1541          || CacheF.Read((unsigned char *)Map->Data() + alloc,
1542             CacheF.Size()) == false)
1543       return false;
1544    Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1545    return true;
1546 }
1547 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1548                         MMap **OutMap, bool AllowMem)
1549    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1550 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1551                         MMap **OutMap,bool AllowMem)
1552 {
1553    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1554
1555    std::vector<pkgIndexFile *> Files;
1556    if (_system->AddStatusFiles(Files) == false)
1557       return false;
1558
1559    // Decide if we can write to the files..
1560    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1561    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1562
1563    // ensure the cache directory exists
1564    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1565    {
1566       string dir = _config->FindDir("Dir::Cache");
1567       size_t const len = dir.size();
1568       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1569          dir = dir.substr(0, len - 5);
1570       if (CacheFile.empty() == false)
1571          CreateDirectory(dir, flNotFile(CacheFile));
1572       if (SrcCacheFile.empty() == false)
1573          CreateDirectory(dir, flNotFile(SrcCacheFile));
1574    }
1575
1576    if (Progress != NULL)
1577       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1578
1579    bool pkgcache_fine = false;
1580    bool srcpkgcache_fine = false;
1581    bool volatile_fine = List.GetVolatileFiles().empty();
1582
1583    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
1584    {
1585       if (Debug == true)
1586          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1587       pkgcache_fine = true;
1588       srcpkgcache_fine = true;
1589    }
1590    if (pkgcache_fine == false)
1591    {
1592       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1593       {
1594          if (Debug == true)
1595             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1596          srcpkgcache_fine = true;
1597       }
1598    }
1599
1600    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1601    {
1602       if (Progress != NULL)
1603          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1604       return true;
1605    }
1606
1607    bool Writeable = false;
1608    if (srcpkgcache_fine == false || pkgcache_fine == false)
1609    {
1610       if (CacheFile.empty() == false)
1611          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1612       else if (SrcCacheFile.empty() == false)
1613          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1614
1615       if (Debug == true)
1616          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1617
1618       if (Writeable == false && AllowMem == false)
1619       {
1620          if (CacheFile.empty() == false)
1621             return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1622          else if (SrcCacheFile.empty() == false)
1623             return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
1624          else
1625             return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
1626       }
1627    }
1628
1629    // At this point we know we need to construct something, so get storage ready
1630    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1631    if (Debug == true)
1632       std::clog << "Open memory Map (not filebased)" << std::endl;
1633
1634    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1635    map_filesize_t CurrentSize = 0;
1636    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1637    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1638    if (srcpkgcache_fine == true && pkgcache_fine == false)
1639    {
1640       if (Debug == true)
1641          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1642       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1643          return false;
1644       srcpkgcache_fine = true;
1645       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1646    }
1647    else if (srcpkgcache_fine == false)
1648    {
1649       if (Debug == true)
1650          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1651       Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1652
1653       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1654       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1655                Files.end(),Files.end()) == false)
1656          return false;
1657
1658       if (Writeable == true && SrcCacheFile.empty() == false)
1659          if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
1660             return false;
1661    }
1662
1663    if (pkgcache_fine == false)
1664    {
1665       if (Debug == true)
1666          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1667       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1668                Files.begin(), Files.end()) == false)
1669          return false;
1670
1671       if (Writeable == true && CacheFile.empty() == false)
1672          if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
1673             return false;
1674    }
1675
1676    if (Debug == true)
1677       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1678
1679    if (volatile_fine == false)
1680    {
1681       if (Gen == nullptr)
1682       {
1683          if (Debug == true)
1684             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1685          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1686             return false;
1687       }
1688
1689       Files = List.GetVolatileFiles();
1690       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1691                Files.begin(), Files.end()) == false)
1692          return false;
1693    }
1694
1695    if (OutMap != nullptr)
1696       *OutMap = Map.UnGuard();
1697
1698    if (Debug == true)
1699       std::clog << "Everything is ready for shipping" << std::endl;
1700    return true;
1701 }
1702                                                                         /*}}}*/
1703 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1704 // ---------------------------------------------------------------------
1705 /* */
1706 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1707    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1708 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1709 {
1710    std::vector<pkgIndexFile *> Files;
1711    if (_system->AddStatusFiles(Files) == false)
1712       return false;
1713
1714    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1715    map_filesize_t CurrentSize = 0;
1716    map_filesize_t TotalSize = 0;
1717
1718    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1719
1720    // Build the status cache
1721    if (Progress != NULL)
1722       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1723    pkgCacheGenerator Gen(Map.Get(),Progress);
1724    if (_error->PendingError() == true)
1725       return false;
1726    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1727                   Files.begin(), Files.end()) == false)
1728       return false;
1729
1730    if (_error->PendingError() == true)
1731       return false;
1732    *OutMap = Map.UnGuard();
1733
1734    return true;
1735 }
1736                                                                         /*}}}*/
1737 // IsDuplicateDescription                                               /*{{{*/
1738 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1739                             MD5SumValue const &CurMd5, std::string const &CurLang)
1740 {
1741    // Descriptions in the same link-list have all the same md5
1742    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1743       return false;
1744    for (; Desc.end() == false; ++Desc)
1745       if (Desc.LanguageCode() == CurLang)
1746          return true;
1747    return false;
1748 }
1749                                                                         /*}}}*/
1750
1751 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1752 pkgCacheListParser::~pkgCacheListParser() {}