apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <memory>
  40 #include <algorithm>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43
  44 #include <apti18n.h>
  45
  46 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  47 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  48 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  49
  50 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  51                             MD5SumValue const &CurMd5, std::string const &CurLang);
  52
  53 using std::string;
  54
  55 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* We set the dirty flag and make sure that is written to the disk */
  58 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  59                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  60                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  61 {
  62    if (_error->PendingError() == true)
  63       return;
  64
  65    if (Map.Size() == 0)
  66    {
  67       // Setup the map interface..
  68       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  69       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       Cache.ReMap();
 106    }
 107    else
 108    {
 109       // Map directly from the existing file
 110       Cache.ReMap();
 111       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 112       if (Cache.VS != _system->VS)
 113       {
 114          _error->Error(_("Cache has an incompatible versioning system"));
 115          return;
 116       }
 117    }
 118
 119    Cache.HeaderP->Dirty = true;
 120    Map.Sync(0,sizeof(pkgCache::Header));
 121 }
 122                                                                         /*}}}*/
 123 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 124 // ---------------------------------------------------------------------
 125 /* We sync the data then unset the dirty flag in two steps so as to
 126    advoid a problem during a crash */
 127 pkgCacheGenerator::~pkgCacheGenerator()
 128 {
 129    if (_error->PendingError() == true)
 130       return;
 131    if (Map.Sync() == false)
 132       return;
 133
 134    Cache.HeaderP->Dirty = false;
 135    Cache.HeaderP->CacheFileSize = Map.Size();
 136    Map.Sync(0,sizeof(pkgCache::Header));
 137 }
 138                                                                         /*}}}*/
 139 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 140    if (oldMap == newMap)
 141       return;
 142
 143    if (_config->FindB("Debug::pkgCacheGen", false))
 144       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 145
 146    Cache.ReMap(false);
 147
 148    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 149    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 150
 151    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 152         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 153       (*i)->ReMap(oldMap, newMap);
 154    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 155         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 156       (*i)->ReMap(oldMap, newMap);
 157    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 158         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 159       (*i)->ReMap(oldMap, newMap);
 160    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 161         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 162       (*i)->ReMap(oldMap, newMap);
 163    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 164         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 165       (*i)->ReMap(oldMap, newMap);
 166    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 168       (*i)->ReMap(oldMap, newMap);
 169    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 170         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 171       (*i)->ReMap(oldMap, newMap);
 172    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 173         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 174       (*i)->ReMap(oldMap, newMap);
 175 }                                                                       /*}}}*/
 176 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 177 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 178                                         const unsigned long &Len) {
 179    void const * const oldMap = Map.Data();
 180    map_stringitem_t const index = Map.WriteString(String, Len);
 181    if (index != 0)
 182       ReMap(oldMap, Map.Data());
 183    return index;
 184 }
 185                                                                         /*}}}*/
 186 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 187 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 188    void const * const oldMap = Map.Data();
 189    map_stringitem_t const index = Map.WriteString(String);
 190    if (index != 0)
 191       ReMap(oldMap, Map.Data());
 192    return index;
 193 }
 194                                                                         /*}}}*/
 195 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 196    void const * const oldMap = Map.Data();
 197    map_pointer_t const index = Map.Allocate(size);
 198    if (index != 0)
 199       ReMap(oldMap, Map.Data());
 200    return index;
 201 }
 202                                                                         /*}}}*/
 203 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 204 // ---------------------------------------------------------------------
 205 /* This provides the generation of the entries in the cache. Each loop
 206    goes through a single package record from the underlying parse engine. */
 207 bool pkgCacheGenerator::MergeList(ListParser &List,
 208                                   pkgCache::VerIterator *OutVer)
 209 {
 210    List.Owner = this;
 211
 212    unsigned int Counter = 0;
 213    while (List.Step() == true)
 214    {
 215       string const PackageName = List.Package();
 216       if (PackageName.empty() == true)
 217          return false;
 218
 219       Counter++;
 220       if (Counter % 100 == 0 && Progress != 0)
 221          Progress->Progress(List.Offset());
 222
 223       string Arch = List.Architecture();
 224       string const Version = List.Version();
 225       if (Version.empty() == true && Arch.empty() == true)
 226       {
 227          // package descriptions
 228          if (MergeListGroup(List, PackageName) == false)
 229             return false;
 230          continue;
 231       }
 232
 233       // Get a pointer to the package structure
 234       pkgCache::PkgIterator Pkg;
 235       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 236       if (NewPackage(Pkg, PackageName, Arch) == false)
 237          // TRANSLATOR: The first placeholder is a package name,
 238          // the other two should be copied verbatim as they include debug info
 239          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 240                               PackageName.c_str(), "NewPackage", 1);
 241
 242
 243       if (Version.empty() == true)
 244       {
 245          if (MergeListPackage(List, Pkg) == false)
 246             return false;
 247       }
 248       else
 249       {
 250          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 251             return false;
 252       }
 253
 254       if (OutVer != 0)
 255          return true;
 256    }
 257
 258    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 259       return _error->Error(_("Wow, you exceeded the number of package "
 260                              "names this APT is capable of."));
 261    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 262       return _error->Error(_("Wow, you exceeded the number of versions "
 263                              "this APT is capable of."));
 264    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 265       return _error->Error(_("Wow, you exceeded the number of descriptions "
 266                              "this APT is capable of."));
 267    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 268       return _error->Error(_("Wow, you exceeded the number of dependencies "
 269                              "this APT is capable of."));
 270
 271    return true;
 272 }
 273 // CacheGenerator::MergeListGroup                                       /*{{{*/
 274 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 275 {
 276    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 277    // a group has no data on it's own, only packages have it but these
 278    // stanzas like this come from Translation- files to add descriptions,
 279    // but without a version we don't need a description for it…
 280    if (Grp.end() == true)
 281       return true;
 282    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 283
 284    pkgCache::PkgIterator Pkg;
 285    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 286    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 287       if (MergeListPackage(List, Pkg) == false)
 288          return false;
 289
 290    return true;
 291 }
 292                                                                         /*}}}*/
 293 // CacheGenerator::MergeListPackage                                     /*{{{*/
 294 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 295 {
 296    // we first process the package, then the descriptions
 297    // (for deb this package processing is in fact a no-op)
 298    pkgCache::VerIterator Ver(Cache);
 299    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 300    if (List.UsePackage(Pkg, Ver) == false)
 301       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 302                            Pkg.Name(), "UsePackage", 1);
 303
 304    // Find the right version to write the description
 305    MD5SumValue CurMd5 = List.Description_md5();
 306    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 307       return true;
 308    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 309    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 310    {
 311       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 312
 313       // a version can only have one md5 describing it
 314       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 315          continue;
 316
 317       map_stringitem_t md5idx = VerDesc->md5sum;
 318       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 319       {
 320          // don't add a new description if we have one for the given
 321          // md5 && language
 322          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 323             continue;
 324
 325          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 326       }
 327
 328       // we can stop here as all "same" versions will share the description
 329       break;
 330    }
 331
 332    return true;
 333 }
 334                                                                         /*}}}*/
 335 // CacheGenerator::MergeListVersion                                     /*{{{*/
 336 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 337                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 338 {
 339    pkgCache::VerIterator Ver = Pkg.VersionList();
 340    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 341    map_pointer_t *LastVer = &Pkg->VersionList;
 342    void const * oldMap = Map.Data();
 343
 344    unsigned short const Hash = List.VersionHash();
 345    if (Ver.end() == false)
 346    {
 347       /* We know the list is sorted so we use that fact in the search.
 348          Insertion of new versions is done with correct sorting */
 349       int Res = 1;
 350       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 351       {
 352          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 353          // Version is higher as current version - insert here
 354          if (Res > 0)
 355             break;
 356          // Versionstrings are equal - is hash also equal?
 357          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 358             break;
 359          // proceed with the next till we have either the right
 360          // or we found another version (which will be lower)
 361       }
 362
 363       /* We already have a version for this item, record that we saw it */
 364       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 365       {
 366          if (List.UsePackage(Pkg,Ver) == false)
 367             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 368                                  Pkg.Name(), "UsePackage", 2);
 369
 370          if (NewFileVer(Ver,List) == false)
 371             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 372                                  Pkg.Name(), "NewFileVer", 1);
 373
 374          // Read only a single record and return
 375          if (OutVer != 0)
 376          {
 377             *OutVer = Ver;
 378             return true;
 379          }
 380
 381          return true;
 382       }
 383    }
 384
 385    // Add a new version
 386    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 387    if (verindex == 0 && _error->PendingError())
 388       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 389                            Pkg.Name(), "NewVersion", 1);
 390
 391    if (oldMap != Map.Data())
 392          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 393    *LastVer = verindex;
 394
 395    if (unlikely(List.NewVersion(Ver) == false))
 396       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 397                            Pkg.Name(), "NewVersion", 2);
 398
 399    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 400       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 401                            Pkg.Name(), "UsePackage", 3);
 402
 403    if (unlikely(NewFileVer(Ver,List) == false))
 404       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 405                            Pkg.Name(), "NewFileVer", 2);
 406
 407    pkgCache::GrpIterator Grp = Pkg.Group();
 408    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 409
 410    /* If it is the first version of this package we need to add implicit
 411       Multi-Arch dependencies to all other package versions in the group now -
 412       otherwise we just add them for this new version */
 413    if (Pkg.VersionList()->NextVer == 0)
 414    {
 415       pkgCache::PkgIterator P = Grp.PackageList();
 416       Dynamic<pkgCache::PkgIterator> DynP(P);
 417       for (; P.end() != true; P = Grp.NextPkg(P))
 418       {
 419          if (P->ID == Pkg->ID)
 420             continue;
 421          pkgCache::VerIterator V = P.VersionList();
 422          Dynamic<pkgCache::VerIterator> DynV(V);
 423          for (; V.end() != true; ++V)
 424             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 425                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 426                                     Pkg.Name(), "AddImplicitDepends", 1);
 427       }
 428    }
 429    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 430       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 431                            Pkg.Name(), "AddImplicitDepends", 2);
 432
 433    // Read only a single record and return
 434    if (OutVer != 0)
 435    {
 436       *OutVer = Ver;
 437       return true;
 438    }
 439
 440    /* Record the Description(s) based on their master md5sum */
 441    MD5SumValue CurMd5 = List.Description_md5();
 442    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 443       return true;
 444
 445    /* Before we add a new description we first search in the group for
 446       a version with a description of the same MD5 - if so we reuse this
 447       description group instead of creating our own for this version */
 448    for (pkgCache::PkgIterator P = Grp.PackageList();
 449         P.end() == false; P = Grp.NextPkg(P))
 450    {
 451       for (pkgCache::VerIterator V = P.VersionList();
 452            V.end() == false; ++V)
 453       {
 454          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 455             continue;
 456          Ver->DescriptionList = V->DescriptionList;
 457       }
 458    }
 459
 460    // We haven't found reusable descriptions, so add the first description(s)
 461    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 462    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 463    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 464       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 465          return false;
 466    return true;
 467 }
 468                                                                         /*}}}*/
 469 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 470 {
 471    pkgCache::DescIterator Desc;
 472    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 473
 474    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 475    if (unlikely(descindex == 0 && _error->PendingError()))
 476       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 477             Ver.ParentPkg().Name(), "NewDescription", 1);
 478
 479    md5idx = Desc->md5sum;
 480    Desc->ParentPkg = Ver.ParentPkg().Index();
 481
 482    // we add at the end, so that the start is constant as we need
 483    // that to be able to efficiently share these lists
 484    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 485    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 486    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 487    *LastNextDesc = descindex;
 488
 489    if (NewFileDesc(Desc,List) == false)
 490       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 491             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 492
 493    return true;
 494 }
 495                                                                         /*}}}*/
 496                                                                         /*}}}*/
 497 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 498 // ---------------------------------------------------------------------
 499 /* This creates a new group structure and adds it to the hash table */
 500 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 501 {
 502    Grp = Cache.FindGrp(Name);
 503    if (Grp.end() == false)
 504       return true;
 505
 506    // Get a structure
 507    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 508    if (unlikely(Group == 0))
 509       return false;
 510
 511    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 512    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 513    if (unlikely(idxName == 0))
 514       return false;
 515    Grp->Name = idxName;
 516
 517    // Insert it into the hash table
 518    unsigned long const Hash = Cache.Hash(Name);
 519    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 520    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 521       insertAt = &(Cache.GrpP + *insertAt)->Next;
 522    Grp->Next = *insertAt;
 523    *insertAt = Group;
 524
 525    Grp->ID = Cache.HeaderP->GroupCount++;
 526    return true;
 527 }
 528                                                                         /*}}}*/
 529 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 530 // ---------------------------------------------------------------------
 531 /* This creates a new package structure and adds it to the hash table */
 532 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 533                                         const string &Arch) {
 534    pkgCache::GrpIterator Grp;
 535    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 536    if (unlikely(NewGroup(Grp, Name) == false))
 537       return false;
 538
 539    Pkg = Grp.FindPkg(Arch);
 540       if (Pkg.end() == false)
 541          return true;
 542
 543    // Get a structure
 544    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 545    if (unlikely(Package == 0))
 546       return false;
 547    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 548
 549    // Set the name, arch and the ID
 550    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 551    Pkg->Group = Grp.Index();
 552    // all is mapped to the native architecture
 553    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 554    if (unlikely(idxArch == 0))
 555       return false;
 556    Pkg->Arch = idxArch;
 557    Pkg->ID = Cache.HeaderP->PackageCount++;
 558
 559    // Insert the package into our package list
 560    if (Grp->FirstPackage == 0) // the group is new
 561    {
 562       Grp->FirstPackage = Package;
 563       // Insert it into the hash table
 564       map_id_t const Hash = Cache.Hash(Name);
 565       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 566       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 567          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 568       Pkg->NextPackage = *insertAt;
 569       *insertAt = Package;
 570    }
 571    else // Group the Packages together
 572    {
 573       // but first get implicit provides done
 574       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 575       {
 576          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 577          if (M.end() == false)
 578             for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 579             {
 580                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 581                   continue;
 582                pkgCache::VerIterator Ver = Prv.OwnerVer();
 583                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 584                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 585                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 586                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 587                      return false;
 588             }
 589
 590          for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 591             for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
 592                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 593                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 594                      return false;
 595       }
 596       // and negative dependencies, don't forget negative dependencies
 597       {
 598          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 599          if (M.end() == false)
 600             for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 601             {
 602                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 603                   continue;
 604                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 605                      Dep->Type != pkgCache::Dep::Replaces)
 606                   continue;
 607                pkgCache::VerIterator Ver = Dep.ParentVer();
 608                map_pointer_t * unused = NULL;
 609                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 610                   return false;
 611             }
 612       }
 613
 614       // this package is the new last package
 615       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 616       Pkg->NextPackage = LastPkg->NextPackage;
 617       LastPkg->NextPackage = Package;
 618    }
 619    Grp->LastPackage = Package;
 620    return true;
 621 }
 622                                                                         /*}}}*/
 623 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 624 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 625                                            pkgCache::PkgIterator &P,
 626                                            pkgCache::VerIterator &V)
 627 {
 628    // copy P.Arch() into a string here as a cache remap
 629    // in NewDepends() later may alter the pointer location
 630    string Arch = P.Arch() == NULL ? "" : P.Arch();
 631    map_pointer_t *OldDepLast = NULL;
 632    /* MultiArch handling introduces a lot of implicit Dependencies:
 633       - MultiArch: same → Co-Installable if they have the same version
 634       - All others conflict with all other group members */
 635    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 636    pkgCache::PkgIterator D = G.PackageList();
 637    Dynamic<pkgCache::PkgIterator> DynD(D);
 638    map_stringitem_t const VerStrIdx = V->VerStr;
 639    for (; D.end() != true; D = G.NextPkg(D))
 640    {
 641       if (Arch == D.Arch() || D->VersionList == 0)
 642          continue;
 643       /* We allow only one installed arch at the time
 644          per group, therefore each group member conflicts
 645          with all other group members */
 646       if (coInstall == true)
 647       {
 648          // Replaces: ${self}:other ( << ${binary:Version})
 649          NewDepends(D, V, VerStrIdx,
 650                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 651                     OldDepLast);
 652          // Breaks: ${self}:other (!= ${binary:Version})
 653          NewDepends(D, V, VerStrIdx,
 654                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 655                     OldDepLast);
 656       } else {
 657          // Conflicts: ${self}:other
 658          NewDepends(D, V, 0,
 659                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 660                     OldDepLast);
 661       }
 662    }
 663    return true;
 664 }
 665 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 666                                            pkgCache::PkgIterator &D)
 667 {
 668    /* MultiArch handling introduces a lot of implicit Dependencies:
 669       - MultiArch: same → Co-Installable if they have the same version
 670       - All others conflict with all other group members */
 671    map_pointer_t *OldDepLast = NULL;
 672    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 673    if (coInstall == true)
 674    {
 675       map_stringitem_t const VerStrIdx = V->VerStr;
 676       // Replaces: ${self}:other ( << ${binary:Version})
 677       NewDepends(D, V, VerStrIdx,
 678                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 679                  OldDepLast);
 680       // Breaks: ${self}:other (!= ${binary:Version})
 681       NewDepends(D, V, VerStrIdx,
 682                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 683                  OldDepLast);
 684    } else {
 685       // Conflicts: ${self}:other
 686       NewDepends(D, V, 0,
 687                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 688                  OldDepLast);
 689    }
 690    return true;
 691 }
 692
 693                                                                         /*}}}*/
 694 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 695 // ---------------------------------------------------------------------
 696 /* */
 697 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 698                                    ListParser &List)
 699 {
 700    if (CurrentFile == 0)
 701       return true;
 702
 703    // Get a structure
 704    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 705    if (VerFile == 0)
 706       return false;
 707
 708    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 709    VF->File = CurrentFile - Cache.PkgFileP;
 710
 711    // Link it to the end of the list
 712    map_pointer_t *Last = &Ver->FileList;
 713    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 714       Last = &V->NextFile;
 715    VF->NextFile = *Last;
 716    *Last = VF.Index();
 717
 718    VF->Offset = List.Offset();
 719    VF->Size = List.Size();
 720    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 721       Cache.HeaderP->MaxVerFileSize = VF->Size;
 722    Cache.HeaderP->VerFileCount++;
 723
 724    return true;
 725 }
 726                                                                         /*}}}*/
 727 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 728 // ---------------------------------------------------------------------
 729 /* This puts a version structure in the linked list */
 730 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 731                                             const string &VerStr,
 732                                             map_pointer_t const ParentPkg,
 733                                             unsigned short const Hash,
 734                                             map_pointer_t const Next)
 735 {
 736    // Get a structure
 737    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 738    if (Version == 0)
 739       return 0;
 740
 741    // Fill it in
 742    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 743    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 744    Ver->NextVer = Next;
 745    Ver->ParentPkg = ParentPkg;
 746    Ver->Hash = Hash;
 747    Ver->ID = Cache.HeaderP->VersionCount++;
 748
 749    // try to find the version string in the group for reuse
 750    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 751    pkgCache::GrpIterator Grp = Pkg.Group();
 752    if (Pkg.end() == false && Grp.end() == false)
 753    {
 754       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 755       {
 756          if (Pkg == P)
 757             continue;
 758          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 759          {
 760             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 761             if (cmp == 0)
 762             {
 763                Ver->VerStr = V->VerStr;
 764                return Version;
 765             }
 766             else if (cmp < 0)
 767                break;
 768          }
 769       }
 770    }
 771    // haven't found the version string, so create
 772    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 773    if (unlikely(idxVerStr == 0))
 774       return 0;
 775    Ver->VerStr = idxVerStr;
 776    return Version;
 777 }
 778                                                                         /*}}}*/
 779 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 780 // ---------------------------------------------------------------------
 781 /* */
 782 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 783                                    ListParser &List)
 784 {
 785    if (CurrentFile == 0)
 786       return true;
 787
 788    // Get a structure
 789    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 790    if (DescFile == 0)
 791       return false;
 792
 793    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 794    DF->File = CurrentFile - Cache.PkgFileP;
 795
 796    // Link it to the end of the list
 797    map_pointer_t *Last = &Desc->FileList;
 798    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 799       Last = &D->NextFile;
 800
 801    DF->NextFile = *Last;
 802    *Last = DF.Index();
 803
 804    DF->Offset = List.Offset();
 805    DF->Size = List.Size();
 806    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 807       Cache.HeaderP->MaxDescFileSize = DF->Size;
 808    Cache.HeaderP->DescFileCount++;
 809
 810    return true;
 811 }
 812                                                                         /*}}}*/
 813 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 814 // ---------------------------------------------------------------------
 815 /* This puts a description structure in the linked list */
 816 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 817                                             const string &Lang,
 818                                             const MD5SumValue &md5sum,
 819                                             map_stringitem_t const idxmd5str)
 820 {
 821    // Get a structure
 822    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 823    if (Description == 0)
 824       return 0;
 825
 826    // Fill it in
 827    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 828    Desc->ID = Cache.HeaderP->DescriptionCount++;
 829    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 830    if (unlikely(idxlanguage_code == 0))
 831       return 0;
 832    Desc->language_code = idxlanguage_code;
 833
 834    if (idxmd5str != 0)
 835       Desc->md5sum = idxmd5str;
 836    else
 837    {
 838       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 839       if (unlikely(idxmd5sum == 0))
 840          return 0;
 841       Desc->md5sum = idxmd5sum;
 842    }
 843
 844    return Description;
 845 }
 846                                                                         /*}}}*/
 847 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This creates a dependency element in the tree. It is linked to the
 850    version and to the package that it is pointing to. */
 851 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 852                                    pkgCache::VerIterator &Ver,
 853                                    map_pointer_t const Version,
 854                                    uint8_t const Op,
 855                                    uint8_t const Type,
 856                                    map_pointer_t* &OldDepLast)
 857 {
 858    void const * const oldMap = Map.Data();
 859    // Get a structure
 860    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 861    if (unlikely(Dependency == 0))
 862       return false;
 863
 864    bool isDuplicate = false;
 865    map_pointer_t DependencyData = 0;
 866    map_pointer_t PreviousData = 0;
 867    if (Pkg->RevDepends != 0)
 868    {
 869       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 870       DependencyData = L->DependencyData;
 871       do {
 872          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 873          if (Version > D->Version)
 874             break;
 875          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 876          {
 877             isDuplicate = true;
 878             break;
 879          }
 880          PreviousData = DependencyData;
 881          DependencyData = D->NextData;
 882       } while (DependencyData != 0);
 883    }
 884
 885    if (isDuplicate == false)
 886    {
 887       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 888       if (unlikely(DependencyData == 0))
 889         return false;
 890    }
 891
 892    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 893    Link->ParentVer = Ver.Index();
 894    Link->DependencyData = DependencyData;
 895    Link->ID = Cache.HeaderP->DependsCount++;
 896
 897    pkgCache::DepIterator Dep(Cache, Link);
 898    if (isDuplicate == false)
 899    {
 900       Dep->Type = Type;
 901       Dep->CompareOp = Op;
 902       Dep->Version = Version;
 903       Dep->Package = Pkg.Index();
 904       ++Cache.HeaderP->DependsDataCount;
 905       if (PreviousData != 0)
 906       {
 907          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 908          Dep->NextData = D->NextData;
 909          D->NextData = DependencyData;
 910       }
 911       else if (Pkg->RevDepends != 0)
 912       {
 913          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 914          Dep->NextData = D->DependencyData;
 915       }
 916    }
 917
 918    if (isDuplicate == true || PreviousData != 0)
 919    {
 920       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 921       Link->NextRevDepends = L->NextRevDepends;
 922       L->NextRevDepends = Dependency;
 923    }
 924    else
 925    {
 926       Link->NextRevDepends = Pkg->RevDepends;
 927       Pkg->RevDepends = Dependency;
 928    }
 929
 930
 931    // Do we know where to link the Dependency to?
 932    if (OldDepLast == NULL)
 933    {
 934       OldDepLast = &Ver->DependsList;
 935       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 936          OldDepLast = &D->NextDepends;
 937    } else if (oldMap != Map.Data())
 938       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 939
 940    Dep->NextDepends = *OldDepLast;
 941    *OldDepLast = Dependency;
 942    OldDepLast = &Dep->NextDepends;
 943    return true;
 944 }
 945                                                                         /*}}}*/
 946 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 947 // ---------------------------------------------------------------------
 948 /* This creates a Group and the Package to link this dependency to if
 949    needed and handles also the caching of the old endpoint */
 950 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
 951                                                const string &PackageName,
 952                                                const string &Arch,
 953                                                const string &Version,
 954                                                uint8_t const Op,
 955                                                uint8_t const Type)
 956 {
 957    pkgCache::GrpIterator Grp;
 958    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 959    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 960       return false;
 961
 962    map_stringitem_t idxVersion = 0;
 963    if (Version.empty() == false)
 964    {
 965       int const CmpOp = Op & 0x0F;
 966       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 967       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 968          idxVersion = Ver->VerStr;
 969
 970       if (idxVersion == 0)
 971       {
 972          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
 973          if (unlikely(idxVersion == 0))
 974             return false;
 975       }
 976    }
 977
 978    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
 979          Type == pkgCache::Dep::Conflicts ||
 980          Type == pkgCache::Dep::Replaces);
 981
 982    pkgCache::PkgIterator Pkg;
 983    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 984    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
 985    {
 986       // Locate the target package
 987       Pkg = Grp.FindPkg(Arch);
 988       if (Pkg.end() == true) {
 989          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 990             return false;
 991       }
 992
 993       /* Caching the old end point speeds up generation substantially */
 994       if (OldDepVer != Ver) {
 995          OldDepLast = NULL;
 996          OldDepVer = Ver;
 997       }
 998
 999       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1000    }
1001    else
1002    {
1003       /* Caching the old end point speeds up generation substantially */
1004       if (OldDepVer != Ver) {
1005          OldDepLast = NULL;
1006          OldDepVer = Ver;
1007       }
1008
1009       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1010       {
1011          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1012                return false;
1013       }
1014    }
1015    return true;
1016 }
1017                                                                         /*}}}*/
1018 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1019 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1020                                                 const string &PkgName,
1021                                                 const string &PkgArch,
1022                                                 const string &Version,
1023                                                 uint8_t const Flags)
1024 {
1025    pkgCache const &Cache = Owner->Cache;
1026
1027    // We do not add self referencing provides
1028    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1029         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1030       return true;
1031
1032    // Locate the target package
1033    pkgCache::PkgIterator Pkg;
1034    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1035    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1036       return false;
1037
1038    map_stringitem_t idxProvideVersion = 0;
1039    if (Version.empty() == false) {
1040       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1041       if (unlikely(idxProvideVersion == 0))
1042          return false;
1043    }
1044    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1045 }
1046 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1047                                     pkgCache::PkgIterator &Pkg,
1048                                     map_pointer_t const ProvideVersion,
1049                                     uint8_t const Flags)
1050 {
1051    // Get a structure
1052    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1053    if (unlikely(Provides == 0))
1054       return false;
1055    ++Cache.HeaderP->ProvidesCount;
1056
1057    // Fill it in
1058    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1059    Prv->Version = Ver.Index();
1060    Prv->ProvideVersion = ProvideVersion;
1061    Prv->Flags = Flags;
1062    Prv->NextPkgProv = Ver->ProvidesList;
1063    Ver->ProvidesList = Prv.Index();
1064
1065    // Link it to the package
1066    Prv->ParentPkg = Pkg.Index();
1067    Prv->NextProvides = Pkg->ProvidesList;
1068    Pkg->ProvidesList = Prv.Index();
1069    return true;
1070 }
1071                                                                         /*}}}*/
1072 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1073 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1074                                 string const &Version, uint8_t const Flags) {
1075    pkgCache &Cache = Owner->Cache;
1076    pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1077    if (Grp.end() == true)
1078       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1079    else
1080    {
1081       map_stringitem_t idxProvideVersion = 0;
1082       if (Version.empty() == false) {
1083          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1084          if (unlikely(idxProvideVersion == 0))
1085             return false;
1086       }
1087
1088       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1089       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1090       pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1091       for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1092       {
1093          if (isImplicit && OwnerPkg == Pkg)
1094             continue;
1095          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1096             continue;
1097          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1098             return false;
1099       }
1100    }
1101    return true;
1102 }
1103                                                                         /*}}}*/
1104 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1105       pkgCache::VerIterator const &Ver)
1106 {
1107    return Hash == Ver->Hash;
1108 }
1109                                                                         /*}}}*/
1110 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1111 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1112                                    unsigned long Flags)
1113 {
1114    if (File.empty() && Site.empty())
1115    {
1116       CurrentRlsFile = NULL;
1117       return true;
1118    }
1119
1120    // Get some space for the structure
1121    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1122    if (unlikely(idxFile == 0))
1123       return false;
1124    CurrentRlsFile = Cache.RlsFileP + idxFile;
1125
1126    // Fill it in
1127    map_stringitem_t const idxFileName = WriteStringInMap(File);
1128    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1129    if (unlikely(idxFileName == 0 || idxSite == 0))
1130       return false;
1131    CurrentRlsFile->FileName = idxFileName;
1132    CurrentRlsFile->Site = idxSite;
1133    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1134    CurrentRlsFile->Flags = Flags;
1135    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1136    RlsFileName = File;
1137    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1138    Cache.HeaderP->ReleaseFileCount++;
1139
1140    return true;
1141 }
1142                                                                         /*}}}*/
1143 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1144 // ---------------------------------------------------------------------
1145 /* This is used to select which file is to be associated with all newly
1146    added versions. The caller is responsible for setting the IMS fields. */
1147 bool pkgCacheGenerator::SelectFile(std::string const &File,
1148                                    pkgIndexFile const &Index,
1149                                    std::string const &Architecture,
1150                                    std::string const &Component,
1151                                    unsigned long const Flags)
1152 {
1153    // Get some space for the structure
1154    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1155    if (unlikely(idxFile == 0))
1156       return false;
1157    CurrentFile = Cache.PkgFileP + idxFile;
1158
1159    // Fill it in
1160    map_stringitem_t const idxFileName = WriteStringInMap(File);
1161    if (unlikely(idxFileName == 0))
1162       return false;
1163    CurrentFile->FileName = idxFileName;
1164    CurrentFile->NextFile = Cache.HeaderP->FileList;
1165    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1166    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1167    if (unlikely(idxIndexType == 0))
1168       return false;
1169    CurrentFile->IndexType = idxIndexType;
1170    if (Architecture.empty())
1171       CurrentFile->Architecture = 0;
1172    else
1173    {
1174       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1175       if (unlikely(arch == 0))
1176          return false;
1177       CurrentFile->Architecture = arch;
1178    }
1179    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1180    if (unlikely(component == 0))
1181       return false;
1182    CurrentFile->Component = component;
1183    CurrentFile->Flags = Flags;
1184    if (CurrentRlsFile != NULL)
1185       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1186    else
1187       CurrentFile->Release = 0;
1188    PkgFileName = File;
1189    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1190    Cache.HeaderP->PackageFileCount++;
1191
1192    if (Progress != 0)
1193       Progress->SubProgress(Index.Size());
1194    return true;
1195 }
1196                                                                         /*}}}*/
1197 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1198 // ---------------------------------------------------------------------
1199 /* This is used to create handles to strings. Given the same text it
1200    always returns the same number */
1201 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1202                                                  unsigned int Size)
1203 {
1204    std::string const key(S, Size);
1205
1206    std::map<std::string,map_stringitem_t> * strings;
1207    switch(type) {
1208       case MIXED: strings = &strMixed; break;
1209       case PKGNAME: strings = &strPkgNames; break;
1210       case VERSIONNUMBER: strings = &strVersions; break;
1211       case SECTION: strings = &strSections; break;
1212       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1213    }
1214
1215    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1216    if (item != strings->end())
1217       return item->second;
1218
1219    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1220    strings->insert(std::make_pair(key, idxString));
1221    return idxString;
1222 }
1223                                                                         /*}}}*/
1224 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1225 // ---------------------------------------------------------------------
1226 /* This just verifies that each file in the list of index files exists,
1227    has matching attributes with the cache and the cache does not have
1228    any extra files. */
1229 static bool CheckValidity(const string &CacheFile,
1230                           pkgSourceList &List,
1231                           FileIterator const Start,
1232                           FileIterator const End,
1233                           MMap **OutMap = 0)
1234 {
1235    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1236    // No file, certainly invalid
1237    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1238    {
1239       if (Debug == true)
1240          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1241       return false;
1242    }
1243
1244    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1245    {
1246       if (Debug == true)
1247          std::clog << "sources.list is newer than the cache" << std::endl;
1248       return false;
1249    }
1250
1251    // Map it
1252    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1253    SPtr<MMap> Map = new MMap(CacheF,0);
1254    pkgCache Cache(Map);
1255    if (_error->PendingError() == true || Map->Size() == 0)
1256    {
1257       if (Debug == true)
1258          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1259       _error->Discard();
1260       return false;
1261    }
1262
1263    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1264    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1265    std::vector<pkgIndexFile *> Files;
1266    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1267    {
1268       if (Debug == true)
1269          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1270       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1271       if (RlsFile.end() == true)
1272       {
1273          if (Debug == true)
1274             std::clog << "FindInCache returned end-Pointer" << std::endl;
1275          return false;
1276       }
1277
1278       RlsVisited[RlsFile->ID] = true;
1279       if (Debug == true)
1280          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1281
1282       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1283       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1284             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1285    }
1286    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1287       if (RlsVisited[I] == false)
1288       {
1289          if (Debug == true)
1290             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1291          return false;
1292       }
1293
1294    std::copy(Start, End, std::back_inserter(Files));
1295
1296    /* Now we check every index file, see if it is in the cache,
1297       verify the IMS data and check that it is on the disk too.. */
1298    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1299    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1300    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1301    {
1302       if (Debug == true)
1303          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1304       if ((*PkgFile)->Exists() == false)
1305       {
1306          if (Debug == true)
1307             std::clog << "file doesn't exist" << std::endl;
1308          continue;
1309       }
1310
1311       // FindInCache is also expected to do an IMS check.
1312       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1313       if (File.end() == true)
1314       {
1315          if (Debug == true)
1316             std::clog << "FindInCache returned end-Pointer" << std::endl;
1317          return false;
1318       }
1319
1320       Visited[File->ID] = true;
1321       if (Debug == true)
1322          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1323    }
1324
1325    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1326       if (Visited[I] == false)
1327       {
1328          if (Debug == true)
1329             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1330          return false;
1331       }
1332
1333    if (_error->PendingError() == true)
1334    {
1335       if (Debug == true)
1336       {
1337          std::clog << "Validity failed because of pending errors:" << std::endl;
1338          _error->DumpErrors();
1339       }
1340       _error->Discard();
1341       return false;
1342    }
1343
1344    if (OutMap != 0)
1345       *OutMap = Map.UnGuard();
1346    return true;
1347 }
1348                                                                         /*}}}*/
1349 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1350 // ---------------------------------------------------------------------
1351 /* Size is kind of an abstract notion that is only used for the progress
1352    meter */
1353 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1354 {
1355    map_filesize_t TotalSize = 0;
1356    if (List !=  NULL)
1357    {
1358       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1359       {
1360          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1361          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1362             if ((*j)->HasPackages() == true)
1363                TotalSize += (*j)->Size();
1364       }
1365    }
1366
1367    for (; Start < End; ++Start)
1368    {
1369       if ((*Start)->HasPackages() == false)
1370          continue;
1371       TotalSize += (*Start)->Size();
1372    }
1373    return TotalSize;
1374 }
1375                                                                         /*}}}*/
1376 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1377 static bool BuildCache(pkgCacheGenerator &Gen,
1378                        OpProgress * const Progress,
1379                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1380                        pkgSourceList const * const List,
1381                        FileIterator const Start, FileIterator const End)
1382 {
1383    std::vector<pkgIndexFile *> Files;
1384    bool mergeFailure = false;
1385
1386    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1387       if (I->HasPackages() == false || mergeFailure)
1388          return;
1389
1390       if (I->Exists() == false)
1391          return;
1392
1393       if (I->FindInCache(Gen.GetCache()).end() == false)
1394       {
1395          _error->Warning("Duplicate sources.list entry %s",
1396                I->Describe().c_str());
1397          return;
1398       }
1399
1400       map_filesize_t const Size = I->Size();
1401       if (Progress != NULL)
1402          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1403       CurrentSize += Size;
1404
1405       if (I->Merge(Gen,Progress) == false)
1406          mergeFailure = true;
1407    };
1408
1409    if (List !=  NULL)
1410    {
1411       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1412       {
1413          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1414          {
1415             _error->Warning("Duplicate sources.list entry %s",
1416                   (*i)->Describe().c_str());
1417             continue;
1418          }
1419
1420          if ((*i)->Merge(Gen, Progress) == false)
1421             return false;
1422
1423          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1424          if (Indexes != NULL)
1425             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1426          if (mergeFailure)
1427             return false;
1428       }
1429    }
1430
1431    if (Start != End)
1432    {
1433       Gen.SelectReleaseFile("", "");
1434       std::for_each(Start, End, indexFileMerge);
1435       if (mergeFailure)
1436          return false;
1437    }
1438    return true;
1439 }
1440                                                                         /*}}}*/
1441 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1442 // ---------------------------------------------------------------------
1443 /* This makes sure that the status cache (the cache that has all
1444    index files from the sources list and all local ones) is ready
1445    to be mmaped. If OutMap is not zero then a MMap object representing
1446    the cache will be stored there. This is pretty much mandetory if you
1447    are using AllowMem. AllowMem lets the function be run as non-root
1448    where it builds the cache 'fast' into a memory buffer. */
1449 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1450 {
1451    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1452    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1453    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1454    Flags |= MMap::Moveable;
1455    if (_config->FindB("APT::Cache-Fallback", false) == true)
1456       Flags |= MMap::Fallback;
1457    if (CacheF != NULL)
1458       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1459    else
1460       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1461 }
1462 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1463       std::string const &FileName)
1464 {
1465    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1466    if (_error->PendingError() == true)
1467       return false;
1468
1469    fchmod(SCacheF.Fd(),0644);
1470
1471    // Write out the main data
1472    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1473       return _error->Error(_("IO Error saving source cache"));
1474    SCacheF.Sync();
1475
1476    // Write out the proper header
1477    Gen->GetCache().HeaderP->Dirty = false;
1478    if (SCacheF.Seek(0) == false ||
1479          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1480       return _error->Error(_("IO Error saving source cache"));
1481    Gen->GetCache().HeaderP->Dirty = true;
1482    SCacheF.Sync();
1483    return true;
1484 }
1485 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1486       SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1487 {
1488    Map = CreateDynamicMMap(NULL, 0);
1489    FileFd CacheF(FileName, FileFd::ReadOnly);
1490    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1491    if ((alloc == 0 && _error->PendingError())
1492          || CacheF.Read((unsigned char *)Map->Data() + alloc,
1493             CacheF.Size()) == false)
1494       return false;
1495    Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1496    return true;
1497 }
1498 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1499                         MMap **OutMap, bool AllowMem)
1500    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1501 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1502                         MMap **OutMap,bool AllowMem)
1503 {
1504    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1505
1506    std::vector<pkgIndexFile *> Files;
1507    if (_system->AddStatusFiles(Files) == false)
1508       return false;
1509
1510    // Decide if we can write to the files..
1511    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1512    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1513
1514    // ensure the cache directory exists
1515    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1516    {
1517       string dir = _config->FindDir("Dir::Cache");
1518       size_t const len = dir.size();
1519       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1520          dir = dir.substr(0, len - 5);
1521       if (CacheFile.empty() == false)
1522          CreateDirectory(dir, flNotFile(CacheFile));
1523       if (SrcCacheFile.empty() == false)
1524          CreateDirectory(dir, flNotFile(SrcCacheFile));
1525    }
1526
1527    if (Progress != NULL)
1528       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1529
1530    bool pkgcache_fine = false;
1531    bool srcpkgcache_fine = false;
1532    bool volatile_fine = List.GetVolatileFiles().empty();
1533
1534    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
1535    {
1536       if (Debug == true)
1537          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1538       pkgcache_fine = true;
1539       srcpkgcache_fine = true;
1540    }
1541    if (pkgcache_fine == false)
1542    {
1543       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1544       {
1545          if (Debug == true)
1546             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1547          srcpkgcache_fine = true;
1548       }
1549    }
1550
1551    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1552    {
1553       if (Progress != NULL)
1554          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1555       return true;
1556    }
1557
1558    bool Writeable = false;
1559    if (srcpkgcache_fine == false || pkgcache_fine == false)
1560    {
1561       if (CacheFile.empty() == false)
1562          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1563       else if (SrcCacheFile.empty() == false)
1564          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1565
1566       if (Debug == true)
1567          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1568
1569       if (Writeable == false && AllowMem == false)
1570       {
1571          if (CacheFile.empty() == false)
1572             return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1573          else if (SrcCacheFile.empty() == false)
1574             return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
1575          else
1576             return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
1577       }
1578    }
1579
1580    // At this point we know we need to construct something, so get storage ready
1581    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1582    if (Debug == true)
1583       std::clog << "Open memory Map (not filebased)" << std::endl;
1584
1585    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1586    map_filesize_t CurrentSize = 0;
1587    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1588    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1589    if (srcpkgcache_fine == true && pkgcache_fine == false)
1590    {
1591       if (Debug == true)
1592          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1593       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1594          return false;
1595       srcpkgcache_fine = true;
1596       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1597    }
1598    else if (srcpkgcache_fine == false)
1599    {
1600       if (Debug == true)
1601          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1602       Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1603
1604       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1605       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1606                Files.end(),Files.end()) == false)
1607          return false;
1608
1609       if (Writeable == true && SrcCacheFile.empty() == false)
1610          if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
1611             return false;
1612    }
1613
1614    if (pkgcache_fine == false)
1615    {
1616       if (Debug == true)
1617          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1618       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1619                Files.begin(), Files.end()) == false)
1620          return false;
1621
1622       if (Writeable == true && CacheFile.empty() == false)
1623          if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
1624             return false;
1625    }
1626
1627    if (Debug == true)
1628       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1629
1630    if (volatile_fine == false)
1631    {
1632       if (Gen == nullptr)
1633       {
1634          if (Debug == true)
1635             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1636          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1637             return false;
1638       }
1639
1640       Files = List.GetVolatileFiles();
1641       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1642                Files.begin(), Files.end()) == false)
1643          return false;
1644    }
1645
1646    if (OutMap != nullptr)
1647       *OutMap = Map.UnGuard();
1648
1649    if (Debug == true)
1650       std::clog << "Everything is ready for shipping" << std::endl;
1651    return true;
1652 }
1653                                                                         /*}}}*/
1654 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1655 // ---------------------------------------------------------------------
1656 /* */
1657 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1658    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1659 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1660 {
1661    std::vector<pkgIndexFile *> Files;
1662    if (_system->AddStatusFiles(Files) == false)
1663       return false;
1664
1665    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1666    map_filesize_t CurrentSize = 0;
1667    map_filesize_t TotalSize = 0;
1668
1669    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1670
1671    // Build the status cache
1672    if (Progress != NULL)
1673       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1674    pkgCacheGenerator Gen(Map.Get(),Progress);
1675    if (_error->PendingError() == true)
1676       return false;
1677    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1678                   Files.begin(), Files.end()) == false)
1679       return false;
1680
1681    if (_error->PendingError() == true)
1682       return false;
1683    *OutMap = Map.UnGuard();
1684
1685    return true;
1686 }
1687                                                                         /*}}}*/
1688 // IsDuplicateDescription                                               /*{{{*/
1689 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1690                             MD5SumValue const &CurMd5, std::string const &CurLang)
1691 {
1692    // Descriptions in the same link-list have all the same md5
1693    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1694       return false;
1695    for (; Desc.end() == false; ++Desc)
1696       if (Desc.LanguageCode() == CurLang)
1697          return true;
1698    return false;
1699 }
1700                                                                         /*}}}*/
1701
1702 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1703 pkgCacheListParser::~pkgCacheListParser() {}