apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180 }                                                                       /*}}}*/
 181 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 182 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 183                                         const unsigned long &Len) {
 184    void const * const oldMap = Map.Data();
 185    map_stringitem_t const index = Map.WriteString(String, Len);
 186    if (index != 0)
 187       ReMap(oldMap, Map.Data());
 188    return index;
 189 }
 190                                                                         /*}}}*/
 191 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 192 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 201    void const * const oldMap = Map.Data();
 202    map_pointer_t const index = Map.Allocate(size);
 203    if (index != 0)
 204       ReMap(oldMap, Map.Data());
 205    return index;
 206 }
 207                                                                         /*}}}*/
 208 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 209 // ---------------------------------------------------------------------
 210 /* This provides the generation of the entries in the cache. Each loop
 211    goes through a single package record from the underlying parse engine. */
 212 bool pkgCacheGenerator::MergeList(ListParser &List,
 213                                   pkgCache::VerIterator *OutVer)
 214 {
 215    List.Owner = this;
 216
 217    unsigned int Counter = 0;
 218    while (List.Step() == true)
 219    {
 220       string const PackageName = List.Package();
 221       if (PackageName.empty() == true)
 222          return false;
 223
 224       Counter++;
 225       if (Counter % 100 == 0 && Progress != 0)
 226          Progress->Progress(List.Offset());
 227
 228       string Arch = List.Architecture();
 229       string const Version = List.Version();
 230       if (Version.empty() == true && Arch.empty() == true)
 231       {
 232          // package descriptions
 233          if (MergeListGroup(List, PackageName) == false)
 234             return false;
 235          continue;
 236       }
 237
 238       // Get a pointer to the package structure
 239       pkgCache::PkgIterator Pkg;
 240       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 241       if (NewPackage(Pkg, PackageName, Arch) == false)
 242          // TRANSLATOR: The first placeholder is a package name,
 243          // the other two should be copied verbatim as they include debug info
 244          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 245                               PackageName.c_str(), "NewPackage", 1);
 246
 247
 248       if (Version.empty() == true)
 249       {
 250          if (MergeListPackage(List, Pkg) == false)
 251             return false;
 252       }
 253       else
 254       {
 255          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 256             return false;
 257       }
 258
 259       if (OutVer != 0)
 260          return true;
 261    }
 262
 263    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 264       return _error->Error(_("Wow, you exceeded the number of package "
 265                              "names this APT is capable of."));
 266    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of versions "
 268                              "this APT is capable of."));
 269    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 270       return _error->Error(_("Wow, you exceeded the number of descriptions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 273       return _error->Error(_("Wow, you exceeded the number of dependencies "
 274                              "this APT is capable of."));
 275
 276    return true;
 277 }
 278 // CacheGenerator::MergeListGroup                                       /*{{{*/
 279 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 280 {
 281    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 282    // a group has no data on it's own, only packages have it but these
 283    // stanzas like this come from Translation- files to add descriptions,
 284    // but without a version we don't need a description for it…
 285    if (Grp.end() == true)
 286       return true;
 287    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 288
 289    pkgCache::PkgIterator Pkg;
 290    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 291    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 292       if (MergeListPackage(List, Pkg) == false)
 293          return false;
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298 // CacheGenerator::MergeListPackage                                     /*{{{*/
 299 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 300 {
 301    // we first process the package, then the descriptions
 302    // (for deb this package processing is in fact a no-op)
 303    pkgCache::VerIterator Ver(Cache);
 304    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 305    if (List.UsePackage(Pkg, Ver) == false)
 306       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 307                            Pkg.Name(), "UsePackage", 1);
 308
 309    // Find the right version to write the description
 310    MD5SumValue CurMd5 = List.Description_md5();
 311    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 312       return true;
 313    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 314    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 315    {
 316       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 317
 318       // a version can only have one md5 describing it
 319       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 320          continue;
 321
 322       map_stringitem_t md5idx = VerDesc->md5sum;
 323       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 324       {
 325          // don't add a new description if we have one for the given
 326          // md5 && language
 327          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 328             continue;
 329
 330          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 331       }
 332
 333       // we can stop here as all "same" versions will share the description
 334       break;
 335    }
 336
 337    return true;
 338 }
 339                                                                         /*}}}*/
 340 // CacheGenerator::MergeListVersion                                     /*{{{*/
 341 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 342                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 343 {
 344    pkgCache::VerIterator Ver = Pkg.VersionList();
 345    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 346    map_pointer_t *LastVer = &Pkg->VersionList;
 347    void const * oldMap = Map.Data();
 348
 349    unsigned short const Hash = List.VersionHash();
 350    if (Ver.end() == false)
 351    {
 352       /* We know the list is sorted so we use that fact in the search.
 353          Insertion of new versions is done with correct sorting */
 354       int Res = 1;
 355       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 356       {
 357          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 358          // Version is higher as current version - insert here
 359          if (Res > 0)
 360             break;
 361          // Versionstrings are equal - is hash also equal?
 362          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 363             break;
 364          // proceed with the next till we have either the right
 365          // or we found another version (which will be lower)
 366       }
 367
 368       /* We already have a version for this item, record that we saw it */
 369       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 370       {
 371          if (List.UsePackage(Pkg,Ver) == false)
 372             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 373                                  Pkg.Name(), "UsePackage", 2);
 374
 375          if (NewFileVer(Ver,List) == false)
 376             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 377                                  Pkg.Name(), "NewFileVer", 1);
 378
 379          // Read only a single record and return
 380          if (OutVer != 0)
 381          {
 382             *OutVer = Ver;
 383             return true;
 384          }
 385
 386          return true;
 387       }
 388    }
 389
 390    // Add a new version
 391    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 392    if (unlikely(verindex == 0))
 393       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 394                            Pkg.Name(), "NewVersion", 1);
 395
 396    if (oldMap != Map.Data())
 397          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 398    *LastVer = verindex;
 399
 400    if (unlikely(List.NewVersion(Ver) == false))
 401       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 402                            Pkg.Name(), "NewVersion", 2);
 403
 404    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 405       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 406                            Pkg.Name(), "UsePackage", 3);
 407
 408    if (unlikely(NewFileVer(Ver,List) == false))
 409       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 410                            Pkg.Name(), "NewFileVer", 2);
 411
 412    pkgCache::GrpIterator Grp = Pkg.Group();
 413    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 414
 415    /* If it is the first version of this package we need to add implicit
 416       Multi-Arch dependencies to all other package versions in the group now -
 417       otherwise we just add them for this new version */
 418    if (Pkg.VersionList()->NextVer == 0)
 419    {
 420       pkgCache::PkgIterator P = Grp.PackageList();
 421       Dynamic<pkgCache::PkgIterator> DynP(P);
 422       for (; P.end() != true; P = Grp.NextPkg(P))
 423       {
 424          if (P->ID == Pkg->ID)
 425             continue;
 426          pkgCache::VerIterator V = P.VersionList();
 427          Dynamic<pkgCache::VerIterator> DynV(V);
 428          for (; V.end() != true; ++V)
 429             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 430                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 431                                     Pkg.Name(), "AddImplicitDepends", 1);
 432       }
 433    }
 434    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 435       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 436                            Pkg.Name(), "AddImplicitDepends", 2);
 437
 438    // Read only a single record and return
 439    if (OutVer != 0)
 440    {
 441       *OutVer = Ver;
 442       return true;
 443    }
 444
 445    /* Record the Description(s) based on their master md5sum */
 446    MD5SumValue CurMd5 = List.Description_md5();
 447    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 448       return true;
 449
 450    /* Before we add a new description we first search in the group for
 451       a version with a description of the same MD5 - if so we reuse this
 452       description group instead of creating our own for this version */
 453    for (pkgCache::PkgIterator P = Grp.PackageList();
 454         P.end() == false; P = Grp.NextPkg(P))
 455    {
 456       for (pkgCache::VerIterator V = P.VersionList();
 457            V.end() == false; ++V)
 458       {
 459          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 460             continue;
 461          Ver->DescriptionList = V->DescriptionList;
 462       }
 463    }
 464
 465    // We haven't found reusable descriptions, so add the first description(s)
 466    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 467    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 468    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 469       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 470          return false;
 471    return true;
 472 }
 473                                                                         /*}}}*/
 474 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 475 {
 476    pkgCache::DescIterator Desc;
 477    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 478
 479    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 480    if (unlikely(descindex == 0))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482             Ver.ParentPkg().Name(), "NewDescription", 1);
 483
 484    md5idx = Desc->md5sum;
 485    Desc->ParentPkg = Ver.ParentPkg().Index();
 486
 487    // we add at the end, so that the start is constant as we need
 488    // that to be able to efficiently share these lists
 489    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 490    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 491    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 492    *LastNextDesc = descindex;
 493
 494    if (NewFileDesc(Desc,List) == false)
 495       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 496             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 497
 498    return true;
 499 }
 500                                                                         /*}}}*/
 501                                                                         /*}}}*/
 502 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 503 // ---------------------------------------------------------------------
 504 /* This creates a new group structure and adds it to the hash table */
 505 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 506 {
 507    Grp = Cache.FindGrp(Name);
 508    if (Grp.end() == false)
 509       return true;
 510
 511    // Get a structure
 512    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 513    if (unlikely(Group == 0))
 514       return false;
 515
 516    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 517    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 518    if (unlikely(idxName == 0))
 519       return false;
 520    Grp->Name = idxName;
 521
 522    // Insert it into the hash table
 523    unsigned long const Hash = Cache.Hash(Name);
 524    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 525    while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 526       insertAt = &(Cache.GrpP + *insertAt)->Next;
 527    Grp->Next = *insertAt;
 528    *insertAt = Group;
 529
 530    Grp->ID = Cache.HeaderP->GroupCount++;
 531    return true;
 532 }
 533                                                                         /*}}}*/
 534 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 535 // ---------------------------------------------------------------------
 536 /* This creates a new package structure and adds it to the hash table */
 537 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 538                                         StringView Arch) {
 539    pkgCache::GrpIterator Grp;
 540    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 541    if (unlikely(NewGroup(Grp, Name) == false))
 542       return false;
 543
 544    Pkg = Grp.FindPkg(Arch);
 545       if (Pkg.end() == false)
 546          return true;
 547
 548    // Get a structure
 549    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 550    if (unlikely(Package == 0))
 551       return false;
 552    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 553
 554    // Set the name, arch and the ID
 555    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 556    Pkg->Group = Grp.Index();
 557    // all is mapped to the native architecture
 558    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 559    if (unlikely(idxArch == 0))
 560       return false;
 561    Pkg->Arch = idxArch;
 562    Pkg->ID = Cache.HeaderP->PackageCount++;
 563
 564    // Insert the package into our package list
 565    if (Grp->FirstPackage == 0) // the group is new
 566    {
 567       Grp->FirstPackage = Package;
 568       // Insert it into the hash table
 569       map_id_t const Hash = Cache.Hash(Name);
 570       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 571       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 572          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 573       Pkg->NextPackage = *insertAt;
 574       *insertAt = Package;
 575    }
 576    else // Group the Packages together
 577    {
 578       // but first get implicit provides done
 579       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 580       {
 581          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 582          if (M.end() == false) {
 583             pkgCache::PrvIterator Prv;
 584             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 585             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 586             {
 587                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 588                   continue;
 589                pkgCache::VerIterator Ver = Prv.OwnerVer();
 590                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 591                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 592                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 593                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 594                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 595                      return false;
 596             }
 597          }
 598
 599
 600          pkgCache::PkgIterator P;
 601          pkgCache::VerIterator Ver;
 602          Dynamic<pkgCache::PkgIterator> DynP(P);
 603          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 604
 605          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 606             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 607                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 608                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 609                      return false;
 610       }
 611       // and negative dependencies, don't forget negative dependencies
 612       {
 613          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 614          if (M.end() == false) {
 615             pkgCache::DepIterator Dep;
 616             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 617             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 618             {
 619                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 620                   continue;
 621                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 622                      Dep->Type != pkgCache::Dep::Replaces)
 623                   continue;
 624                pkgCache::VerIterator Ver = Dep.ParentVer();
 625                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 626                map_pointer_t * unused = NULL;
 627                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 628                   return false;
 629             }
 630          }
 631       }
 632
 633       // this package is the new last package
 634       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 635       Pkg->NextPackage = LastPkg->NextPackage;
 636       LastPkg->NextPackage = Package;
 637    }
 638    Grp->LastPackage = Package;
 639
 640    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 641    if (Arch == "any")
 642    {
 643       size_t const found = Name.find(':');
 644       StringView const NameA = Name.substr(0, found);
 645       StringView const ArchA = Name.substr(found + 1);
 646       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 647       if (PkgA.end() == false)
 648       {
 649          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 650          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 651          for (; Prv.end() == false; ++Prv)
 652          {
 653             if (Prv.IsMultiArchImplicit())
 654                continue;
 655             pkgCache::VerIterator V = Prv.OwnerVer();
 656             if (ArchA != V.ParentPkg().Arch())
 657                continue;
 658             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 659                return false;
 660          }
 661          pkgCache::VerIterator V = PkgA.VersionList();
 662          Dynamic<pkgCache::VerIterator> DynV(V);
 663          for (; V.end() == false; ++V)
 664          {
 665             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 666                return false;
 667          }
 668       }
 669    }
 670    return true;
 671 }
 672                                                                         /*}}}*/
 673 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 674 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 675                                            pkgCache::PkgIterator &P,
 676                                            pkgCache::VerIterator &V)
 677 {
 678    // copy P.Arch() into a string here as a cache remap
 679    // in NewDepends() later may alter the pointer location
 680    string Arch = P.Arch() == NULL ? "" : P.Arch();
 681    map_pointer_t *OldDepLast = NULL;
 682    /* MultiArch handling introduces a lot of implicit Dependencies:
 683       - MultiArch: same → Co-Installable if they have the same version
 684       - All others conflict with all other group members */
 685    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 686    pkgCache::PkgIterator D = G.PackageList();
 687    Dynamic<pkgCache::PkgIterator> DynD(D);
 688    map_stringitem_t const VerStrIdx = V->VerStr;
 689    for (; D.end() != true; D = G.NextPkg(D))
 690    {
 691       if (Arch == D.Arch() || D->VersionList == 0)
 692          continue;
 693       /* We allow only one installed arch at the time
 694          per group, therefore each group member conflicts
 695          with all other group members */
 696       if (coInstall == true)
 697       {
 698          // Replaces: ${self}:other ( << ${binary:Version})
 699          NewDepends(D, V, VerStrIdx,
 700                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 701                     OldDepLast);
 702          // Breaks: ${self}:other (!= ${binary:Version})
 703          NewDepends(D, V, VerStrIdx,
 704                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 705                     OldDepLast);
 706       } else {
 707          // Conflicts: ${self}:other
 708          NewDepends(D, V, 0,
 709                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 710                     OldDepLast);
 711       }
 712    }
 713    return true;
 714 }
 715 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 716                                            pkgCache::PkgIterator &D)
 717 {
 718    /* MultiArch handling introduces a lot of implicit Dependencies:
 719       - MultiArch: same → Co-Installable if they have the same version
 720       - All others conflict with all other group members */
 721    map_pointer_t *OldDepLast = NULL;
 722    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 723    if (coInstall == true)
 724    {
 725       map_stringitem_t const VerStrIdx = V->VerStr;
 726       // Replaces: ${self}:other ( << ${binary:Version})
 727       NewDepends(D, V, VerStrIdx,
 728                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 729                  OldDepLast);
 730       // Breaks: ${self}:other (!= ${binary:Version})
 731       NewDepends(D, V, VerStrIdx,
 732                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 733                  OldDepLast);
 734    } else {
 735       // Conflicts: ${self}:other
 736       NewDepends(D, V, 0,
 737                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 738                  OldDepLast);
 739    }
 740    return true;
 741 }
 742
 743                                                                         /*}}}*/
 744 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 745 // ---------------------------------------------------------------------
 746 /* */
 747 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 748                                    ListParser &List)
 749 {
 750    if (CurrentFile == 0)
 751       return true;
 752
 753    // Get a structure
 754    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 755    if (VerFile == 0)
 756       return false;
 757
 758    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 759    VF->File = CurrentFile - Cache.PkgFileP;
 760
 761    // Link it to the end of the list
 762    map_pointer_t *Last = &Ver->FileList;
 763    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 764       Last = &V->NextFile;
 765    VF->NextFile = *Last;
 766    *Last = VF.Index();
 767
 768    VF->Offset = List.Offset();
 769    VF->Size = List.Size();
 770    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 771       Cache.HeaderP->MaxVerFileSize = VF->Size;
 772    Cache.HeaderP->VerFileCount++;
 773
 774    return true;
 775 }
 776                                                                         /*}}}*/
 777 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 778 // ---------------------------------------------------------------------
 779 /* This puts a version structure in the linked list */
 780 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 781                                             const string &VerStr,
 782                                             map_pointer_t const ParentPkg,
 783                                             unsigned short const Hash,
 784                                             map_pointer_t const Next)
 785 {
 786    // Get a structure
 787    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 788    if (Version == 0)
 789       return 0;
 790
 791    // Fill it in
 792    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 793    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 794    Ver->NextVer = Next;
 795    Ver->ParentPkg = ParentPkg;
 796    Ver->Hash = Hash;
 797    Ver->ID = Cache.HeaderP->VersionCount++;
 798
 799    // try to find the version string in the group for reuse
 800    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 801    pkgCache::GrpIterator Grp = Pkg.Group();
 802    if (Pkg.end() == false && Grp.end() == false)
 803    {
 804       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 805       {
 806          if (Pkg == P)
 807             continue;
 808          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 809          {
 810             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 811             if (cmp == 0)
 812             {
 813                Ver->VerStr = V->VerStr;
 814                return Version;
 815             }
 816             else if (cmp < 0)
 817                break;
 818          }
 819       }
 820    }
 821    // haven't found the version string, so create
 822    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 823    if (unlikely(idxVerStr == 0))
 824       return 0;
 825    Ver->VerStr = idxVerStr;
 826    return Version;
 827 }
 828                                                                         /*}}}*/
 829 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 830 // ---------------------------------------------------------------------
 831 /* */
 832 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 833                                    ListParser &List)
 834 {
 835    if (CurrentFile == 0)
 836       return true;
 837
 838    // Get a structure
 839    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 840    if (DescFile == 0)
 841       return false;
 842
 843    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 844    DF->File = CurrentFile - Cache.PkgFileP;
 845
 846    // Link it to the end of the list
 847    map_pointer_t *Last = &Desc->FileList;
 848    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 849       Last = &D->NextFile;
 850
 851    DF->NextFile = *Last;
 852    *Last = DF.Index();
 853
 854    DF->Offset = List.Offset();
 855    DF->Size = List.Size();
 856    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 857       Cache.HeaderP->MaxDescFileSize = DF->Size;
 858    Cache.HeaderP->DescFileCount++;
 859
 860    return true;
 861 }
 862                                                                         /*}}}*/
 863 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 864 // ---------------------------------------------------------------------
 865 /* This puts a description structure in the linked list */
 866 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 867                                             const string &Lang,
 868                                             const MD5SumValue &md5sum,
 869                                             map_stringitem_t const idxmd5str)
 870 {
 871    // Get a structure
 872    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 873    if (Description == 0)
 874       return 0;
 875
 876    // Fill it in
 877    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 878    Desc->ID = Cache.HeaderP->DescriptionCount++;
 879    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 880    if (unlikely(idxlanguage_code == 0))
 881       return 0;
 882    Desc->language_code = idxlanguage_code;
 883
 884    if (idxmd5str != 0)
 885       Desc->md5sum = idxmd5str;
 886    else
 887    {
 888       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 889       if (unlikely(idxmd5sum == 0))
 890          return 0;
 891       Desc->md5sum = idxmd5sum;
 892    }
 893
 894    return Description;
 895 }
 896                                                                         /*}}}*/
 897 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 898 // ---------------------------------------------------------------------
 899 /* This creates a dependency element in the tree. It is linked to the
 900    version and to the package that it is pointing to. */
 901 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 902                                    pkgCache::VerIterator &Ver,
 903                                    map_pointer_t const Version,
 904                                    uint8_t const Op,
 905                                    uint8_t const Type,
 906                                    map_pointer_t* &OldDepLast)
 907 {
 908    void const * const oldMap = Map.Data();
 909    // Get a structure
 910    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 911    if (unlikely(Dependency == 0))
 912       return false;
 913
 914    bool isDuplicate = false;
 915    map_pointer_t DependencyData = 0;
 916    map_pointer_t PreviousData = 0;
 917    if (Pkg->RevDepends != 0)
 918    {
 919       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 920       DependencyData = L->DependencyData;
 921       do {
 922          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 923          if (Version > D->Version)
 924             break;
 925          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 926          {
 927             isDuplicate = true;
 928             break;
 929          }
 930          PreviousData = DependencyData;
 931          DependencyData = D->NextData;
 932       } while (DependencyData != 0);
 933    }
 934
 935    if (isDuplicate == false)
 936    {
 937       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 938       if (unlikely(DependencyData == 0))
 939         return false;
 940    }
 941
 942    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 943    Link->ParentVer = Ver.Index();
 944    Link->DependencyData = DependencyData;
 945    Link->ID = Cache.HeaderP->DependsCount++;
 946
 947    pkgCache::DepIterator Dep(Cache, Link);
 948    if (isDuplicate == false)
 949    {
 950       Dep->Type = Type;
 951       Dep->CompareOp = Op;
 952       Dep->Version = Version;
 953       Dep->Package = Pkg.Index();
 954       ++Cache.HeaderP->DependsDataCount;
 955       if (PreviousData != 0)
 956       {
 957          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 958          Dep->NextData = D->NextData;
 959          D->NextData = DependencyData;
 960       }
 961       else if (Pkg->RevDepends != 0)
 962       {
 963          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 964          Dep->NextData = D->DependencyData;
 965       }
 966    }
 967
 968    if (isDuplicate == true || PreviousData != 0)
 969    {
 970       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 971       Link->NextRevDepends = L->NextRevDepends;
 972       L->NextRevDepends = Dependency;
 973    }
 974    else
 975    {
 976       Link->NextRevDepends = Pkg->RevDepends;
 977       Pkg->RevDepends = Dependency;
 978    }
 979
 980
 981    // Do we know where to link the Dependency to?
 982    if (OldDepLast == NULL)
 983    {
 984       OldDepLast = &Ver->DependsList;
 985       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 986          OldDepLast = &D->NextDepends;
 987    } else if (oldMap != Map.Data())
 988       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 989
 990    Dep->NextDepends = *OldDepLast;
 991    *OldDepLast = Dependency;
 992    OldDepLast = &Dep->NextDepends;
 993    return true;
 994 }
 995                                                                         /*}}}*/
 996 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 997 // ---------------------------------------------------------------------
 998 /* This creates a Group and the Package to link this dependency to if
 999    needed and handles also the caching of the old endpoint */
1000 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1001                                                StringView PackageName,
1002                                                StringView Arch,
1003                                                StringView Version,
1004                                                uint8_t const Op,
1005                                                uint8_t const Type)
1006 {
1007    pkgCache::GrpIterator Grp;
1008    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1009    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1010       return false;
1011
1012    map_stringitem_t idxVersion = 0;
1013    if (Version.empty() == false)
1014    {
1015       int const CmpOp = Op & 0x0F;
1016       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1017       if (CmpOp == pkgCache::Dep::Equals && Version.compare(Ver.VerStr()) == 0)
1018          idxVersion = Ver->VerStr;
1019
1020       if (idxVersion == 0)
1021       {
1022          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1023          if (unlikely(idxVersion == 0))
1024             return false;
1025       }
1026    }
1027
1028    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1029          Type == pkgCache::Dep::Conflicts ||
1030          Type == pkgCache::Dep::Replaces);
1031
1032    pkgCache::PkgIterator Pkg;
1033    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1034    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1035    {
1036       // Locate the target package
1037       Pkg = Grp.FindPkg(Arch);
1038       if (Pkg.end() == true) {
1039          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1040             return false;
1041       }
1042
1043       /* Caching the old end point speeds up generation substantially */
1044       if (OldDepVer != Ver) {
1045          OldDepLast = NULL;
1046          OldDepVer = Ver;
1047       }
1048
1049       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1050    }
1051    else
1052    {
1053       /* Caching the old end point speeds up generation substantially */
1054       if (OldDepVer != Ver) {
1055          OldDepLast = NULL;
1056          OldDepVer = Ver;
1057       }
1058
1059       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1060       {
1061          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1062                return false;
1063       }
1064    }
1065    return true;
1066 }
1067                                                                         /*}}}*/
1068 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1069 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1070                                                 StringView PkgName,
1071                                                 StringView PkgArch,
1072                                                 StringView Version,
1073                                                 uint8_t const Flags)
1074 {
1075    pkgCache const &Cache = Owner->Cache;
1076
1077    // We do not add self referencing provides
1078    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1079         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1080          (Version.empty() || Version == Ver.VerStr()))
1081       return true;
1082
1083    // Locate the target package
1084    pkgCache::PkgIterator Pkg;
1085    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1086    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1087       return false;
1088
1089    map_stringitem_t idxProvideVersion = 0;
1090    if (Version.empty() == false) {
1091       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1092       if (unlikely(idxProvideVersion == 0))
1093          return false;
1094    }
1095    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1096 }
1097 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1098                                     pkgCache::PkgIterator &Pkg,
1099                                     map_pointer_t const ProvideVersion,
1100                                     uint8_t const Flags)
1101 {
1102    // Get a structure
1103    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1104    if (unlikely(Provides == 0))
1105       return false;
1106    ++Cache.HeaderP->ProvidesCount;
1107
1108    // Fill it in
1109    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1110    Prv->Version = Ver.Index();
1111    Prv->ProvideVersion = ProvideVersion;
1112    Prv->Flags = Flags;
1113    Prv->NextPkgProv = Ver->ProvidesList;
1114    Ver->ProvidesList = Prv.Index();
1115
1116    // Link it to the package
1117    Prv->ParentPkg = Pkg.Index();
1118    Prv->NextProvides = Pkg->ProvidesList;
1119    Pkg->ProvidesList = Prv.Index();
1120    return true;
1121 }
1122                                                                         /*}}}*/
1123 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1124 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1125                                 StringView Version, uint8_t const Flags) {
1126    pkgCache &Cache = Owner->Cache;
1127    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1128    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1129
1130    if (Grp.end() == true)
1131       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1132    else
1133    {
1134       map_stringitem_t idxProvideVersion = 0;
1135       if (Version.empty() == false) {
1136          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1137          if (unlikely(idxProvideVersion == 0))
1138             return false;
1139       }
1140
1141       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1142       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1143       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1144       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1145       pkgCache::PkgIterator Pkg;
1146       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1147       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1148       {
1149          if (isImplicit && OwnerPkg == Pkg)
1150             continue;
1151          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1152             continue;
1153          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1154             return false;
1155       }
1156    }
1157    return true;
1158 }
1159                                                                         /*}}}*/
1160 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1161       pkgCache::VerIterator const &Ver)
1162 {
1163    return Hash == Ver->Hash;
1164 }
1165                                                                         /*}}}*/
1166 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1167 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1168                                    unsigned long Flags)
1169 {
1170    if (File.empty() && Site.empty())
1171    {
1172       CurrentRlsFile = NULL;
1173       return true;
1174    }
1175
1176    // Get some space for the structure
1177    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1178    if (unlikely(idxFile == 0))
1179       return false;
1180    CurrentRlsFile = Cache.RlsFileP + idxFile;
1181
1182    // Fill it in
1183    map_stringitem_t const idxFileName = WriteStringInMap(File);
1184    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1185    if (unlikely(idxFileName == 0 || idxSite == 0))
1186       return false;
1187    CurrentRlsFile->FileName = idxFileName;
1188    CurrentRlsFile->Site = idxSite;
1189    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1190    CurrentRlsFile->Flags = Flags;
1191    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1192    RlsFileName = File;
1193    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1194    Cache.HeaderP->ReleaseFileCount++;
1195
1196    return true;
1197 }
1198                                                                         /*}}}*/
1199 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1200 // ---------------------------------------------------------------------
1201 /* This is used to select which file is to be associated with all newly
1202    added versions. The caller is responsible for setting the IMS fields. */
1203 bool pkgCacheGenerator::SelectFile(std::string const &File,
1204                                    pkgIndexFile const &Index,
1205                                    std::string const &Architecture,
1206                                    std::string const &Component,
1207                                    unsigned long const Flags)
1208 {
1209    // Get some space for the structure
1210    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1211    if (unlikely(idxFile == 0))
1212       return false;
1213    CurrentFile = Cache.PkgFileP + idxFile;
1214
1215    // Fill it in
1216    map_stringitem_t const idxFileName = WriteStringInMap(File);
1217    if (unlikely(idxFileName == 0))
1218       return false;
1219    CurrentFile->FileName = idxFileName;
1220    CurrentFile->NextFile = Cache.HeaderP->FileList;
1221    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1222    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1223    if (unlikely(idxIndexType == 0))
1224       return false;
1225    CurrentFile->IndexType = idxIndexType;
1226    if (Architecture.empty())
1227       CurrentFile->Architecture = 0;
1228    else
1229    {
1230       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1231       if (unlikely(arch == 0))
1232          return false;
1233       CurrentFile->Architecture = arch;
1234    }
1235    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1236    if (unlikely(component == 0))
1237       return false;
1238    CurrentFile->Component = component;
1239    CurrentFile->Flags = Flags;
1240    if (CurrentRlsFile != NULL)
1241       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1242    else
1243       CurrentFile->Release = 0;
1244    PkgFileName = File;
1245    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1246    Cache.HeaderP->PackageFileCount++;
1247
1248    if (Progress != 0)
1249       Progress->SubProgress(Index.Size());
1250    return true;
1251 }
1252                                                                         /*}}}*/
1253 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1254 // ---------------------------------------------------------------------
1255 /* This is used to create handles to strings. Given the same text it
1256    always returns the same number */
1257 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1258                                                  unsigned int Size)
1259 {
1260    std::string key(S, Size);
1261
1262    std::unordered_map<std::string,map_stringitem_t> * strings;
1263    switch(type) {
1264       case MIXED: strings = &strMixed; break;
1265       case PKGNAME: strings = &strPkgNames; break;
1266       case VERSIONNUMBER: strings = &strVersions; break;
1267       case SECTION: strings = &strSections; break;
1268       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1269    }
1270
1271    std::unordered_map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1272    if (item != strings->end())
1273       return item->second;
1274
1275    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1276    strings->insert(std::make_pair(std::move(key), idxString));
1277    return idxString;
1278 }
1279                                                                         /*}}}*/
1280 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1281 // ---------------------------------------------------------------------
1282 /* This just verifies that each file in the list of index files exists,
1283    has matching attributes with the cache and the cache does not have
1284    any extra files. */
1285 class APT_HIDDEN ScopedErrorRevert {
1286 public:
1287    ScopedErrorRevert() { _error->PushToStack(); }
1288    ~ScopedErrorRevert() { _error->RevertToStack(); }
1289 };
1290 static bool CheckValidity(const string &CacheFile,
1291                           pkgSourceList &List,
1292                           FileIterator const Start,
1293                           FileIterator const End,
1294                           MMap **OutMap = 0,
1295                           pkgCache **OutCache = 0)
1296 {
1297    ScopedErrorRevert ser;
1298    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1299    // No file, certainly invalid
1300    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1301    {
1302       if (Debug == true)
1303          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1304       return false;
1305    }
1306
1307    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1308    {
1309       if (Debug == true)
1310          std::clog << "sources.list is newer than the cache" << std::endl;
1311       return false;
1312    }
1313
1314    // Map it
1315    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1316    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1317    if (unlikely(Map->validData()) == false)
1318       return false;
1319    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1320    pkgCache &Cache = *CacheP.get();
1321    if (_error->PendingError() || Map->Size() == 0)
1322    {
1323       if (Debug == true)
1324          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1325       return false;
1326    }
1327
1328    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1329    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1330    std::vector<pkgIndexFile *> Files;
1331    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1332    {
1333       if (Debug == true)
1334          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1335       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1336       if (RlsFile.end() == true)
1337       {
1338          if (Debug == true)
1339             std::clog << "FindInCache returned end-Pointer" << std::endl;
1340          return false;
1341       }
1342
1343       RlsVisited[RlsFile->ID] = true;
1344       if (Debug == true)
1345          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1346
1347       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1348       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1349             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1350    }
1351    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1352       if (RlsVisited[I] == false)
1353       {
1354          if (Debug == true)
1355             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1356          return false;
1357       }
1358
1359    std::copy(Start, End, std::back_inserter(Files));
1360
1361    /* Now we check every index file, see if it is in the cache,
1362       verify the IMS data and check that it is on the disk too.. */
1363    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1364    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1365    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1366    {
1367       if (Debug == true)
1368          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1369       if ((*PkgFile)->Exists() == false)
1370       {
1371          if (Debug == true)
1372             std::clog << "file doesn't exist" << std::endl;
1373          continue;
1374       }
1375
1376       // FindInCache is also expected to do an IMS check.
1377       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1378       if (File.end() == true)
1379       {
1380          if (Debug == true)
1381             std::clog << "FindInCache returned end-Pointer" << std::endl;
1382          return false;
1383       }
1384
1385       Visited[File->ID] = true;
1386       if (Debug == true)
1387          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1388    }
1389
1390    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1391       if (Visited[I] == false)
1392       {
1393          if (Debug == true)
1394             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1395          return false;
1396       }
1397
1398    if (_error->PendingError() == true)
1399    {
1400       if (Debug == true)
1401       {
1402          std::clog << "Validity failed because of pending errors:" << std::endl;
1403          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1404       }
1405       return false;
1406    }
1407
1408    if (OutMap != 0)
1409       *OutMap = Map.release();
1410    if (OutCache != 0)
1411       *OutCache = CacheP.release();
1412    return true;
1413 }
1414                                                                         /*}}}*/
1415 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1416 // ---------------------------------------------------------------------
1417 /* Size is kind of an abstract notion that is only used for the progress
1418    meter */
1419 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1420 {
1421    map_filesize_t TotalSize = 0;
1422    if (List !=  NULL)
1423    {
1424       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1425       {
1426          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1427          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1428             if ((*j)->HasPackages() == true)
1429                TotalSize += (*j)->Size();
1430       }
1431    }
1432
1433    for (; Start < End; ++Start)
1434    {
1435       if ((*Start)->HasPackages() == false)
1436          continue;
1437       TotalSize += (*Start)->Size();
1438    }
1439    return TotalSize;
1440 }
1441                                                                         /*}}}*/
1442 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1443 static bool BuildCache(pkgCacheGenerator &Gen,
1444                        OpProgress * const Progress,
1445                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1446                        pkgSourceList const * const List,
1447                        FileIterator const Start, FileIterator const End)
1448 {
1449    bool mergeFailure = false;
1450
1451    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1452       if (I->HasPackages() == false || mergeFailure)
1453          return;
1454
1455       if (I->Exists() == false)
1456          return;
1457
1458       if (I->FindInCache(Gen.GetCache()).end() == false)
1459       {
1460          _error->Warning("Duplicate sources.list entry %s",
1461                I->Describe().c_str());
1462          return;
1463       }
1464
1465       map_filesize_t const Size = I->Size();
1466       if (Progress != NULL)
1467          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1468       CurrentSize += Size;
1469
1470       if (I->Merge(Gen,Progress) == false)
1471          mergeFailure = true;
1472    };
1473
1474    if (List !=  NULL)
1475    {
1476       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1477       {
1478          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1479          {
1480             _error->Warning("Duplicate sources.list entry %s",
1481                   (*i)->Describe().c_str());
1482             continue;
1483          }
1484
1485          if ((*i)->Merge(Gen, Progress) == false)
1486             return false;
1487
1488          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1489          if (Indexes != NULL)
1490             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1491          if (mergeFailure)
1492             return false;
1493       }
1494    }
1495
1496    if (Start != End)
1497    {
1498       Gen.SelectReleaseFile("", "");
1499       std::for_each(Start, End, indexFileMerge);
1500       if (mergeFailure)
1501          return false;
1502    }
1503    return true;
1504 }
1505                                                                         /*}}}*/
1506 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1507 // ---------------------------------------------------------------------
1508 /* This makes sure that the status cache (the cache that has all
1509    index files from the sources list and all local ones) is ready
1510    to be mmaped. If OutMap is not zero then a MMap object representing
1511    the cache will be stored there. This is pretty much mandetory if you
1512    are using AllowMem. AllowMem lets the function be run as non-root
1513    where it builds the cache 'fast' into a memory buffer. */
1514 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1515 {
1516    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1517    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1518    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1519    Flags |= MMap::Moveable;
1520    if (_config->FindB("APT::Cache-Fallback", false) == true)
1521       Flags |= MMap::Fallback;
1522    if (CacheF != NULL)
1523       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1524    else
1525       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1526 }
1527 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1528       std::string const &FileName)
1529 {
1530    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1531    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1532       return false;
1533
1534    fchmod(SCacheF.Fd(),0644);
1535
1536    // Write out the main data
1537    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1538       return _error->Error(_("IO Error saving source cache"));
1539
1540    // Write out the proper header
1541    Gen->GetCache().HeaderP->Dirty = false;
1542    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1543    if (SCacheF.Seek(0) == false ||
1544          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1545       return _error->Error(_("IO Error saving source cache"));
1546    Gen->GetCache().HeaderP->Dirty = true;
1547    return true;
1548 }
1549 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1550       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1551 {
1552    Map.reset(CreateDynamicMMap(NULL, 0));
1553    if (unlikely(Map->validData()) == false)
1554       return false;
1555    FileFd CacheF(FileName, FileFd::ReadOnly);
1556    if (CacheF.IsOpen() == false || CacheF.Failed())
1557       return false;
1558    _error->PushToStack();
1559    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1560    bool const newError = _error->PendingError();
1561    _error->MergeWithStack();
1562    if (alloc == 0 && newError)
1563       return false;
1564    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1565       return false;
1566    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1567    return true;
1568 }
1569 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1570                         MMap **OutMap, bool AllowMem)
1571    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1572 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1573                         MMap **OutMap,bool)
1574 {
1575    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1576 }
1577 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1578                         MMap **OutMap,pkgCache **OutCache, bool)
1579 {
1580    // FIXME: deprecate the ignored AllowMem parameter
1581    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1582
1583    std::vector<pkgIndexFile *> Files;
1584    if (_system->AddStatusFiles(Files) == false)
1585       return false;
1586
1587    // Decide if we can write to the files..
1588    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1589    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1590
1591    // ensure the cache directory exists
1592    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1593    {
1594       string dir = _config->FindDir("Dir::Cache");
1595       size_t const len = dir.size();
1596       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1597          dir = dir.substr(0, len - 5);
1598       if (CacheFile.empty() == false)
1599          CreateDirectory(dir, flNotFile(CacheFile));
1600       if (SrcCacheFile.empty() == false)
1601          CreateDirectory(dir, flNotFile(SrcCacheFile));
1602    }
1603
1604    if (Progress != NULL)
1605       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1606
1607    bool pkgcache_fine = false;
1608    bool srcpkgcache_fine = false;
1609    bool volatile_fine = List.GetVolatileFiles().empty();
1610
1611    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1612                      volatile_fine ? OutCache : NULL) == true)
1613    {
1614       if (Debug == true)
1615          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1616       pkgcache_fine = true;
1617       srcpkgcache_fine = true;
1618    }
1619    if (pkgcache_fine == false)
1620    {
1621       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1622       {
1623          if (Debug == true)
1624             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1625          srcpkgcache_fine = true;
1626       }
1627    }
1628
1629    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1630    {
1631       if (Progress != NULL)
1632          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1633       return true;
1634    }
1635
1636    bool Writeable = false;
1637    if (srcpkgcache_fine == false || pkgcache_fine == false)
1638    {
1639       if (CacheFile.empty() == false)
1640          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1641       else if (SrcCacheFile.empty() == false)
1642          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1643
1644       if (Debug == true)
1645          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1646    }
1647
1648    // At this point we know we need to construct something, so get storage ready
1649    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1650    if (unlikely(Map->validData()) == false)
1651       return false;
1652    if (Debug == true)
1653       std::clog << "Open memory Map (not filebased)" << std::endl;
1654
1655    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1656    map_filesize_t CurrentSize = 0;
1657    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1658    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1659    if (srcpkgcache_fine == true && pkgcache_fine == false)
1660    {
1661       if (Debug == true)
1662          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1663       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1664          return false;
1665       srcpkgcache_fine = true;
1666       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1667    }
1668    else if (srcpkgcache_fine == false)
1669    {
1670       if (Debug == true)
1671          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1672       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1673
1674       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1675       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1676                Files.end(),Files.end()) == false)
1677          return false;
1678
1679       if (Writeable == true && SrcCacheFile.empty() == false)
1680          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1681             return false;
1682    }
1683
1684    if (pkgcache_fine == false)
1685    {
1686       if (Debug == true)
1687          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1688       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1689                Files.begin(), Files.end()) == false)
1690          return false;
1691
1692       if (Writeable == true && CacheFile.empty() == false)
1693          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1694             return false;
1695    }
1696
1697    if (Debug == true)
1698       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1699
1700    if (volatile_fine == false)
1701    {
1702       if (Gen == nullptr)
1703       {
1704          if (Debug == true)
1705             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1706          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1707             return false;
1708       }
1709
1710       Files = List.GetVolatileFiles();
1711       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1712                Files.begin(), Files.end()) == false)
1713          return false;
1714    }
1715
1716    if (OutMap != nullptr)
1717       *OutMap = Map.release();
1718
1719    if (Debug == true)
1720       std::clog << "Everything is ready for shipping" << std::endl;
1721    return true;
1722 }
1723                                                                         /*}}}*/
1724 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1725 class APT_HIDDEN ScopedErrorMerge {
1726 public:
1727    ScopedErrorMerge() { _error->PushToStack(); }
1728    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1729 };
1730 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1731    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1732 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1733 {
1734    std::vector<pkgIndexFile *> Files;
1735    if (_system->AddStatusFiles(Files) == false)
1736       return false;
1737
1738    ScopedErrorMerge sem;
1739    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1740    if (unlikely(Map->validData()) == false)
1741       return false;
1742    map_filesize_t CurrentSize = 0;
1743    map_filesize_t TotalSize = 0;
1744    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1745
1746    // Build the status cache
1747    if (Progress != NULL)
1748       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1749    pkgCacheGenerator Gen(Map.get(),Progress);
1750    if (_error->PendingError() == true)
1751       return false;
1752    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1753                   Files.begin(), Files.end()) == false)
1754       return false;
1755
1756    if (_error->PendingError() == true)
1757       return false;
1758    *OutMap = Map.release();
1759
1760    return true;
1761 }
1762                                                                         /*}}}*/
1763 // IsDuplicateDescription                                               /*{{{*/
1764 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1765                             MD5SumValue const &CurMd5, std::string const &CurLang)
1766 {
1767    // Descriptions in the same link-list have all the same md5
1768    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1769       return false;
1770    for (; Desc.end() == false; ++Desc)
1771       if (Desc.LanguageCode() == CurLang)
1772          return true;
1773    return false;
1774 }
1775                                                                         /*}}}*/
1776
1777 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1778 pkgCacheListParser::~pkgCacheListParser() {}