apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/pkgsystem.h>
  22 #include <apt-pkg/macros.h>
  23 #include <apt-pkg/metaindex.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/hashsum_template.h>
  26 #include <apt-pkg/indexfile.h>
  27 #include <apt-pkg/md5.h>
  28 #include <apt-pkg/mmap.h>
  29 #include <apt-pkg/pkgcache.h>
  30 #include <apt-pkg/cacheiterators.h>
  31
  32 #include <stddef.h>
  33 #include <string.h>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37 #include <memory>
  38 #include <algorithm>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43
  44 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                        /*}}}*/
  45 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  46 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  47
  48 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  49                             MD5SumValue const &CurMd5, std::string const &CurLang);
  50
  51 using std::string;
  52 using APT::StringView;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
  60 {
  61    if (Map.Size() == 0)
  62    {
  63       // Setup the map interface..
  64       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  65       _error->PushToStack();
  66       Map.RawAllocate(sizeof(pkgCache::Header));
  67       bool const newError = _error->PendingError();
  68       _error->MergeWithStack();
  69       if (newError)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  86       if (unlikely(idxArchitecture == 0))
  87          return;
  88       Cache.HeaderP->Architecture = idxArchitecture;
  89
  90       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  91       if (archs.size() > 1)
  92       {
  93          std::vector<std::string>::const_iterator a = archs.begin();
  94          std::string list = *a;
  95          for (++a; a != archs.end(); ++a)
  96             list.append(",").append(*a);
  97          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  98          if (unlikely(idxArchitectures == 0))
  99             return;
 100          Cache.HeaderP->SetArchitectures(idxArchitectures);
 101       }
 102       else
 103          Cache.HeaderP->SetArchitectures(idxArchitecture);
 104
 105       // Calculate the hash for the empty map, so ReMap does not fail
 106       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true || Map.validData() == false)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 138
 139    if (_config->FindB("Debug::pkgCacheGen", false))
 140       std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
 141    Map.Sync(0,sizeof(pkgCache::Header));
 142 }
 143                                                                         /*}}}*/
 144 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 145    if (oldMap == newMap)
 146       return;
 147
 148    if (_config->FindB("Debug::pkgCacheGen", false))
 149       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 150
 151    Cache.ReMap(false);
 152
 153    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 154    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 178         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 179       (*i)->ReMap(oldMap, newMap);
 180 }                                                                       /*}}}*/
 181 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 182 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 183                                         const unsigned long &Len) {
 184    void const * const oldMap = Map.Data();
 185    map_stringitem_t const index = Map.WriteString(String, Len);
 186    if (index != 0)
 187       ReMap(oldMap, Map.Data());
 188    return index;
 189 }
 190                                                                         /*}}}*/
 191 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 192 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 193    void const * const oldMap = Map.Data();
 194    map_stringitem_t const index = Map.WriteString(String);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 201    void const * const oldMap = Map.Data();
 202    map_pointer_t const index = Map.Allocate(size);
 203    if (index != 0)
 204       ReMap(oldMap, Map.Data());
 205    return index;
 206 }
 207                                                                         /*}}}*/
 208 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 209 // ---------------------------------------------------------------------
 210 /* This provides the generation of the entries in the cache. Each loop
 211    goes through a single package record from the underlying parse engine. */
 212 bool pkgCacheGenerator::MergeList(ListParser &List,
 213                                   pkgCache::VerIterator *OutVer)
 214 {
 215    List.Owner = this;
 216
 217    unsigned int Counter = 0;
 218    while (List.Step() == true)
 219    {
 220       string const PackageName = List.Package();
 221       if (PackageName.empty() == true)
 222          return false;
 223
 224       Counter++;
 225       if (Counter % 100 == 0 && Progress != 0)
 226          Progress->Progress(List.Offset());
 227
 228       string Arch = List.Architecture();
 229       string const Version = List.Version();
 230       if (Version.empty() == true && Arch.empty() == true)
 231       {
 232          // package descriptions
 233          if (MergeListGroup(List, PackageName) == false)
 234             return false;
 235          continue;
 236       }
 237
 238       // Get a pointer to the package structure
 239       pkgCache::PkgIterator Pkg;
 240       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 241       if (NewPackage(Pkg, PackageName, Arch) == false)
 242          // TRANSLATOR: The first placeholder is a package name,
 243          // the other two should be copied verbatim as they include debug info
 244          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 245                               PackageName.c_str(), "NewPackage", 1);
 246
 247
 248       if (Version.empty() == true)
 249       {
 250          if (MergeListPackage(List, Pkg) == false)
 251             return false;
 252       }
 253       else
 254       {
 255          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 256             return false;
 257       }
 258
 259       if (OutVer != 0)
 260          return true;
 261    }
 262
 263    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 264       return _error->Error(_("Wow, you exceeded the number of package "
 265                              "names this APT is capable of."));
 266    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 267       return _error->Error(_("Wow, you exceeded the number of versions "
 268                              "this APT is capable of."));
 269    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 270       return _error->Error(_("Wow, you exceeded the number of descriptions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 273       return _error->Error(_("Wow, you exceeded the number of dependencies "
 274                              "this APT is capable of."));
 275
 276    return true;
 277 }
 278 // CacheGenerator::MergeListGroup                                       /*{{{*/
 279 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 280 {
 281    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 282    // a group has no data on it's own, only packages have it but these
 283    // stanzas like this come from Translation- files to add descriptions,
 284    // but without a version we don't need a description for it…
 285    if (Grp.end() == true)
 286       return true;
 287    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 288
 289    pkgCache::PkgIterator Pkg;
 290    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 291    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 292       if (MergeListPackage(List, Pkg) == false)
 293          return false;
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298 // CacheGenerator::MergeListPackage                                     /*{{{*/
 299 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 300 {
 301    // we first process the package, then the descriptions
 302    // (for deb this package processing is in fact a no-op)
 303    pkgCache::VerIterator Ver(Cache);
 304    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 305    if (List.UsePackage(Pkg, Ver) == false)
 306       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 307                            Pkg.Name(), "UsePackage", 1);
 308
 309    // Find the right version to write the description
 310    MD5SumValue CurMd5 = List.Description_md5();
 311    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 312    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 313    {
 314       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 315
 316       // a version can only have one md5 describing it
 317       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 318          continue;
 319
 320       map_stringitem_t md5idx = VerDesc->md5sum;
 321       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 322       {
 323          // don't add a new description if we have one for the given
 324          // md5 && language
 325          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 326             continue;
 327
 328          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 329       }
 330
 331       // we can stop here as all "same" versions will share the description
 332       break;
 333    }
 334
 335    return true;
 336 }
 337                                                                         /*}}}*/
 338 // CacheGenerator::MergeListVersion                                     /*{{{*/
 339 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 340                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 341 {
 342    pkgCache::VerIterator Ver = Pkg.VersionList();
 343    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 344    map_pointer_t *LastVer = &Pkg->VersionList;
 345    void const * oldMap = Map.Data();
 346
 347    unsigned short const Hash = List.VersionHash();
 348    if (Ver.end() == false)
 349    {
 350       /* We know the list is sorted so we use that fact in the search.
 351          Insertion of new versions is done with correct sorting */
 352       int Res = 1;
 353       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 354       {
 355          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 356          // Version is higher as current version - insert here
 357          if (Res > 0)
 358             break;
 359          // Versionstrings are equal - is hash also equal?
 360          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 361             break;
 362          // proceed with the next till we have either the right
 363          // or we found another version (which will be lower)
 364       }
 365
 366       /* We already have a version for this item, record that we saw it */
 367       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 368       {
 369          if (List.UsePackage(Pkg,Ver) == false)
 370             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 371                                  Pkg.Name(), "UsePackage", 2);
 372
 373          if (NewFileVer(Ver,List) == false)
 374             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 375                                  Pkg.Name(), "NewFileVer", 1);
 376
 377          // Read only a single record and return
 378          if (OutVer != 0)
 379          {
 380             *OutVer = Ver;
 381             return true;
 382          }
 383
 384          return true;
 385       }
 386    }
 387
 388    // Add a new version
 389    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 390    if (unlikely(verindex == 0))
 391       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 392                            Pkg.Name(), "NewVersion", 1);
 393
 394    if (oldMap != Map.Data())
 395          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 396    *LastVer = verindex;
 397
 398    if (unlikely(List.NewVersion(Ver) == false))
 399       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 400                            Pkg.Name(), "NewVersion", 2);
 401
 402    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "UsePackage", 3);
 405
 406    if (unlikely(NewFileVer(Ver,List) == false))
 407       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 408                            Pkg.Name(), "NewFileVer", 2);
 409
 410    pkgCache::GrpIterator Grp = Pkg.Group();
 411    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 412
 413    /* If it is the first version of this package we need to add implicit
 414       Multi-Arch dependencies to all other package versions in the group now -
 415       otherwise we just add them for this new version */
 416    if (Pkg.VersionList()->NextVer == 0)
 417    {
 418       pkgCache::PkgIterator P = Grp.PackageList();
 419       Dynamic<pkgCache::PkgIterator> DynP(P);
 420       for (; P.end() != true; P = Grp.NextPkg(P))
 421       {
 422          if (P->ID == Pkg->ID)
 423             continue;
 424          pkgCache::VerIterator V = P.VersionList();
 425          Dynamic<pkgCache::VerIterator> DynV(V);
 426          for (; V.end() != true; ++V)
 427             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 428                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 429                                     Pkg.Name(), "AddImplicitDepends", 1);
 430       }
 431    }
 432    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 433       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 434                            Pkg.Name(), "AddImplicitDepends", 2);
 435
 436    // Read only a single record and return
 437    if (OutVer != 0)
 438    {
 439       *OutVer = Ver;
 440       return true;
 441    }
 442
 443    /* Record the Description(s) based on their master md5sum */
 444    MD5SumValue CurMd5 = List.Description_md5();
 445
 446    /* Before we add a new description we first search in the group for
 447       a version with a description of the same MD5 - if so we reuse this
 448       description group instead of creating our own for this version */
 449    for (pkgCache::PkgIterator P = Grp.PackageList();
 450         P.end() == false; P = Grp.NextPkg(P))
 451    {
 452       for (pkgCache::VerIterator V = P.VersionList();
 453            V.end() == false; ++V)
 454       {
 455          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 456             continue;
 457          Ver->DescriptionList = V->DescriptionList;
 458       }
 459    }
 460
 461    // We haven't found reusable descriptions, so add the first description(s)
 462    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 463    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 464    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 465       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 466          return false;
 467    return true;
 468 }
 469                                                                         /*}}}*/
 470 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 471 {
 472    pkgCache::DescIterator Desc;
 473    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 474
 475    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 476    if (unlikely(descindex == 0))
 477       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 478             Ver.ParentPkg().Name(), "NewDescription", 1);
 479
 480    md5idx = Desc->md5sum;
 481    Desc->ParentPkg = Ver.ParentPkg().Index();
 482
 483    // we add at the end, so that the start is constant as we need
 484    // that to be able to efficiently share these lists
 485    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 486    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 487    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 488    *LastNextDesc = descindex;
 489
 490    if (NewFileDesc(Desc,List) == false)
 491       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 492             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 493
 494    return true;
 495 }
 496                                                                         /*}}}*/
 497                                                                         /*}}}*/
 498 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 499 // ---------------------------------------------------------------------
 500 /* This creates a new group structure and adds it to the hash table */
 501 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 502 {
 503    Grp = Cache.FindGrp(Name);
 504    if (Grp.end() == false)
 505       return true;
 506
 507    // Get a structure
 508    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 509    if (unlikely(Group == 0))
 510       return false;
 511
 512    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 513    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 514    if (unlikely(idxName == 0))
 515       return false;
 516    Grp->Name = idxName;
 517
 518    // Insert it into the hash table
 519    unsigned long const Hash = Cache.Hash(Name);
 520    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 521
 522    while (*insertAt != 0 && Name.compare(Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
 523       insertAt = &(Cache.GrpP + *insertAt)->Next;
 524    Grp->Next = *insertAt;
 525    *insertAt = Group;
 526
 527    Grp->ID = Cache.HeaderP->GroupCount++;
 528    return true;
 529 }
 530                                                                         /*}}}*/
 531 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 532 // ---------------------------------------------------------------------
 533 /* This creates a new package structure and adds it to the hash table */
 534 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
 535                                         StringView Arch) {
 536    pkgCache::GrpIterator Grp;
 537    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 538    if (unlikely(NewGroup(Grp, Name) == false))
 539       return false;
 540
 541    Pkg = Grp.FindPkg(Arch);
 542       if (Pkg.end() == false)
 543          return true;
 544
 545    // Get a structure
 546    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 547    if (unlikely(Package == 0))
 548       return false;
 549    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 550
 551    // Set the name, arch and the ID
 552    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 553    Pkg->Group = Grp.Index();
 554    // all is mapped to the native architecture
 555    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 556    if (unlikely(idxArch == 0))
 557       return false;
 558    Pkg->Arch = idxArch;
 559    Pkg->ID = Cache.HeaderP->PackageCount++;
 560
 561    // Insert the package into our package list
 562    if (Grp->FirstPackage == 0) // the group is new
 563    {
 564       Grp->FirstPackage = Package;
 565       // Insert it into the hash table
 566       map_id_t const Hash = Cache.Hash(Name);
 567       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 568       while (*insertAt != 0 && Name.compare(Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 569          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 570       Pkg->NextPackage = *insertAt;
 571       *insertAt = Package;
 572    }
 573    else // Group the Packages together
 574    {
 575       // if sibling is provided by another package, this one is too
 576       {
 577          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 578          if (M.end() == false) {
 579             pkgCache::PrvIterator Prv;
 580             Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 581             for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 582             {
 583                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 584                   continue;
 585                pkgCache::VerIterator Ver = Prv.OwnerVer();
 586                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 587                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 588                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 589                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 590                {
 591                   if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
 592                      continue;
 593                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 594                      return false;
 595                }
 596             }
 597          }
 598       }
 599       // let M-A:foreign package siblings provide this package
 600       {
 601          pkgCache::PkgIterator P;
 602          pkgCache::VerIterator Ver;
 603          Dynamic<pkgCache::PkgIterator> DynP(P);
 604          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 605
 606          for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 607          {
 608             if (APT::Configuration::checkArchitecture(P.Arch()) == false)
 609                continue;
 610             for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
 611                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 612                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 613                      return false;
 614          }
 615       }
 616       // and negative dependencies, don't forget negative dependencies
 617       {
 618          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 619          if (M.end() == false) {
 620             pkgCache::DepIterator Dep;
 621             Dynamic<pkgCache::DepIterator> DynDep(Dep);
 622             for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 623             {
 624                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 625                   continue;
 626                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 627                      Dep->Type != pkgCache::Dep::Replaces)
 628                   continue;
 629                pkgCache::VerIterator Ver = Dep.ParentVer();
 630                Dynamic<pkgCache::VerIterator> DynVer(Ver);
 631                map_pointer_t * unused = NULL;
 632                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 633                   return false;
 634             }
 635          }
 636       }
 637
 638       // this package is the new last package
 639       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 640       Pkg->NextPackage = LastPkg->NextPackage;
 641       LastPkg->NextPackage = Package;
 642    }
 643    Grp->LastPackage = Package;
 644
 645    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
 646    if (Arch == "any")
 647    {
 648       size_t const found = Name.find(':');
 649       StringView const NameA = Name.substr(0, found);
 650       StringView const ArchA = Name.substr(found + 1);
 651       pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
 652       if (PkgA.end() == false)
 653       {
 654          Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
 655          pkgCache::PrvIterator Prv = PkgA.ProvidesList();
 656          for (; Prv.end() == false; ++Prv)
 657          {
 658             if (Prv.IsMultiArchImplicit())
 659                continue;
 660             pkgCache::VerIterator V = Prv.OwnerVer();
 661             if (ArchA != V.ParentPkg().Arch())
 662                continue;
 663             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 664                return false;
 665          }
 666          pkgCache::VerIterator V = PkgA.VersionList();
 667          Dynamic<pkgCache::VerIterator> DynV(V);
 668          for (; V.end() == false; ++V)
 669          {
 670             if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
 671                return false;
 672          }
 673       }
 674    }
 675    return true;
 676 }
 677                                                                         /*}}}*/
 678 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 679 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 680                                            pkgCache::PkgIterator &P,
 681                                            pkgCache::VerIterator &V)
 682 {
 683    // copy P.Arch() into a string here as a cache remap
 684    // in NewDepends() later may alter the pointer location
 685    string Arch = P.Arch() == NULL ? "" : P.Arch();
 686    map_pointer_t *OldDepLast = NULL;
 687    /* MultiArch handling introduces a lot of implicit Dependencies:
 688       - MultiArch: same → Co-Installable if they have the same version
 689       - All others conflict with all other group members */
 690    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 691    pkgCache::PkgIterator D = G.PackageList();
 692    Dynamic<pkgCache::PkgIterator> DynD(D);
 693    map_stringitem_t const VerStrIdx = V->VerStr;
 694    for (; D.end() != true; D = G.NextPkg(D))
 695    {
 696       if (Arch == D.Arch() || D->VersionList == 0)
 697          continue;
 698       /* We allow only one installed arch at the time
 699          per group, therefore each group member conflicts
 700          with all other group members */
 701       if (coInstall == true)
 702       {
 703          // Replaces: ${self}:other ( << ${binary:Version})
 704          NewDepends(D, V, VerStrIdx,
 705                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 706                     OldDepLast);
 707          // Breaks: ${self}:other (!= ${binary:Version})
 708          NewDepends(D, V, VerStrIdx,
 709                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 710                     OldDepLast);
 711       } else {
 712          // Conflicts: ${self}:other
 713          NewDepends(D, V, 0,
 714                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 715                     OldDepLast);
 716       }
 717    }
 718    return true;
 719 }
 720 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 721                                            pkgCache::PkgIterator &D)
 722 {
 723    /* MultiArch handling introduces a lot of implicit Dependencies:
 724       - MultiArch: same → Co-Installable if they have the same version
 725       - All others conflict with all other group members */
 726    map_pointer_t *OldDepLast = NULL;
 727    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 728    if (coInstall == true)
 729    {
 730       map_stringitem_t const VerStrIdx = V->VerStr;
 731       // Replaces: ${self}:other ( << ${binary:Version})
 732       NewDepends(D, V, VerStrIdx,
 733                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 734                  OldDepLast);
 735       // Breaks: ${self}:other (!= ${binary:Version})
 736       NewDepends(D, V, VerStrIdx,
 737                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 738                  OldDepLast);
 739    } else {
 740       // Conflicts: ${self}:other
 741       NewDepends(D, V, 0,
 742                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 743                  OldDepLast);
 744    }
 745    return true;
 746 }
 747
 748                                                                         /*}}}*/
 749 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 750 // ---------------------------------------------------------------------
 751 /* */
 752 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 753                                    ListParser &List)
 754 {
 755    if (CurrentFile == 0)
 756       return true;
 757
 758    // Get a structure
 759    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 760    if (VerFile == 0)
 761       return false;
 762
 763    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 764    VF->File = CurrentFile - Cache.PkgFileP;
 765
 766    // Link it to the end of the list
 767    map_pointer_t *Last = &Ver->FileList;
 768    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 769       Last = &V->NextFile;
 770    VF->NextFile = *Last;
 771    *Last = VF.Index();
 772
 773    VF->Offset = List.Offset();
 774    VF->Size = List.Size();
 775    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 776       Cache.HeaderP->MaxVerFileSize = VF->Size;
 777    Cache.HeaderP->VerFileCount++;
 778
 779    return true;
 780 }
 781                                                                         /*}}}*/
 782 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 783 // ---------------------------------------------------------------------
 784 /* This puts a version structure in the linked list */
 785 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 786                                             const string &VerStr,
 787                                             map_pointer_t const ParentPkg,
 788                                             unsigned short const Hash,
 789                                             map_pointer_t const Next)
 790 {
 791    // Get a structure
 792    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 793    if (Version == 0)
 794       return 0;
 795
 796    // Fill it in
 797    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 798    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 799    Ver->NextVer = Next;
 800    Ver->ParentPkg = ParentPkg;
 801    Ver->Hash = Hash;
 802    Ver->ID = Cache.HeaderP->VersionCount++;
 803
 804    // try to find the version string in the group for reuse
 805    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 806    pkgCache::GrpIterator Grp = Pkg.Group();
 807    if (Pkg.end() == false && Grp.end() == false)
 808    {
 809       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 810       {
 811          if (Pkg == P)
 812             continue;
 813          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 814          {
 815             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 816             if (cmp == 0)
 817             {
 818                Ver->VerStr = V->VerStr;
 819                return Version;
 820             }
 821             else if (cmp < 0)
 822                break;
 823          }
 824       }
 825    }
 826    // haven't found the version string, so create
 827    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 828    if (unlikely(idxVerStr == 0))
 829       return 0;
 830    Ver->VerStr = idxVerStr;
 831    return Version;
 832 }
 833                                                                         /*}}}*/
 834 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 835 // ---------------------------------------------------------------------
 836 /* */
 837 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 838                                    ListParser &List)
 839 {
 840    if (CurrentFile == 0)
 841       return true;
 842
 843    // Get a structure
 844    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 845    if (DescFile == 0)
 846       return false;
 847
 848    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 849    DF->File = CurrentFile - Cache.PkgFileP;
 850
 851    // Link it to the end of the list
 852    map_pointer_t *Last = &Desc->FileList;
 853    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 854       Last = &D->NextFile;
 855
 856    DF->NextFile = *Last;
 857    *Last = DF.Index();
 858
 859    DF->Offset = List.Offset();
 860    DF->Size = List.Size();
 861    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 862       Cache.HeaderP->MaxDescFileSize = DF->Size;
 863    Cache.HeaderP->DescFileCount++;
 864
 865    return true;
 866 }
 867                                                                         /*}}}*/
 868 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 869 // ---------------------------------------------------------------------
 870 /* This puts a description structure in the linked list */
 871 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 872                                             const string &Lang,
 873                                             const MD5SumValue &md5sum,
 874                                             map_stringitem_t const idxmd5str)
 875 {
 876    // Get a structure
 877    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 878    if (Description == 0)
 879       return 0;
 880
 881    // Fill it in
 882    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 883    Desc->ID = Cache.HeaderP->DescriptionCount++;
 884    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 885    if (unlikely(idxlanguage_code == 0))
 886       return 0;
 887    Desc->language_code = idxlanguage_code;
 888
 889    if (idxmd5str != 0)
 890       Desc->md5sum = idxmd5str;
 891    else
 892    {
 893       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 894       if (unlikely(idxmd5sum == 0))
 895          return 0;
 896       Desc->md5sum = idxmd5sum;
 897    }
 898
 899    return Description;
 900 }
 901                                                                         /*}}}*/
 902 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 903 // ---------------------------------------------------------------------
 904 /* This creates a dependency element in the tree. It is linked to the
 905    version and to the package that it is pointing to. */
 906 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 907                                    pkgCache::VerIterator &Ver,
 908                                    map_pointer_t const Version,
 909                                    uint8_t const Op,
 910                                    uint8_t const Type,
 911                                    map_pointer_t* &OldDepLast)
 912 {
 913    void const * const oldMap = Map.Data();
 914    // Get a structure
 915    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 916    if (unlikely(Dependency == 0))
 917       return false;
 918
 919    bool isDuplicate = false;
 920    map_pointer_t DependencyData = 0;
 921    map_pointer_t PreviousData = 0;
 922    if (Pkg->RevDepends != 0)
 923    {
 924       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 925       DependencyData = L->DependencyData;
 926       do {
 927          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 928          if (Version > D->Version)
 929             break;
 930          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 931          {
 932             isDuplicate = true;
 933             break;
 934          }
 935          PreviousData = DependencyData;
 936          DependencyData = D->NextData;
 937       } while (DependencyData != 0);
 938    }
 939
 940    if (isDuplicate == false)
 941    {
 942       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 943       if (unlikely(DependencyData == 0))
 944         return false;
 945    }
 946
 947    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 948    Link->ParentVer = Ver.Index();
 949    Link->DependencyData = DependencyData;
 950    Link->ID = Cache.HeaderP->DependsCount++;
 951
 952    pkgCache::DepIterator Dep(Cache, Link);
 953    if (isDuplicate == false)
 954    {
 955       Dep->Type = Type;
 956       Dep->CompareOp = Op;
 957       Dep->Version = Version;
 958       Dep->Package = Pkg.Index();
 959       ++Cache.HeaderP->DependsDataCount;
 960       if (PreviousData != 0)
 961       {
 962          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 963          Dep->NextData = D->NextData;
 964          D->NextData = DependencyData;
 965       }
 966       else if (Pkg->RevDepends != 0)
 967       {
 968          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
 969          Dep->NextData = D->DependencyData;
 970       }
 971    }
 972
 973    if (isDuplicate == true || PreviousData != 0)
 974    {
 975       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
 976       Link->NextRevDepends = L->NextRevDepends;
 977       L->NextRevDepends = Dependency;
 978    }
 979    else
 980    {
 981       Link->NextRevDepends = Pkg->RevDepends;
 982       Pkg->RevDepends = Dependency;
 983    }
 984
 985
 986    // Do we know where to link the Dependency to?
 987    if (OldDepLast == NULL)
 988    {
 989       OldDepLast = &Ver->DependsList;
 990       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 991          OldDepLast = &D->NextDepends;
 992    } else if (oldMap != Map.Data())
 993       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 994
 995    Dep->NextDepends = *OldDepLast;
 996    *OldDepLast = Dependency;
 997    OldDepLast = &Dep->NextDepends;
 998    return true;
 999 }
1000                                                                         /*}}}*/
1001 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1002 // ---------------------------------------------------------------------
1003 /* This creates a Group and the Package to link this dependency to if
1004    needed and handles also the caching of the old endpoint */
1005 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1006                                                StringView PackageName,
1007                                                StringView Arch,
1008                                                StringView Version,
1009                                                uint8_t const Op,
1010                                                uint8_t const Type)
1011 {
1012    pkgCache::GrpIterator Grp;
1013    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1014    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1015       return false;
1016
1017    map_stringitem_t idxVersion = 0;
1018    if (Version.empty() == false)
1019    {
1020       int const CmpOp = Op & 0x0F;
1021       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1022       if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
1023          idxVersion = Ver->VerStr;
1024
1025       if (idxVersion == 0)
1026       {
1027          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1028          if (unlikely(idxVersion == 0))
1029             return false;
1030       }
1031    }
1032
1033    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1034          Type == pkgCache::Dep::Conflicts ||
1035          Type == pkgCache::Dep::Replaces);
1036
1037    pkgCache::PkgIterator Pkg;
1038    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1039    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1040    {
1041       // Locate the target package
1042       Pkg = Grp.FindPkg(Arch);
1043       if (Pkg.end() == true) {
1044          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1045             return false;
1046       }
1047
1048       /* Caching the old end point speeds up generation substantially */
1049       if (OldDepVer != Ver) {
1050          OldDepLast = NULL;
1051          OldDepVer = Ver;
1052       }
1053
1054       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1055    }
1056    else
1057    {
1058       /* Caching the old end point speeds up generation substantially */
1059       if (OldDepVer != Ver) {
1060          OldDepLast = NULL;
1061          OldDepVer = Ver;
1062       }
1063
1064       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1065       {
1066          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1067                return false;
1068       }
1069    }
1070    return true;
1071 }
1072                                                                         /*}}}*/
1073 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1074 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1075                                                 StringView PkgName,
1076                                                 StringView PkgArch,
1077                                                 StringView Version,
1078                                                 uint8_t const Flags)
1079 {
1080    pkgCache const &Cache = Owner->Cache;
1081
1082    // We do not add self referencing provides
1083    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1084         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
1085          (Version.empty() || Version == Ver.VerStr()))
1086       return true;
1087
1088    // Locate the target package
1089    pkgCache::PkgIterator Pkg;
1090    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1091    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1092       return false;
1093
1094    map_stringitem_t idxProvideVersion = 0;
1095    if (Version.empty() == false) {
1096       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1097       if (unlikely(idxProvideVersion == 0))
1098          return false;
1099    }
1100    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1101 }
1102 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1103                                     pkgCache::PkgIterator &Pkg,
1104                                     map_pointer_t const ProvideVersion,
1105                                     uint8_t const Flags)
1106 {
1107    // Get a structure
1108    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1109    if (unlikely(Provides == 0))
1110       return false;
1111    ++Cache.HeaderP->ProvidesCount;
1112
1113    // Fill it in
1114    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1115    Prv->Version = Ver.Index();
1116    Prv->ProvideVersion = ProvideVersion;
1117    Prv->Flags = Flags;
1118    Prv->NextPkgProv = Ver->ProvidesList;
1119    Ver->ProvidesList = Prv.Index();
1120
1121    // Link it to the package
1122    Prv->ParentPkg = Pkg.Index();
1123    Prv->NextProvides = Pkg->ProvidesList;
1124    Pkg->ProvidesList = Prv.Index();
1125    return true;
1126 }
1127                                                                         /*}}}*/
1128 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1129 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
1130                                 StringView Version, uint8_t const Flags) {
1131    pkgCache &Cache = Owner->Cache;
1132    pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
1133    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1134
1135    if (Grp.end() == true)
1136       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1137    else
1138    {
1139       map_stringitem_t idxProvideVersion = 0;
1140       if (Version.empty() == false) {
1141          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1142          if (unlikely(idxProvideVersion == 0))
1143             return false;
1144       }
1145
1146       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1147       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1148       pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
1149       Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
1150       pkgCache::PkgIterator Pkg;
1151       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1152       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1153       {
1154          if (isImplicit && OwnerPkg == Pkg)
1155             continue;
1156          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1157             continue;
1158          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1159             return false;
1160       }
1161    }
1162    return true;
1163 }
1164                                                                         /*}}}*/
1165 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1166       pkgCache::VerIterator const &Ver)
1167 {
1168    return Hash == Ver->Hash;
1169 }
1170                                                                         /*}}}*/
1171 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1172 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1173                                    unsigned long Flags)
1174 {
1175    if (File.empty() && Site.empty())
1176    {
1177       CurrentRlsFile = NULL;
1178       return true;
1179    }
1180
1181    // Get some space for the structure
1182    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1183    if (unlikely(idxFile == 0))
1184       return false;
1185    CurrentRlsFile = Cache.RlsFileP + idxFile;
1186
1187    // Fill it in
1188    map_stringitem_t const idxFileName = WriteStringInMap(File);
1189    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1190    if (unlikely(idxFileName == 0 || idxSite == 0))
1191       return false;
1192    CurrentRlsFile->FileName = idxFileName;
1193    CurrentRlsFile->Site = idxSite;
1194    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1195    CurrentRlsFile->Flags = Flags;
1196    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1197    RlsFileName = File;
1198    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1199    Cache.HeaderP->ReleaseFileCount++;
1200
1201    return true;
1202 }
1203                                                                         /*}}}*/
1204 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1205 // ---------------------------------------------------------------------
1206 /* This is used to select which file is to be associated with all newly
1207    added versions. The caller is responsible for setting the IMS fields. */
1208 bool pkgCacheGenerator::SelectFile(std::string const &File,
1209                                    pkgIndexFile const &Index,
1210                                    std::string const &Architecture,
1211                                    std::string const &Component,
1212                                    unsigned long const Flags)
1213 {
1214    // Get some space for the structure
1215    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1216    if (unlikely(idxFile == 0))
1217       return false;
1218    CurrentFile = Cache.PkgFileP + idxFile;
1219
1220    // Fill it in
1221    map_stringitem_t const idxFileName = WriteStringInMap(File);
1222    if (unlikely(idxFileName == 0))
1223       return false;
1224    CurrentFile->FileName = idxFileName;
1225    CurrentFile->NextFile = Cache.HeaderP->FileList;
1226    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1227    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1228    if (unlikely(idxIndexType == 0))
1229       return false;
1230    CurrentFile->IndexType = idxIndexType;
1231    if (Architecture.empty())
1232       CurrentFile->Architecture = 0;
1233    else
1234    {
1235       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1236       if (unlikely(arch == 0))
1237          return false;
1238       CurrentFile->Architecture = arch;
1239    }
1240    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1241    if (unlikely(component == 0))
1242       return false;
1243    CurrentFile->Component = component;
1244    CurrentFile->Flags = Flags;
1245    if (CurrentRlsFile != NULL)
1246       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1247    else
1248       CurrentFile->Release = 0;
1249    PkgFileName = File;
1250    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1251    Cache.HeaderP->PackageFileCount++;
1252
1253    if (Progress != 0)
1254       Progress->SubProgress(Index.Size());
1255    return true;
1256 }
1257                                                                         /*}}}*/
1258 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1259 // ---------------------------------------------------------------------
1260 /* This is used to create handles to strings. Given the same text it
1261    always returns the same number */
1262 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1263                                                  unsigned int Size)
1264 {
1265    auto strings = &strMixed;
1266    switch(type) {
1267       case MIXED: strings = &strMixed; break;
1268       case PKGNAME: strings = &strPkgNames; break;
1269       case VERSIONNUMBER: strings = &strVersions; break;
1270       case SECTION: strings = &strSections; break;
1271       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
1272    }
1273
1274    auto const item = strings->find({S, Size, nullptr, 0});
1275    if (item != strings->end())
1276       return item->item;
1277
1278    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1279    strings->insert({nullptr, Size, this, idxString});
1280    return idxString;
1281 }
1282                                                                         /*}}}*/
1283 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1284 // ---------------------------------------------------------------------
1285 /* This just verifies that each file in the list of index files exists,
1286    has matching attributes with the cache and the cache does not have
1287    any extra files. */
1288 class APT_HIDDEN ScopedErrorRevert {
1289 public:
1290    ScopedErrorRevert() { _error->PushToStack(); }
1291    ~ScopedErrorRevert() { _error->RevertToStack(); }
1292 };
1293 static bool CheckValidity(const string &CacheFile,
1294                           pkgSourceList &List,
1295                           FileIterator const Start,
1296                           FileIterator const End,
1297                           MMap **OutMap = 0,
1298                           pkgCache **OutCache = 0)
1299 {
1300    ScopedErrorRevert ser;
1301    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1302    // No file, certainly invalid
1303    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1304    {
1305       if (Debug == true)
1306          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1307       return false;
1308    }
1309
1310    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1311    {
1312       if (Debug == true)
1313          std::clog << "sources.list is newer than the cache" << std::endl;
1314       return false;
1315    }
1316
1317    // Map it
1318    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1319    std::unique_ptr<MMap> Map(new MMap(CacheF,0));
1320    if (unlikely(Map->validData()) == false)
1321       return false;
1322    std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
1323    pkgCache &Cache = *CacheP.get();
1324    if (_error->PendingError() || Map->Size() == 0)
1325    {
1326       if (Debug == true)
1327          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1328       return false;
1329    }
1330
1331    std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
1332    memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
1333    std::vector<pkgIndexFile *> Files;
1334    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1335    {
1336       if (Debug == true)
1337          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1338       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1339       if (RlsFile.end() == true)
1340       {
1341          if (Debug == true)
1342             std::clog << "FindInCache returned end-Pointer" << std::endl;
1343          return false;
1344       }
1345
1346       RlsVisited[RlsFile->ID] = true;
1347       if (Debug == true)
1348          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1349
1350       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1351       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1352             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1353    }
1354    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1355       if (RlsVisited[I] == false)
1356       {
1357          if (Debug == true)
1358             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1359          return false;
1360       }
1361
1362    std::copy(Start, End, std::back_inserter(Files));
1363
1364    /* Now we check every index file, see if it is in the cache,
1365       verify the IMS data and check that it is on the disk too.. */
1366    std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
1367    memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
1368    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1369    {
1370       if (Debug == true)
1371          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1372       if ((*PkgFile)->Exists() == false)
1373       {
1374          if (Debug == true)
1375             std::clog << "file doesn't exist" << std::endl;
1376          continue;
1377       }
1378
1379       // FindInCache is also expected to do an IMS check.
1380       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1381       if (File.end() == true)
1382       {
1383          if (Debug == true)
1384             std::clog << "FindInCache returned end-Pointer" << std::endl;
1385          return false;
1386       }
1387
1388       Visited[File->ID] = true;
1389       if (Debug == true)
1390          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1391    }
1392
1393    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1394       if (Visited[I] == false)
1395       {
1396          if (Debug == true)
1397             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1398          return false;
1399       }
1400
1401    if (_error->PendingError() == true)
1402    {
1403       if (Debug == true)
1404       {
1405          std::clog << "Validity failed because of pending errors:" << std::endl;
1406          _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
1407       }
1408       return false;
1409    }
1410
1411    if (OutMap != 0)
1412       *OutMap = Map.release();
1413    if (OutCache != 0)
1414       *OutCache = CacheP.release();
1415    return true;
1416 }
1417                                                                         /*}}}*/
1418 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1419 // ---------------------------------------------------------------------
1420 /* Size is kind of an abstract notion that is only used for the progress
1421    meter */
1422 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1423 {
1424    map_filesize_t TotalSize = 0;
1425    if (List !=  NULL)
1426    {
1427       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1428       {
1429          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1430          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1431             if ((*j)->HasPackages() == true)
1432                TotalSize += (*j)->Size();
1433       }
1434    }
1435
1436    for (; Start < End; ++Start)
1437    {
1438       if ((*Start)->HasPackages() == false)
1439          continue;
1440       TotalSize += (*Start)->Size();
1441    }
1442    return TotalSize;
1443 }
1444                                                                         /*}}}*/
1445 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1446 static bool BuildCache(pkgCacheGenerator &Gen,
1447                        OpProgress * const Progress,
1448                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1449                        pkgSourceList const * const List,
1450                        FileIterator const Start, FileIterator const End)
1451 {
1452    bool mergeFailure = false;
1453
1454    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1455       if (I->HasPackages() == false || mergeFailure)
1456          return;
1457
1458       if (I->Exists() == false)
1459          return;
1460
1461       if (I->FindInCache(Gen.GetCache()).end() == false)
1462       {
1463          _error->Warning("Duplicate sources.list entry %s",
1464                I->Describe().c_str());
1465          return;
1466       }
1467
1468       map_filesize_t const Size = I->Size();
1469       if (Progress != NULL)
1470          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1471       CurrentSize += Size;
1472
1473       if (I->Merge(Gen,Progress) == false)
1474          mergeFailure = true;
1475    };
1476
1477    if (List !=  NULL)
1478    {
1479       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1480       {
1481          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1482          {
1483             _error->Warning("Duplicate sources.list entry %s",
1484                   (*i)->Describe().c_str());
1485             continue;
1486          }
1487
1488          if ((*i)->Merge(Gen, Progress) == false)
1489             return false;
1490
1491          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1492          if (Indexes != NULL)
1493             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1494          if (mergeFailure)
1495             return false;
1496       }
1497    }
1498
1499    if (Start != End)
1500    {
1501       Gen.SelectReleaseFile("", "");
1502       std::for_each(Start, End, indexFileMerge);
1503       if (mergeFailure)
1504          return false;
1505    }
1506    return true;
1507 }
1508                                                                         /*}}}*/
1509 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1510 // ---------------------------------------------------------------------
1511 /* This makes sure that the status cache (the cache that has all
1512    index files from the sources list and all local ones) is ready
1513    to be mmaped. If OutMap is not zero then a MMap object representing
1514    the cache will be stored there. This is pretty much mandetory if you
1515    are using AllowMem. AllowMem lets the function be run as non-root
1516    where it builds the cache 'fast' into a memory buffer. */
1517 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1518 {
1519    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1520    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1521    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1522    Flags |= MMap::Moveable;
1523    if (_config->FindB("APT::Cache-Fallback", false) == true)
1524       Flags |= MMap::Fallback;
1525    if (CacheF != NULL)
1526       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1527    else
1528       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1529 }
1530 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1531       std::string const &FileName)
1532 {
1533    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1534    if (SCacheF.IsOpen() == false || SCacheF.Failed())
1535       return false;
1536
1537    fchmod(SCacheF.Fd(),0644);
1538
1539    // Write out the main data
1540    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1541       return _error->Error(_("IO Error saving source cache"));
1542
1543    // Write out the proper header
1544    Gen->GetCache().HeaderP->Dirty = false;
1545    Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
1546    if (SCacheF.Seek(0) == false ||
1547          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1548       return _error->Error(_("IO Error saving source cache"));
1549    Gen->GetCache().HeaderP->Dirty = true;
1550    return true;
1551 }
1552 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1553       std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1554 {
1555    Map.reset(CreateDynamicMMap(NULL, 0));
1556    if (unlikely(Map->validData()) == false)
1557       return false;
1558    FileFd CacheF(FileName, FileFd::ReadOnly);
1559    if (CacheF.IsOpen() == false || CacheF.Failed())
1560       return false;
1561    _error->PushToStack();
1562    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1563    bool const newError = _error->PendingError();
1564    _error->MergeWithStack();
1565    if (alloc == 0 && newError)
1566       return false;
1567    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
1568       return false;
1569    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1570    return true;
1571 }
1572 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1573                         MMap **OutMap, bool AllowMem)
1574    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1575 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1576                         MMap **OutMap,bool)
1577 {
1578    return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
1579 }
1580 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1581                         MMap **OutMap,pkgCache **OutCache, bool)
1582 {
1583    // FIXME: deprecate the ignored AllowMem parameter
1584    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1585
1586    std::vector<pkgIndexFile *> Files;
1587    if (_system->AddStatusFiles(Files) == false)
1588       return false;
1589
1590    // Decide if we can write to the files..
1591    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1592    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1593
1594    // ensure the cache directory exists
1595    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1596    {
1597       string dir = _config->FindDir("Dir::Cache");
1598       size_t const len = dir.size();
1599       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1600          dir = dir.substr(0, len - 5);
1601       if (CacheFile.empty() == false)
1602          CreateDirectory(dir, flNotFile(CacheFile));
1603       if (SrcCacheFile.empty() == false)
1604          CreateDirectory(dir, flNotFile(SrcCacheFile));
1605    }
1606
1607    if (Progress != NULL)
1608       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1609
1610    bool pkgcache_fine = false;
1611    bool srcpkgcache_fine = false;
1612    bool volatile_fine = List.GetVolatileFiles().empty();
1613
1614    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
1615                      volatile_fine ? OutCache : NULL) == true)
1616    {
1617       if (Debug == true)
1618          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1619       pkgcache_fine = true;
1620       srcpkgcache_fine = true;
1621    }
1622    if (pkgcache_fine == false)
1623    {
1624       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1625       {
1626          if (Debug == true)
1627             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1628          srcpkgcache_fine = true;
1629       }
1630    }
1631
1632    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1633    {
1634       if (Progress != NULL)
1635          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1636       return true;
1637    }
1638
1639    bool Writeable = false;
1640    if (srcpkgcache_fine == false || pkgcache_fine == false)
1641    {
1642       if (CacheFile.empty() == false)
1643          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1644       else if (SrcCacheFile.empty() == false)
1645          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1646
1647       if (Debug == true)
1648          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1649    }
1650
1651    // At this point we know we need to construct something, so get storage ready
1652    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1653    if (unlikely(Map->validData()) == false)
1654       return false;
1655    if (Debug == true)
1656       std::clog << "Open memory Map (not filebased)" << std::endl;
1657
1658    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1659    map_filesize_t CurrentSize = 0;
1660    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1661    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1662    if (srcpkgcache_fine == true && pkgcache_fine == false)
1663    {
1664       if (Debug == true)
1665          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1666       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1667          return false;
1668       srcpkgcache_fine = true;
1669       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1670    }
1671    else if (srcpkgcache_fine == false)
1672    {
1673       if (Debug == true)
1674          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1675       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
1676
1677       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1678       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1679                Files.end(),Files.end()) == false)
1680          return false;
1681
1682       if (Writeable == true && SrcCacheFile.empty() == false)
1683          if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
1684             return false;
1685    }
1686
1687    if (pkgcache_fine == false)
1688    {
1689       if (Debug == true)
1690          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1691       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1692                Files.begin(), Files.end()) == false)
1693          return false;
1694
1695       if (Writeable == true && CacheFile.empty() == false)
1696          if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
1697             return false;
1698    }
1699
1700    if (Debug == true)
1701       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1702
1703    if (volatile_fine == false)
1704    {
1705       if (Gen == nullptr)
1706       {
1707          if (Debug == true)
1708             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1709          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1710             return false;
1711       }
1712
1713       Files = List.GetVolatileFiles();
1714       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1715                Files.begin(), Files.end()) == false)
1716          return false;
1717    }
1718
1719    if (OutMap != nullptr)
1720       *OutMap = Map.release();
1721
1722    if (Debug == true)
1723       std::clog << "Everything is ready for shipping" << std::endl;
1724    return true;
1725 }
1726                                                                         /*}}}*/
1727 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1728 class APT_HIDDEN ScopedErrorMerge {
1729 public:
1730    ScopedErrorMerge() { _error->PushToStack(); }
1731    ~ScopedErrorMerge() { _error->MergeWithStack(); }
1732 };
1733 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1734    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1735 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1736 {
1737    std::vector<pkgIndexFile *> Files;
1738    if (_system->AddStatusFiles(Files) == false)
1739       return false;
1740
1741    ScopedErrorMerge sem;
1742    std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
1743    if (unlikely(Map->validData()) == false)
1744       return false;
1745    map_filesize_t CurrentSize = 0;
1746    map_filesize_t TotalSize = 0;
1747    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1748
1749    // Build the status cache
1750    if (Progress != NULL)
1751       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1752    pkgCacheGenerator Gen(Map.get(),Progress);
1753    if (_error->PendingError() == true)
1754       return false;
1755    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1756                   Files.begin(), Files.end()) == false)
1757       return false;
1758
1759    if (_error->PendingError() == true)
1760       return false;
1761    *OutMap = Map.release();
1762
1763    return true;
1764 }
1765                                                                         /*}}}*/
1766 // IsDuplicateDescription                                               /*{{{*/
1767 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1768                             MD5SumValue const &CurMd5, std::string const &CurLang)
1769 {
1770    // Descriptions in the same link-list have all the same md5
1771    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1772       return false;
1773    for (; Desc.end() == false; ++Desc)
1774       if (Desc.LanguageCode() == CurLang)
1775          return true;
1776    return false;
1777 }
1778                                                                         /*}}}*/
1779
1780 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), d(NULL) {}
1781 pkgCacheListParser::~pkgCacheListParser() {}