apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <memory>
  40 #include <algorithm>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  47 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  48
  49 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  50                             MD5SumValue const &CurMd5, std::string const &CurLang);
  51
  52 using std::string;
  53
  54 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* We set the dirty flag and make sure that is written to the disk */
  57 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  58                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  59                      CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
  60 {
  61    if (_error->PendingError() == true)
  62       return;
  63
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  69          return;
  70
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72
  73       // Starting header
  74       *Cache.HeaderP = pkgCache::Header();
  75
  76       // make room for the hashtables for packages and groups
  77       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  78          return;
  79
  80       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  81       if (unlikely(idxVerSysName == 0))
  82          return;
  83       Cache.HeaderP->VerSysName = idxVerSysName;
  84       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  85       if (unlikely(idxArchitecture == 0))
  86          return;
  87       Cache.HeaderP->Architecture = idxArchitecture;
  88
  89       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  90       if (archs.size() > 1)
  91       {
  92          std::vector<std::string>::const_iterator a = archs.begin();
  93          std::string list = *a;
  94          for (++a; a != archs.end(); ++a)
  95             list.append(",").append(*a);
  96          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  97          if (unlikely(idxArchitectures == 0))
  98             return;
  99          Cache.HeaderP->SetArchitectures(idxArchitectures);
 100       }
 101       else
 102          Cache.HeaderP->SetArchitectures(idxArchitecture);
 103
 104       Cache.ReMap();
 105    }
 106    else
 107    {
 108       // Map directly from the existing file
 109       Cache.ReMap();
 110       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 111       if (Cache.VS != _system->VS)
 112       {
 113          _error->Error(_("Cache has an incompatible versioning system"));
 114          return;
 115       }
 116    }
 117
 118    Cache.HeaderP->Dirty = true;
 119    Map.Sync(0,sizeof(pkgCache::Header));
 120 }
 121                                                                         /*}}}*/
 122 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* We sync the data then unset the dirty flag in two steps so as to
 125    advoid a problem during a crash */
 126 pkgCacheGenerator::~pkgCacheGenerator()
 127 {
 128    if (_error->PendingError() == true)
 129       return;
 130    if (Map.Sync() == false)
 131       return;
 132
 133    Cache.HeaderP->Dirty = false;
 134    Cache.HeaderP->CacheFileSize = Map.Size();
 135    Map.Sync(0,sizeof(pkgCache::Header));
 136 }
 137                                                                         /*}}}*/
 138 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 139    if (oldMap == newMap)
 140       return;
 141
 142    if (_config->FindB("Debug::pkgCacheGen", false))
 143       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 144
 145    Cache.ReMap(false);
 146
 147    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 148    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 149
 150    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 151         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 152       (*i)->ReMap(oldMap, newMap);
 153    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 154         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 155       (*i)->ReMap(oldMap, newMap);
 156    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174 }                                                                       /*}}}*/
 175 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 176 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 177                                         const unsigned long &Len) {
 178    void const * const oldMap = Map.Data();
 179    map_stringitem_t const index = Map.WriteString(String, Len);
 180    if (index != 0)
 181       ReMap(oldMap, Map.Data());
 182    return index;
 183 }
 184                                                                         /*}}}*/
 185 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 186 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 187    void const * const oldMap = Map.Data();
 188    map_stringitem_t const index = Map.WriteString(String);
 189    if (index != 0)
 190       ReMap(oldMap, Map.Data());
 191    return index;
 192 }
 193                                                                         /*}}}*/
 194 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 195    void const * const oldMap = Map.Data();
 196    map_pointer_t const index = Map.Allocate(size);
 197    if (index != 0)
 198       ReMap(oldMap, Map.Data());
 199    return index;
 200 }
 201                                                                         /*}}}*/
 202 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 203 // ---------------------------------------------------------------------
 204 /* This provides the generation of the entries in the cache. Each loop
 205    goes through a single package record from the underlying parse engine. */
 206 bool pkgCacheGenerator::MergeList(ListParser &List,
 207                                   pkgCache::VerIterator *OutVer)
 208 {
 209    List.Owner = this;
 210
 211    unsigned int Counter = 0;
 212    while (List.Step() == true)
 213    {
 214       string const PackageName = List.Package();
 215       if (PackageName.empty() == true)
 216          return false;
 217
 218       Counter++;
 219       if (Counter % 100 == 0 && Progress != 0)
 220          Progress->Progress(List.Offset());
 221
 222       string Arch = List.Architecture();
 223       string const Version = List.Version();
 224       if (Version.empty() == true && Arch.empty() == true)
 225       {
 226          // package descriptions
 227          if (MergeListGroup(List, PackageName) == false)
 228             return false;
 229          continue;
 230       }
 231
 232       if (Arch.empty() == true)
 233       {
 234          // use the pseudo arch 'none' for arch-less packages
 235          Arch = "none";
 236          /* We might built a SingleArchCache here, which we don't want to blow up
 237             just for these :none packages to a proper MultiArchCache, so just ensure
 238             that we have always a native package structure first for SingleArch */
 239          pkgCache::PkgIterator NP;
 240          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 241          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 242          // TRANSLATOR: The first placeholder is a package name,
 243          // the other two should be copied verbatim as they include debug info
 244          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 245                               PackageName.c_str(), "NewPackage", 0);
 246       }
 247
 248       // Get a pointer to the package structure
 249       pkgCache::PkgIterator Pkg;
 250       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 251       if (NewPackage(Pkg, PackageName, Arch) == false)
 252          // TRANSLATOR: The first placeholder is a package name,
 253          // the other two should be copied verbatim as they include debug info
 254          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 255                               PackageName.c_str(), "NewPackage", 1);
 256
 257
 258       if (Version.empty() == true)
 259       {
 260          if (MergeListPackage(List, Pkg) == false)
 261             return false;
 262       }
 263       else
 264       {
 265          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 266             return false;
 267       }
 268
 269       if (OutVer != 0)
 270       {
 271          FoundFileDeps |= List.HasFileDeps();
 272          return true;
 273       }
 274    }
 275
 276    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 277       return _error->Error(_("Wow, you exceeded the number of package "
 278                              "names this APT is capable of."));
 279    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 280       return _error->Error(_("Wow, you exceeded the number of versions "
 281                              "this APT is capable of."));
 282    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 283       return _error->Error(_("Wow, you exceeded the number of descriptions "
 284                              "this APT is capable of."));
 285    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 286       return _error->Error(_("Wow, you exceeded the number of dependencies "
 287                              "this APT is capable of."));
 288
 289    FoundFileDeps |= List.HasFileDeps();
 290    return true;
 291 }
 292 // CacheGenerator::MergeListGroup                                       /*{{{*/
 293 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 294 {
 295    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 296    // a group has no data on it's own, only packages have it but these
 297    // stanzas like this come from Translation- files to add descriptions,
 298    // but without a version we don't need a description for it…
 299    if (Grp.end() == true)
 300       return true;
 301    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 302
 303    pkgCache::PkgIterator Pkg;
 304    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 305    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 306       if (MergeListPackage(List, Pkg) == false)
 307          return false;
 308
 309    return true;
 310 }
 311                                                                         /*}}}*/
 312 // CacheGenerator::MergeListPackage                                     /*{{{*/
 313 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 314 {
 315    // we first process the package, then the descriptions
 316    // (for deb this package processing is in fact a no-op)
 317    pkgCache::VerIterator Ver(Cache);
 318    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 319    if (List.UsePackage(Pkg, Ver) == false)
 320       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 321                            Pkg.Name(), "UsePackage", 1);
 322
 323    // Find the right version to write the description
 324    MD5SumValue CurMd5 = List.Description_md5();
 325    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 326       return true;
 327    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 328    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 329    {
 330       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 331
 332       // a version can only have one md5 describing it
 333       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 334          continue;
 335
 336       map_stringitem_t md5idx = VerDesc->md5sum;
 337       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 338       {
 339          // don't add a new description if we have one for the given
 340          // md5 && language
 341          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 342             continue;
 343
 344          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 345       }
 346
 347       // we can stop here as all "same" versions will share the description
 348       break;
 349    }
 350
 351    return true;
 352 }
 353                                                                         /*}}}*/
 354 // CacheGenerator::MergeListVersion                                     /*{{{*/
 355 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 356                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 357 {
 358    pkgCache::VerIterator Ver = Pkg.VersionList();
 359    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 360    map_pointer_t *LastVer = &Pkg->VersionList;
 361    void const * oldMap = Map.Data();
 362
 363    unsigned short const Hash = List.VersionHash();
 364    if (Ver.end() == false)
 365    {
 366       /* We know the list is sorted so we use that fact in the search.
 367          Insertion of new versions is done with correct sorting */
 368       int Res = 1;
 369       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 370       {
 371          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 372          // Version is higher as current version - insert here
 373          if (Res > 0)
 374             break;
 375          // Versionstrings are equal - is hash also equal?
 376          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 377             break;
 378          // proceed with the next till we have either the right
 379          // or we found another version (which will be lower)
 380       }
 381
 382       /* We already have a version for this item, record that we saw it */
 383       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 384       {
 385          if (List.UsePackage(Pkg,Ver) == false)
 386             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 387                                  Pkg.Name(), "UsePackage", 2);
 388
 389          if (NewFileVer(Ver,List) == false)
 390             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 391                                  Pkg.Name(), "NewFileVer", 1);
 392
 393          // Read only a single record and return
 394          if (OutVer != 0)
 395          {
 396             *OutVer = Ver;
 397             return true;
 398          }
 399
 400          return true;
 401       }
 402    }
 403
 404    // Add a new version
 405    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 406    if (verindex == 0 && _error->PendingError())
 407       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 408                            Pkg.Name(), "NewVersion", 1);
 409
 410    if (oldMap != Map.Data())
 411          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 412    *LastVer = verindex;
 413
 414    if (unlikely(List.NewVersion(Ver) == false))
 415       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 416                            Pkg.Name(), "NewVersion", 2);
 417
 418    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 419       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 420                            Pkg.Name(), "UsePackage", 3);
 421
 422    if (unlikely(NewFileVer(Ver,List) == false))
 423       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 424                            Pkg.Name(), "NewFileVer", 2);
 425
 426    pkgCache::GrpIterator Grp = Pkg.Group();
 427    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 428
 429    /* If it is the first version of this package we need to add implicit
 430       Multi-Arch dependencies to all other package versions in the group now -
 431       otherwise we just add them for this new version */
 432    if (Pkg.VersionList()->NextVer == 0)
 433    {
 434       pkgCache::PkgIterator P = Grp.PackageList();
 435       Dynamic<pkgCache::PkgIterator> DynP(P);
 436       for (; P.end() != true; P = Grp.NextPkg(P))
 437       {
 438          if (P->ID == Pkg->ID)
 439             continue;
 440          pkgCache::VerIterator V = P.VersionList();
 441          Dynamic<pkgCache::VerIterator> DynV(V);
 442          for (; V.end() != true; ++V)
 443             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 444                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 445                                     Pkg.Name(), "AddImplicitDepends", 1);
 446       }
 447       /* :none packages are packages without an architecture. They are forbidden by
 448          debian-policy, so usually they will only be in (old) dpkg status files -
 449          and dpkg will complain about them - and are pretty rare. We therefore do
 450          usually not create conflicts while the parent is created, but only if a :none
 451          package (= the target) appears. This creates incorrect dependencies on :none
 452          for architecture-specific dependencies on the package we copy from, but we
 453          will ignore this bug as architecture-specific dependencies are only allowed
 454          in jessie and until then the :none packages should be extinct (hopefully).
 455          In other words: This should work long enough to allow graceful removal of
 456          these packages, it is not supposed to allow users to keep using them … */
 457       if (strcmp(Pkg.Arch(), "none") == 0)
 458       {
 459          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 460          if (M.end() == false && Pkg != M)
 461          {
 462             pkgCache::DepIterator D = M.RevDependsList();
 463             Dynamic<pkgCache::DepIterator> DynD(D);
 464             for (; D.end() == false; ++D)
 465             {
 466                if ((D->Type != pkgCache::Dep::Conflicts &&
 467                     D->Type != pkgCache::Dep::DpkgBreaks &&
 468                     D->Type != pkgCache::Dep::Replaces) ||
 469                    D.ParentPkg().Group() == Grp)
 470                   continue;
 471
 472                map_pointer_t *OldDepLast = NULL;
 473                pkgCache::VerIterator ConVersion = D.ParentVer();
 474                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 475                // duplicate the Conflicts/Breaks/Replaces for :none arch
 476                NewDepends(Pkg, ConVersion, D->Version,
 477                      D->CompareOp, D->Type, OldDepLast);
 478             }
 479          }
 480       }
 481    }
 482    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 483       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 484                            Pkg.Name(), "AddImplicitDepends", 2);
 485
 486    // Read only a single record and return
 487    if (OutVer != 0)
 488    {
 489       *OutVer = Ver;
 490       return true;
 491    }
 492
 493    /* Record the Description(s) based on their master md5sum */
 494    MD5SumValue CurMd5 = List.Description_md5();
 495    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 496       return true;
 497
 498    /* Before we add a new description we first search in the group for
 499       a version with a description of the same MD5 - if so we reuse this
 500       description group instead of creating our own for this version */
 501    for (pkgCache::PkgIterator P = Grp.PackageList();
 502         P.end() == false; P = Grp.NextPkg(P))
 503    {
 504       for (pkgCache::VerIterator V = P.VersionList();
 505            V.end() == false; ++V)
 506       {
 507          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 508             continue;
 509          Ver->DescriptionList = V->DescriptionList;
 510       }
 511    }
 512
 513    // We haven't found reusable descriptions, so add the first description(s)
 514    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 515    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 516    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 517       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 518          return false;
 519    return true;
 520 }
 521                                                                         /*}}}*/
 522 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 523 {
 524    pkgCache::DescIterator Desc;
 525    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 526
 527    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 528    if (unlikely(descindex == 0 && _error->PendingError()))
 529       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 530             Ver.ParentPkg().Name(), "NewDescription", 1);
 531
 532    md5idx = Desc->md5sum;
 533    Desc->ParentPkg = Ver.ParentPkg().Index();
 534
 535    // we add at the end, so that the start is constant as we need
 536    // that to be able to efficiently share these lists
 537    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 538    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 539    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 540    *LastNextDesc = descindex;
 541
 542    if (NewFileDesc(Desc,List) == false)
 543       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 544             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 545
 546    return true;
 547 }
 548                                                                         /*}}}*/
 549                                                                         /*}}}*/
 550 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* If we found any file depends while parsing the main list we need to
 553    resolve them. Since it is undesired to load the entire list of files
 554    into the cache as virtual packages we do a two stage effort. MergeList
 555    identifies the file depends and this creates Provdies for them by
 556    re-parsing all the indexs. */
 557 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 558 {
 559    List.Owner = this;
 560
 561    unsigned int Counter = 0;
 562    while (List.Step() == true)
 563    {
 564       string PackageName = List.Package();
 565       if (PackageName.empty() == true)
 566          return false;
 567       string Version = List.Version();
 568       if (Version.empty() == true)
 569          continue;
 570
 571       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 572       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 573       if (Pkg.end() == true)
 574          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 575                                 PackageName.c_str(), "FindPkg", 1);
 576       Counter++;
 577       if (Counter % 100 == 0 && Progress != 0)
 578          Progress->Progress(List.Offset());
 579
 580       unsigned short Hash = List.VersionHash();
 581       pkgCache::VerIterator Ver = Pkg.VersionList();
 582       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 583       for (; Ver.end() == false; ++Ver)
 584       {
 585          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 586          {
 587             if (List.CollectFileProvides(Cache,Ver) == false)
 588                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 589                                     PackageName.c_str(), "CollectFileProvides", 1);
 590             break;
 591          }
 592       }
 593
 594       if (Ver.end() == true)
 595          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 596    }
 597
 598    return true;
 599 }
 600                                                                         /*}}}*/
 601 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 602 // ---------------------------------------------------------------------
 603 /* This creates a new group structure and adds it to the hash table */
 604 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 605 {
 606    Grp = Cache.FindGrp(Name);
 607    if (Grp.end() == false)
 608       return true;
 609
 610    // Get a structure
 611    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 612    if (unlikely(Group == 0))
 613       return false;
 614
 615    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 616    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 617    if (unlikely(idxName == 0))
 618       return false;
 619    Grp->Name = idxName;
 620
 621    // Insert it into the hash table
 622    unsigned long const Hash = Cache.Hash(Name);
 623    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 624    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 625       insertAt = &(Cache.GrpP + *insertAt)->Next;
 626    Grp->Next = *insertAt;
 627    *insertAt = Group;
 628
 629    Grp->ID = Cache.HeaderP->GroupCount++;
 630    return true;
 631 }
 632                                                                         /*}}}*/
 633 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 634 // ---------------------------------------------------------------------
 635 /* This creates a new package structure and adds it to the hash table */
 636 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 637                                         const string &Arch) {
 638    pkgCache::GrpIterator Grp;
 639    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 640    if (unlikely(NewGroup(Grp, Name) == false))
 641       return false;
 642
 643    Pkg = Grp.FindPkg(Arch);
 644       if (Pkg.end() == false)
 645          return true;
 646
 647    // Get a structure
 648    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 649    if (unlikely(Package == 0))
 650       return false;
 651    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 652
 653    // Set the name, arch and the ID
 654    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 655    Pkg->Group = Grp.Index();
 656    // all is mapped to the native architecture
 657    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 658    if (unlikely(idxArch == 0))
 659       return false;
 660    Pkg->Arch = idxArch;
 661    Pkg->ID = Cache.HeaderP->PackageCount++;
 662
 663    // Insert the package into our package list
 664    if (Grp->FirstPackage == 0) // the group is new
 665    {
 666       Grp->FirstPackage = Package;
 667       // Insert it into the hash table
 668       map_id_t const Hash = Cache.Hash(Name);
 669       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 670       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 671          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 672       Pkg->NextPackage = *insertAt;
 673       *insertAt = Package;
 674    }
 675    else // Group the Packages together
 676    {
 677       // but first get implicit provides done
 678       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 679       {
 680          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 681          if (M.end() == false)
 682             for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 683             {
 684                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 685                   continue;
 686                pkgCache::VerIterator Ver = Prv.OwnerVer();
 687                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 688                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 689                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 690                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 691                      return false;
 692             }
 693
 694          for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 695             for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
 696                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 697                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 698                      return false;
 699       }
 700       // and negative dependencies, don't forget negative dependencies
 701       {
 702          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 703          if (M.end() == false)
 704             for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 705             {
 706                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 707                   continue;
 708                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 709                      Dep->Type != pkgCache::Dep::Replaces)
 710                   continue;
 711                pkgCache::VerIterator Ver = Dep.ParentVer();
 712                map_pointer_t * unused = NULL;
 713                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 714                   return false;
 715             }
 716       }
 717
 718       // this package is the new last package
 719       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 720       Pkg->NextPackage = LastPkg->NextPackage;
 721       LastPkg->NextPackage = Package;
 722    }
 723    Grp->LastPackage = Package;
 724    return true;
 725 }
 726                                                                         /*}}}*/
 727 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 728 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 729                                            pkgCache::PkgIterator &P,
 730                                            pkgCache::VerIterator &V)
 731 {
 732    // copy P.Arch() into a string here as a cache remap
 733    // in NewDepends() later may alter the pointer location
 734    string Arch = P.Arch() == NULL ? "" : P.Arch();
 735    map_pointer_t *OldDepLast = NULL;
 736    /* MultiArch handling introduces a lot of implicit Dependencies:
 737       - MultiArch: same → Co-Installable if they have the same version
 738       - All others conflict with all other group members */
 739    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 740    pkgCache::PkgIterator D = G.PackageList();
 741    Dynamic<pkgCache::PkgIterator> DynD(D);
 742    map_stringitem_t const VerStrIdx = V->VerStr;
 743    for (; D.end() != true; D = G.NextPkg(D))
 744    {
 745       if (Arch == D.Arch() || D->VersionList == 0)
 746          continue;
 747       /* We allow only one installed arch at the time
 748          per group, therefore each group member conflicts
 749          with all other group members */
 750       if (coInstall == true)
 751       {
 752          // Replaces: ${self}:other ( << ${binary:Version})
 753          NewDepends(D, V, VerStrIdx,
 754                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 755                     OldDepLast);
 756          // Breaks: ${self}:other (!= ${binary:Version})
 757          NewDepends(D, V, VerStrIdx,
 758                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 759                     OldDepLast);
 760       } else {
 761          // Conflicts: ${self}:other
 762          NewDepends(D, V, 0,
 763                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 764                     OldDepLast);
 765       }
 766    }
 767    return true;
 768 }
 769 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 770                                            pkgCache::PkgIterator &D)
 771 {
 772    /* MultiArch handling introduces a lot of implicit Dependencies:
 773       - MultiArch: same → Co-Installable if they have the same version
 774       - All others conflict with all other group members */
 775    map_pointer_t *OldDepLast = NULL;
 776    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 777    if (coInstall == true)
 778    {
 779       map_stringitem_t const VerStrIdx = V->VerStr;
 780       // Replaces: ${self}:other ( << ${binary:Version})
 781       NewDepends(D, V, VerStrIdx,
 782                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 783                  OldDepLast);
 784       // Breaks: ${self}:other (!= ${binary:Version})
 785       NewDepends(D, V, VerStrIdx,
 786                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 787                  OldDepLast);
 788    } else {
 789       // Conflicts: ${self}:other
 790       NewDepends(D, V, 0,
 791                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 792                  OldDepLast);
 793    }
 794    return true;
 795 }
 796
 797                                                                         /*}}}*/
 798 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 799 // ---------------------------------------------------------------------
 800 /* */
 801 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 802                                    ListParser &List)
 803 {
 804    if (CurrentFile == 0)
 805       return true;
 806
 807    // Get a structure
 808    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 809    if (VerFile == 0)
 810       return false;
 811
 812    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 813    VF->File = CurrentFile - Cache.PkgFileP;
 814
 815    // Link it to the end of the list
 816    map_pointer_t *Last = &Ver->FileList;
 817    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 818       Last = &V->NextFile;
 819    VF->NextFile = *Last;
 820    *Last = VF.Index();
 821
 822    VF->Offset = List.Offset();
 823    VF->Size = List.Size();
 824    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 825       Cache.HeaderP->MaxVerFileSize = VF->Size;
 826    Cache.HeaderP->VerFileCount++;
 827
 828    return true;
 829 }
 830                                                                         /*}}}*/
 831 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 832 // ---------------------------------------------------------------------
 833 /* This puts a version structure in the linked list */
 834 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 835                                             const string &VerStr,
 836                                             map_pointer_t const ParentPkg,
 837                                             unsigned short const Hash,
 838                                             map_pointer_t const Next)
 839 {
 840    // Get a structure
 841    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 842    if (Version == 0)
 843       return 0;
 844
 845    // Fill it in
 846    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 847    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 848    Ver->NextVer = Next;
 849    Ver->ParentPkg = ParentPkg;
 850    Ver->Hash = Hash;
 851    Ver->ID = Cache.HeaderP->VersionCount++;
 852
 853    // try to find the version string in the group for reuse
 854    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 855    pkgCache::GrpIterator Grp = Pkg.Group();
 856    if (Pkg.end() == false && Grp.end() == false)
 857    {
 858       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 859       {
 860          if (Pkg == P)
 861             continue;
 862          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 863          {
 864             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 865             if (cmp == 0)
 866             {
 867                Ver->VerStr = V->VerStr;
 868                return Version;
 869             }
 870             else if (cmp < 0)
 871                break;
 872          }
 873       }
 874    }
 875    // haven't found the version string, so create
 876    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 877    if (unlikely(idxVerStr == 0))
 878       return 0;
 879    Ver->VerStr = idxVerStr;
 880    return Version;
 881 }
 882                                                                         /*}}}*/
 883 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 884 // ---------------------------------------------------------------------
 885 /* */
 886 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 887                                    ListParser &List)
 888 {
 889    if (CurrentFile == 0)
 890       return true;
 891
 892    // Get a structure
 893    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 894    if (DescFile == 0)
 895       return false;
 896
 897    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 898    DF->File = CurrentFile - Cache.PkgFileP;
 899
 900    // Link it to the end of the list
 901    map_pointer_t *Last = &Desc->FileList;
 902    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 903       Last = &D->NextFile;
 904
 905    DF->NextFile = *Last;
 906    *Last = DF.Index();
 907
 908    DF->Offset = List.Offset();
 909    DF->Size = List.Size();
 910    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 911       Cache.HeaderP->MaxDescFileSize = DF->Size;
 912    Cache.HeaderP->DescFileCount++;
 913
 914    return true;
 915 }
 916                                                                         /*}}}*/
 917 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 918 // ---------------------------------------------------------------------
 919 /* This puts a description structure in the linked list */
 920 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 921                                             const string &Lang,
 922                                             const MD5SumValue &md5sum,
 923                                             map_stringitem_t const idxmd5str)
 924 {
 925    // Get a structure
 926    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 927    if (Description == 0)
 928       return 0;
 929
 930    // Fill it in
 931    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 932    Desc->ID = Cache.HeaderP->DescriptionCount++;
 933    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 934    if (unlikely(idxlanguage_code == 0))
 935       return 0;
 936    Desc->language_code = idxlanguage_code;
 937
 938    if (idxmd5str != 0)
 939       Desc->md5sum = idxmd5str;
 940    else
 941    {
 942       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 943       if (unlikely(idxmd5sum == 0))
 944          return 0;
 945       Desc->md5sum = idxmd5sum;
 946    }
 947
 948    return Description;
 949 }
 950                                                                         /*}}}*/
 951 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 952 // ---------------------------------------------------------------------
 953 /* This creates a dependency element in the tree. It is linked to the
 954    version and to the package that it is pointing to. */
 955 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 956                                    pkgCache::VerIterator &Ver,
 957                                    map_pointer_t const Version,
 958                                    uint8_t const Op,
 959                                    uint8_t const Type,
 960                                    map_pointer_t* &OldDepLast)
 961 {
 962    void const * const oldMap = Map.Data();
 963    // Get a structure
 964    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 965    if (unlikely(Dependency == 0))
 966       return false;
 967
 968    bool isDuplicate = false;
 969    map_pointer_t DependencyData = 0;
 970    map_pointer_t PreviousData = 0;
 971    if (Pkg->RevDepends != 0)
 972    {
 973       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 974       DependencyData = L->DependencyData;
 975       do {
 976          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 977          if (Version > D->Version)
 978             break;
 979          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 980          {
 981             isDuplicate = true;
 982             break;
 983          }
 984          PreviousData = DependencyData;
 985          DependencyData = D->NextData;
 986       } while (DependencyData != 0);
 987    }
 988
 989    if (isDuplicate == false)
 990    {
 991       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 992       if (unlikely(DependencyData == 0))
 993         return false;
 994    }
 995
 996    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 997    Link->ParentVer = Ver.Index();
 998    Link->DependencyData = DependencyData;
 999    Link->ID = Cache.HeaderP->DependsCount++;
1000
1001    pkgCache::DepIterator Dep(Cache, Link);
1002    if (isDuplicate == false)
1003    {
1004       Dep->Type = Type;
1005       Dep->CompareOp = Op;
1006       Dep->Version = Version;
1007       Dep->Package = Pkg.Index();
1008       ++Cache.HeaderP->DependsDataCount;
1009       if (PreviousData != 0)
1010       {
1011          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1012          Dep->NextData = D->NextData;
1013          D->NextData = DependencyData;
1014       }
1015       else if (Pkg->RevDepends != 0)
1016       {
1017          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1018          Dep->NextData = D->DependencyData;
1019       }
1020    }
1021
1022    if (isDuplicate == true || PreviousData != 0)
1023    {
1024       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1025       Link->NextRevDepends = L->NextRevDepends;
1026       L->NextRevDepends = Dependency;
1027    }
1028    else
1029    {
1030       Link->NextRevDepends = Pkg->RevDepends;
1031       Pkg->RevDepends = Dependency;
1032    }
1033
1034
1035    // Do we know where to link the Dependency to?
1036    if (OldDepLast == NULL)
1037    {
1038       OldDepLast = &Ver->DependsList;
1039       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1040          OldDepLast = &D->NextDepends;
1041    } else if (oldMap != Map.Data())
1042       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1043
1044    Dep->NextDepends = *OldDepLast;
1045    *OldDepLast = Dependency;
1046    OldDepLast = &Dep->NextDepends;
1047    return true;
1048 }
1049                                                                         /*}}}*/
1050 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1051 // ---------------------------------------------------------------------
1052 /* This creates a Group and the Package to link this dependency to if
1053    needed and handles also the caching of the old endpoint */
1054 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
1055                                                const string &PackageName,
1056                                                const string &Arch,
1057                                                const string &Version,
1058                                                uint8_t const Op,
1059                                                uint8_t const Type)
1060 {
1061    pkgCache::GrpIterator Grp;
1062    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1063    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1064       return false;
1065
1066    // Is it a file dependency?
1067    if (unlikely(PackageName[0] == '/'))
1068       FoundFileDeps = true;
1069
1070    map_stringitem_t idxVersion = 0;
1071    if (Version.empty() == false)
1072    {
1073       int const CmpOp = Op & 0x0F;
1074       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1075       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1076          idxVersion = Ver->VerStr;
1077
1078       if (idxVersion == 0)
1079       {
1080          idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1081          if (unlikely(idxVersion == 0))
1082             return false;
1083       }
1084    }
1085
1086    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1087          Type == pkgCache::Dep::Conflicts ||
1088          Type == pkgCache::Dep::Replaces);
1089
1090    pkgCache::PkgIterator Pkg;
1091    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1092    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1093    {
1094       // Locate the target package
1095       Pkg = Grp.FindPkg(Arch);
1096       if (Pkg.end() == true) {
1097          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1098             return false;
1099       }
1100
1101       /* Caching the old end point speeds up generation substantially */
1102       if (OldDepVer != Ver) {
1103          OldDepLast = NULL;
1104          OldDepVer = Ver;
1105       }
1106
1107       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1108    }
1109    else
1110    {
1111       /* Caching the old end point speeds up generation substantially */
1112       if (OldDepVer != Ver) {
1113          OldDepLast = NULL;
1114          OldDepVer = Ver;
1115       }
1116
1117       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1118       {
1119          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1120                return false;
1121       }
1122    }
1123    return true;
1124 }
1125                                                                         /*}}}*/
1126 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1127 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
1128                                                 const string &PkgName,
1129                                                 const string &PkgArch,
1130                                                 const string &Version,
1131                                                 uint8_t const Flags)
1132 {
1133    pkgCache const &Cache = Owner->Cache;
1134
1135    // We do not add self referencing provides
1136    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1137         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1138       return true;
1139
1140    // Locate the target package
1141    pkgCache::PkgIterator Pkg;
1142    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1143    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1144       return false;
1145
1146    map_stringitem_t idxProvideVersion = 0;
1147    if (Version.empty() == false) {
1148       idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1149       if (unlikely(idxProvideVersion == 0))
1150          return false;
1151    }
1152    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1153 }
1154 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1155                                     pkgCache::PkgIterator &Pkg,
1156                                     map_pointer_t const ProvideVersion,
1157                                     uint8_t const Flags)
1158 {
1159    // Get a structure
1160    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1161    if (unlikely(Provides == 0))
1162       return false;
1163    ++Cache.HeaderP->ProvidesCount;
1164
1165    // Fill it in
1166    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1167    Prv->Version = Ver.Index();
1168    Prv->ProvideVersion = ProvideVersion;
1169    Prv->Flags = Flags;
1170    Prv->NextPkgProv = Ver->ProvidesList;
1171    Ver->ProvidesList = Prv.Index();
1172
1173    // Link it to the package
1174    Prv->ParentPkg = Pkg.Index();
1175    Prv->NextProvides = Pkg->ProvidesList;
1176    Pkg->ProvidesList = Prv.Index();
1177    return true;
1178 }
1179                                                                         /*}}}*/
1180 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1181 bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1182                                 string const &Version, uint8_t const Flags) {
1183    pkgCache &Cache = Owner->Cache;
1184    pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1185    if (Grp.end() == true)
1186       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1187    else
1188    {
1189       map_stringitem_t idxProvideVersion = 0;
1190       if (Version.empty() == false) {
1191          idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version);
1192          if (unlikely(idxProvideVersion == 0))
1193             return false;
1194       }
1195
1196       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1197       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1198       pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1199       for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1200       {
1201          if (isImplicit && OwnerPkg == Pkg)
1202             continue;
1203          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1204             continue;
1205          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1206             return false;
1207       }
1208    }
1209    return true;
1210 }
1211                                                                         /*}}}*/
1212 bool pkgCacheListParser::SameVersion(unsigned short const Hash,         /*{{{*/
1213       pkgCache::VerIterator const &Ver)
1214 {
1215    return Hash == Ver->Hash;
1216 }
1217                                                                         /*}}}*/
1218 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1219 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1220                                    unsigned long Flags)
1221 {
1222    if (File.empty() && Site.empty())
1223    {
1224       CurrentRlsFile = NULL;
1225       return true;
1226    }
1227
1228    // Get some space for the structure
1229    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1230    if (unlikely(idxFile == 0))
1231       return false;
1232    CurrentRlsFile = Cache.RlsFileP + idxFile;
1233
1234    // Fill it in
1235    map_stringitem_t const idxFileName = WriteStringInMap(File);
1236    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1237    if (unlikely(idxFileName == 0 || idxSite == 0))
1238       return false;
1239    CurrentRlsFile->FileName = idxFileName;
1240    CurrentRlsFile->Site = idxSite;
1241    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1242    CurrentRlsFile->Flags = Flags;
1243    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1244    RlsFileName = File;
1245    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1246    Cache.HeaderP->ReleaseFileCount++;
1247
1248    return true;
1249 }
1250                                                                         /*}}}*/
1251 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1252 // ---------------------------------------------------------------------
1253 /* This is used to select which file is to be associated with all newly
1254    added versions. The caller is responsible for setting the IMS fields. */
1255 bool pkgCacheGenerator::SelectFile(std::string const &File,
1256                                    pkgIndexFile const &Index,
1257                                    std::string const &Architecture,
1258                                    std::string const &Component,
1259                                    unsigned long const Flags)
1260 {
1261    // Get some space for the structure
1262    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1263    if (unlikely(idxFile == 0))
1264       return false;
1265    CurrentFile = Cache.PkgFileP + idxFile;
1266
1267    // Fill it in
1268    map_stringitem_t const idxFileName = WriteStringInMap(File);
1269    if (unlikely(idxFileName == 0))
1270       return false;
1271    CurrentFile->FileName = idxFileName;
1272    CurrentFile->NextFile = Cache.HeaderP->FileList;
1273    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1274    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1275    if (unlikely(idxIndexType == 0))
1276       return false;
1277    CurrentFile->IndexType = idxIndexType;
1278    if (Architecture.empty())
1279       CurrentFile->Architecture = 0;
1280    else
1281    {
1282       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1283       if (unlikely(arch == 0))
1284          return false;
1285       CurrentFile->Architecture = arch;
1286    }
1287    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1288    if (unlikely(component == 0))
1289       return false;
1290    CurrentFile->Component = component;
1291    CurrentFile->Flags = Flags;
1292    if (CurrentRlsFile != NULL)
1293       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1294    else
1295       CurrentFile->Release = 0;
1296    PkgFileName = File;
1297    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1298    Cache.HeaderP->PackageFileCount++;
1299
1300    if (Progress != 0)
1301       Progress->SubProgress(Index.Size());
1302    return true;
1303 }
1304                                                                         /*}}}*/
1305 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1306 // ---------------------------------------------------------------------
1307 /* This is used to create handles to strings. Given the same text it
1308    always returns the same number */
1309 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1310                                                  unsigned int Size)
1311 {
1312    std::string const key(S, Size);
1313
1314    std::map<std::string,map_stringitem_t> * strings;
1315    switch(type) {
1316       case MIXED: strings = &strMixed; break;
1317       case PKGNAME: strings = &strPkgNames; break;
1318       case VERSIONNUMBER: strings = &strVersions; break;
1319       case SECTION: strings = &strSections; break;
1320       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1321    }
1322
1323    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1324    if (item != strings->end())
1325       return item->second;
1326
1327    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1328    strings->insert(std::make_pair(key, idxString));
1329    return idxString;
1330 }
1331                                                                         /*}}}*/
1332 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1333 // ---------------------------------------------------------------------
1334 /* This just verifies that each file in the list of index files exists,
1335    has matching attributes with the cache and the cache does not have
1336    any extra files. */
1337 static bool CheckValidity(const string &CacheFile,
1338                           pkgSourceList &List,
1339                           FileIterator const Start,
1340                           FileIterator const End,
1341                           MMap **OutMap = 0)
1342 {
1343    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1344    // No file, certainly invalid
1345    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1346    {
1347       if (Debug == true)
1348          std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
1349       return false;
1350    }
1351
1352    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1353    {
1354       if (Debug == true)
1355          std::clog << "sources.list is newer than the cache" << std::endl;
1356       return false;
1357    }
1358
1359    // Map it
1360    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1361    SPtr<MMap> Map = new MMap(CacheF,0);
1362    pkgCache Cache(Map);
1363    if (_error->PendingError() == true || Map->Size() == 0)
1364    {
1365       if (Debug == true)
1366          std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
1367       _error->Discard();
1368       return false;
1369    }
1370
1371    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1372    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1373    std::vector<pkgIndexFile *> Files;
1374    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1375    {
1376       if (Debug == true)
1377          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1378       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1379       if (RlsFile.end() == true)
1380       {
1381          if (Debug == true)
1382             std::clog << "FindInCache returned end-Pointer" << std::endl;
1383          return false;
1384       }
1385
1386       RlsVisited[RlsFile->ID] = true;
1387       if (Debug == true)
1388          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1389
1390       std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
1391       std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
1392             [](pkgIndexFile const * const I) { return I->HasPackages(); });
1393    }
1394    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1395       if (RlsVisited[I] == false)
1396       {
1397          if (Debug == true)
1398             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1399          return false;
1400       }
1401
1402    std::copy(Start, End, std::back_inserter(Files));
1403
1404    /* Now we check every index file, see if it is in the cache,
1405       verify the IMS data and check that it is on the disk too.. */
1406    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1407    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1408    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1409    {
1410       if (Debug == true)
1411          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1412       if ((*PkgFile)->Exists() == false)
1413       {
1414          if (Debug == true)
1415             std::clog << "file doesn't exist" << std::endl;
1416          continue;
1417       }
1418
1419       // FindInCache is also expected to do an IMS check.
1420       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1421       if (File.end() == true)
1422       {
1423          if (Debug == true)
1424             std::clog << "FindInCache returned end-Pointer" << std::endl;
1425          return false;
1426       }
1427
1428       Visited[File->ID] = true;
1429       if (Debug == true)
1430          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1431    }
1432
1433    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1434       if (Visited[I] == false)
1435       {
1436          if (Debug == true)
1437             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1438          return false;
1439       }
1440
1441    if (_error->PendingError() == true)
1442    {
1443       if (Debug == true)
1444       {
1445          std::clog << "Validity failed because of pending errors:" << std::endl;
1446          _error->DumpErrors();
1447       }
1448       _error->Discard();
1449       return false;
1450    }
1451
1452    if (OutMap != 0)
1453       *OutMap = Map.UnGuard();
1454    return true;
1455 }
1456                                                                         /*}}}*/
1457 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1458 // ---------------------------------------------------------------------
1459 /* Size is kind of an abstract notion that is only used for the progress
1460    meter */
1461 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1462 {
1463    map_filesize_t TotalSize = 0;
1464    if (List !=  NULL)
1465    {
1466       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1467       {
1468          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1469          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1470             if ((*j)->HasPackages() == true)
1471                TotalSize += (*j)->Size();
1472       }
1473    }
1474
1475    for (; Start < End; ++Start)
1476    {
1477       if ((*Start)->HasPackages() == false)
1478          continue;
1479       TotalSize += (*Start)->Size();
1480    }
1481    return TotalSize;
1482 }
1483                                                                         /*}}}*/
1484 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1485 static bool BuildCache(pkgCacheGenerator &Gen,
1486                        OpProgress * const Progress,
1487                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1488                        pkgSourceList const * const List,
1489                        FileIterator const Start, FileIterator const End)
1490 {
1491    std::vector<pkgIndexFile *> Files;
1492    bool const HasFileDeps = Gen.HasFileDeps();
1493    bool mergeFailure = false;
1494
1495    auto const indexFileMerge = [&](pkgIndexFile * const I) {
1496       if (HasFileDeps)
1497          Files.push_back(I);
1498
1499       if (I->HasPackages() == false || mergeFailure)
1500          return;
1501
1502       if (I->Exists() == false)
1503          return;
1504
1505       if (I->FindInCache(Gen.GetCache()).end() == false)
1506       {
1507          _error->Warning("Duplicate sources.list entry %s",
1508                I->Describe().c_str());
1509          return;
1510       }
1511
1512       map_filesize_t const Size = I->Size();
1513       if (Progress != NULL)
1514          Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
1515       CurrentSize += Size;
1516
1517       if (I->Merge(Gen,Progress) == false)
1518          mergeFailure = true;
1519    };
1520
1521    if (List !=  NULL)
1522    {
1523       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1524       {
1525          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1526          {
1527             _error->Warning("Duplicate sources.list entry %s",
1528                   (*i)->Describe().c_str());
1529             continue;
1530          }
1531
1532          if ((*i)->Merge(Gen, Progress) == false)
1533             return false;
1534
1535          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1536          if (Indexes != NULL)
1537             std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
1538          if (mergeFailure)
1539             return false;
1540       }
1541    }
1542
1543    if (Start != End)
1544    {
1545       Gen.SelectReleaseFile("", "");
1546       std::for_each(Start, End, indexFileMerge);
1547       if (mergeFailure)
1548          return false;
1549    }
1550
1551    if (HasFileDeps == true)
1552    {
1553       if (Progress != NULL)
1554          Progress->Done();
1555       TotalSize = ComputeSize(List, Start, End);
1556       CurrentSize = 0;
1557       for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
1558       {
1559          map_filesize_t Size = (*I)->Size();
1560          if (Progress != NULL)
1561             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1562          CurrentSize += Size;
1563          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1564             return false;
1565       }
1566    }
1567
1568    return true;
1569 }
1570                                                                         /*}}}*/
1571 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1572 // ---------------------------------------------------------------------
1573 /* This makes sure that the status cache (the cache that has all
1574    index files from the sources list and all local ones) is ready
1575    to be mmaped. If OutMap is not zero then a MMap object representing
1576    the cache will be stored there. This is pretty much mandetory if you
1577    are using AllowMem. AllowMem lets the function be run as non-root
1578    where it builds the cache 'fast' into a memory buffer. */
1579 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
1580 {
1581    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1582    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1583    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1584    Flags |= MMap::Moveable;
1585    if (_config->FindB("APT::Cache-Fallback", false) == true)
1586       Flags |= MMap::Fallback;
1587    if (CacheF != NULL)
1588       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1589    else
1590       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1591 }
1592 static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
1593       std::string const &FileName)
1594 {
1595    FileFd SCacheF(FileName, FileFd::WriteAtomic);
1596    if (_error->PendingError() == true)
1597       return false;
1598
1599    fchmod(SCacheF.Fd(),0644);
1600
1601    // Write out the main data
1602    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1603       return _error->Error(_("IO Error saving source cache"));
1604    SCacheF.Sync();
1605
1606    // Write out the proper header
1607    Gen->GetCache().HeaderP->Dirty = false;
1608    if (SCacheF.Seek(0) == false ||
1609          SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
1610       return _error->Error(_("IO Error saving source cache"));
1611    Gen->GetCache().HeaderP->Dirty = true;
1612    SCacheF.Sync();
1613    return true;
1614 }
1615 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
1616       SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
1617 {
1618    Map = CreateDynamicMMap(NULL, 0);
1619    FileFd CacheF(FileName, FileFd::ReadOnly);
1620    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
1621    if ((alloc == 0 && _error->PendingError())
1622          || CacheF.Read((unsigned char *)Map->Data() + alloc,
1623             CacheF.Size()) == false)
1624       return false;
1625    Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1626    return true;
1627 }
1628 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1629                         MMap **OutMap, bool AllowMem)
1630    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1631 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1632                         MMap **OutMap,bool AllowMem)
1633 {
1634    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1635
1636    std::vector<pkgIndexFile *> Files;
1637    if (_system->AddStatusFiles(Files) == false)
1638       return false;
1639
1640    // Decide if we can write to the files..
1641    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1642    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1643
1644    // ensure the cache directory exists
1645    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1646    {
1647       string dir = _config->FindDir("Dir::Cache");
1648       size_t const len = dir.size();
1649       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1650          dir = dir.substr(0, len - 5);
1651       if (CacheFile.empty() == false)
1652          CreateDirectory(dir, flNotFile(CacheFile));
1653       if (SrcCacheFile.empty() == false)
1654          CreateDirectory(dir, flNotFile(SrcCacheFile));
1655    }
1656
1657    if (Progress != NULL)
1658       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1659
1660    bool pkgcache_fine = false;
1661    bool srcpkgcache_fine = false;
1662    bool volatile_fine = List.GetVolatileFiles().empty();
1663
1664    if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
1665    {
1666       if (Debug == true)
1667          std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
1668       pkgcache_fine = true;
1669       srcpkgcache_fine = true;
1670    }
1671    if (pkgcache_fine == false)
1672    {
1673       if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
1674       {
1675          if (Debug == true)
1676             std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
1677          srcpkgcache_fine = true;
1678       }
1679    }
1680
1681    if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
1682    {
1683       if (Progress != NULL)
1684          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1685       return true;
1686    }
1687
1688    bool Writeable = false;
1689    if (srcpkgcache_fine == false || pkgcache_fine == false)
1690    {
1691       if (CacheFile.empty() == false)
1692          Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1693       else if (SrcCacheFile.empty() == false)
1694          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1695
1696       if (Debug == true)
1697          std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1698
1699       if (Writeable == false && AllowMem == false)
1700       {
1701          if (CacheFile.empty() == false)
1702             return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1703          else if (SrcCacheFile.empty() == false)
1704             return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
1705          else
1706             return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
1707       }
1708    }
1709
1710    // At this point we know we need to construct something, so get storage ready
1711    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1712    if (Debug == true)
1713       std::clog << "Open memory Map (not filebased)" << std::endl;
1714
1715    std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
1716    map_filesize_t CurrentSize = 0;
1717    std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
1718    map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
1719    if (srcpkgcache_fine == true && pkgcache_fine == false)
1720    {
1721       if (Debug == true)
1722          std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
1723       if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
1724          return false;
1725       srcpkgcache_fine = true;
1726       TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
1727    }
1728    else if (srcpkgcache_fine == false)
1729    {
1730       if (Debug == true)
1731          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1732       Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
1733
1734       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
1735       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
1736                Files.end(),Files.end()) == false)
1737          return false;
1738
1739       if (Writeable == true && SrcCacheFile.empty() == false)
1740          if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
1741             return false;
1742    }
1743
1744    if (pkgcache_fine == false)
1745    {
1746       if (Debug == true)
1747          std::clog << "Building status cache in pkgcache.bin now" << std::endl;
1748       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1749                Files.begin(), Files.end()) == false)
1750          return false;
1751
1752       if (Writeable == true && CacheFile.empty() == false)
1753          if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
1754             return false;
1755    }
1756
1757    if (Debug == true)
1758       std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
1759
1760    if (volatile_fine == false)
1761    {
1762       if (Gen == nullptr)
1763       {
1764          if (Debug == true)
1765             std::clog << "Populate new MMap with cachefile contents" << std::endl;
1766          if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
1767             return false;
1768       }
1769
1770       Files = List.GetVolatileFiles();
1771       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
1772                Files.begin(), Files.end()) == false)
1773          return false;
1774    }
1775
1776    if (OutMap != nullptr)
1777       *OutMap = Map.UnGuard();
1778
1779    if (Debug == true)
1780       std::clog << "Everything is ready for shipping" << std::endl;
1781    return true;
1782 }
1783                                                                         /*}}}*/
1784 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1785 // ---------------------------------------------------------------------
1786 /* */
1787 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1788    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1789 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1790 {
1791    std::vector<pkgIndexFile *> Files;
1792    if (_system->AddStatusFiles(Files) == false)
1793       return false;
1794
1795    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
1796    map_filesize_t CurrentSize = 0;
1797    map_filesize_t TotalSize = 0;
1798
1799    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1800
1801    // Build the status cache
1802    if (Progress != NULL)
1803       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1804    pkgCacheGenerator Gen(Map.Get(),Progress);
1805    if (_error->PendingError() == true)
1806       return false;
1807    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1808                   Files.begin(), Files.end()) == false)
1809       return false;
1810
1811    if (_error->PendingError() == true)
1812       return false;
1813    *OutMap = Map.UnGuard();
1814
1815    return true;
1816 }
1817                                                                         /*}}}*/
1818 // IsDuplicateDescription                                               /*{{{*/
1819 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1820                             MD5SumValue const &CurMd5, std::string const &CurLang)
1821 {
1822    // Descriptions in the same link-list have all the same md5
1823    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1824       return false;
1825    for (; Desc.end() == false; ++Desc)
1826       if (Desc.LanguageCode() == CurLang)
1827          return true;
1828    return false;
1829 }
1830                                                                         /*}}}*/
1831 // CacheGenerator::FinishCache                                          /*{{{*/
1832 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1833 {
1834    return true;
1835 }
1836                                                                         /*}}}*/
1837
1838 pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
1839 pkgCacheListParser::~pkgCacheListParser() {}