apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                      CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
  58 {
  59    if (_error->PendingError() == true)
  60       return;
  61
  62    if (Map.Size() == 0)
  63    {
  64       // Setup the map interface..
  65       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  66       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  67          return;
  68
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70
  71       // Starting header
  72       *Cache.HeaderP = pkgCache::Header();
  73
  74       // make room for the hashtables for packages and groups
  75       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  76          return;
  77
  78       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  79       if (unlikely(idxVerSysName == 0))
  80          return;
  81       Cache.HeaderP->VerSysName = idxVerSysName;
  82       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  83       if (unlikely(idxArchitecture == 0))
  84          return;
  85       Cache.HeaderP->Architecture = idxArchitecture;
  86
  87       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  88       if (archs.size() > 1)
  89       {
  90          std::vector<std::string>::const_iterator a = archs.begin();
  91          std::string list = *a;
  92          for (++a; a != archs.end(); ++a)
  93             list.append(",").append(*a);
  94          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  95          if (unlikely(idxArchitectures == 0))
  96             return;
  97          Cache.HeaderP->SetArchitectures(idxArchitectures);
  98       }
  99       else
 100          Cache.HeaderP->SetArchitectures(idxArchitecture);
 101
 102       Cache.ReMap();
 103    }
 104    else
 105    {
 106       // Map directly from the existing file
 107       Cache.ReMap();
 108       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 109       if (Cache.VS != _system->VS)
 110       {
 111          _error->Error(_("Cache has an incompatible versioning system"));
 112          return;
 113       }
 114    }
 115
 116    Cache.HeaderP->Dirty = true;
 117    Map.Sync(0,sizeof(pkgCache::Header));
 118 }
 119                                                                         /*}}}*/
 120 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 121 // ---------------------------------------------------------------------
 122 /* We sync the data then unset the dirty flag in two steps so as to
 123    advoid a problem during a crash */
 124 pkgCacheGenerator::~pkgCacheGenerator()
 125 {
 126    if (_error->PendingError() == true)
 127       return;
 128    if (Map.Sync() == false)
 129       return;
 130
 131    Cache.HeaderP->Dirty = false;
 132    Cache.HeaderP->CacheFileSize = Map.Size();
 133    Map.Sync(0,sizeof(pkgCache::Header));
 134 }
 135                                                                         /*}}}*/
 136 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 137    if (oldMap == newMap)
 138       return;
 139
 140    if (_config->FindB("Debug::pkgCacheGen", false))
 141       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 142
 143    Cache.ReMap(false);
 144
 145    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 146    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 147
 148    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 149         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 150       (*i)->ReMap(oldMap, newMap);
 151    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 152         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 153       (*i)->ReMap(oldMap, newMap);
 154    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 155         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 156       (*i)->ReMap(oldMap, newMap);
 157    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 158         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 159       (*i)->ReMap(oldMap, newMap);
 160    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 161         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 162       (*i)->ReMap(oldMap, newMap);
 163    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 164         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 165       (*i)->ReMap(oldMap, newMap);
 166    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 168       (*i)->ReMap(oldMap, newMap);
 169    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 170         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 171       (*i)->ReMap(oldMap, newMap);
 172 }                                                                       /*}}}*/
 173 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 174 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 175                                         const unsigned long &Len) {
 176    void const * const oldMap = Map.Data();
 177    map_stringitem_t const index = Map.WriteString(String, Len);
 178    if (index != 0)
 179       ReMap(oldMap, Map.Data());
 180    return index;
 181 }
 182                                                                         /*}}}*/
 183 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 184 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 185    void const * const oldMap = Map.Data();
 186    map_stringitem_t const index = Map.WriteString(String);
 187    if (index != 0)
 188       ReMap(oldMap, Map.Data());
 189    return index;
 190 }
 191                                                                         /*}}}*/
 192 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 193    void const * const oldMap = Map.Data();
 194    map_pointer_t const index = Map.Allocate(size);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This provides the generation of the entries in the cache. Each loop
 203    goes through a single package record from the underlying parse engine. */
 204 bool pkgCacheGenerator::MergeList(ListParser &List,
 205                                   pkgCache::VerIterator *OutVer)
 206 {
 207    List.Owner = this;
 208
 209    unsigned int Counter = 0;
 210    while (List.Step() == true)
 211    {
 212       string const PackageName = List.Package();
 213       if (PackageName.empty() == true)
 214          return false;
 215
 216       Counter++;
 217       if (Counter % 100 == 0 && Progress != 0)
 218          Progress->Progress(List.Offset());
 219
 220       string Arch = List.Architecture();
 221       string const Version = List.Version();
 222       if (Version.empty() == true && Arch.empty() == true)
 223       {
 224          // package descriptions
 225          if (MergeListGroup(List, PackageName) == false)
 226             return false;
 227          continue;
 228       }
 229
 230       if (Arch.empty() == true)
 231       {
 232          // use the pseudo arch 'none' for arch-less packages
 233          Arch = "none";
 234          /* We might built a SingleArchCache here, which we don't want to blow up
 235             just for these :none packages to a proper MultiArchCache, so just ensure
 236             that we have always a native package structure first for SingleArch */
 237          pkgCache::PkgIterator NP;
 238          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 239          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 240          // TRANSLATOR: The first placeholder is a package name,
 241          // the other two should be copied verbatim as they include debug info
 242          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 243                               PackageName.c_str(), "NewPackage", 0);
 244       }
 245
 246       // Get a pointer to the package structure
 247       pkgCache::PkgIterator Pkg;
 248       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 249       if (NewPackage(Pkg, PackageName, Arch) == false)
 250          // TRANSLATOR: The first placeholder is a package name,
 251          // the other two should be copied verbatim as they include debug info
 252          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 253                               PackageName.c_str(), "NewPackage", 1);
 254
 255
 256       if (Version.empty() == true)
 257       {
 258          if (MergeListPackage(List, Pkg) == false)
 259             return false;
 260       }
 261       else
 262       {
 263          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 264             return false;
 265       }
 266
 267       if (OutVer != 0)
 268       {
 269          FoundFileDeps |= List.HasFileDeps();
 270          return true;
 271       }
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    FoundFileDeps |= List.HasFileDeps();
 288    return true;
 289 }
 290 // CacheGenerator::MergeListGroup                                       /*{{{*/
 291 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 292 {
 293    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 294    // a group has no data on it's own, only packages have it but these
 295    // stanzas like this come from Translation- files to add descriptions,
 296    // but without a version we don't need a description for it…
 297    if (Grp.end() == true)
 298       return true;
 299    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 300
 301    pkgCache::PkgIterator Pkg;
 302    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 303    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 304       if (MergeListPackage(List, Pkg) == false)
 305          return false;
 306
 307    return true;
 308 }
 309                                                                         /*}}}*/
 310 // CacheGenerator::MergeListPackage                                     /*{{{*/
 311 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 312 {
 313    // we first process the package, then the descriptions
 314    // (for deb this package processing is in fact a no-op)
 315    pkgCache::VerIterator Ver(Cache);
 316    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 317    if (List.UsePackage(Pkg, Ver) == false)
 318       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 319                            Pkg.Name(), "UsePackage", 1);
 320
 321    // Find the right version to write the description
 322    MD5SumValue CurMd5 = List.Description_md5();
 323    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 324       return true;
 325    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 326    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 327    {
 328       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 329
 330       // a version can only have one md5 describing it
 331       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 332          continue;
 333
 334       map_stringitem_t md5idx = VerDesc->md5sum;
 335       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 336       {
 337          // don't add a new description if we have one for the given
 338          // md5 && language
 339          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 340             continue;
 341
 342          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 343       }
 344
 345       // we can stop here as all "same" versions will share the description
 346       break;
 347    }
 348
 349    return true;
 350 }
 351                                                                         /*}}}*/
 352 // CacheGenerator::MergeListVersion                                     /*{{{*/
 353 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 354                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 355 {
 356    pkgCache::VerIterator Ver = Pkg.VersionList();
 357    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 358    map_pointer_t *LastVer = &Pkg->VersionList;
 359    void const * oldMap = Map.Data();
 360
 361    unsigned short const Hash = List.VersionHash();
 362    if (Ver.end() == false)
 363    {
 364       /* We know the list is sorted so we use that fact in the search.
 365          Insertion of new versions is done with correct sorting */
 366       int Res = 1;
 367       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 368       {
 369          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 370          // Version is higher as current version - insert here
 371          if (Res > 0)
 372             break;
 373          // Versionstrings are equal - is hash also equal?
 374          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 375             break;
 376          // proceed with the next till we have either the right
 377          // or we found another version (which will be lower)
 378       }
 379
 380       /* We already have a version for this item, record that we saw it */
 381       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 382       {
 383          if (List.UsePackage(Pkg,Ver) == false)
 384             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 385                                  Pkg.Name(), "UsePackage", 2);
 386
 387          if (NewFileVer(Ver,List) == false)
 388             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 389                                  Pkg.Name(), "NewFileVer", 1);
 390
 391          // Read only a single record and return
 392          if (OutVer != 0)
 393          {
 394             *OutVer = Ver;
 395             return true;
 396          }
 397
 398          return true;
 399       }
 400    }
 401
 402    // Add a new version
 403    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 404    if (verindex == 0 && _error->PendingError())
 405       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 406                            Pkg.Name(), "NewVersion", 1);
 407
 408    if (oldMap != Map.Data())
 409          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 410    *LastVer = verindex;
 411
 412    if (unlikely(List.NewVersion(Ver) == false))
 413       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 414                            Pkg.Name(), "NewVersion", 2);
 415
 416    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 417       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 418                            Pkg.Name(), "UsePackage", 3);
 419
 420    if (unlikely(NewFileVer(Ver,List) == false))
 421       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 422                            Pkg.Name(), "NewFileVer", 2);
 423
 424    pkgCache::GrpIterator Grp = Pkg.Group();
 425    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 426
 427    /* If it is the first version of this package we need to add implicit
 428       Multi-Arch dependencies to all other package versions in the group now -
 429       otherwise we just add them for this new version */
 430    if (Pkg.VersionList()->NextVer == 0)
 431    {
 432       pkgCache::PkgIterator P = Grp.PackageList();
 433       Dynamic<pkgCache::PkgIterator> DynP(P);
 434       for (; P.end() != true; P = Grp.NextPkg(P))
 435       {
 436          if (P->ID == Pkg->ID)
 437             continue;
 438          pkgCache::VerIterator V = P.VersionList();
 439          Dynamic<pkgCache::VerIterator> DynV(V);
 440          for (; V.end() != true; ++V)
 441             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 442                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 443                                     Pkg.Name(), "AddImplicitDepends", 1);
 444       }
 445       /* :none packages are packages without an architecture. They are forbidden by
 446          debian-policy, so usually they will only be in (old) dpkg status files -
 447          and dpkg will complain about them - and are pretty rare. We therefore do
 448          usually not create conflicts while the parent is created, but only if a :none
 449          package (= the target) appears. This creates incorrect dependencies on :none
 450          for architecture-specific dependencies on the package we copy from, but we
 451          will ignore this bug as architecture-specific dependencies are only allowed
 452          in jessie and until then the :none packages should be extinct (hopefully).
 453          In other words: This should work long enough to allow graceful removal of
 454          these packages, it is not supposed to allow users to keep using them … */
 455       if (strcmp(Pkg.Arch(), "none") == 0)
 456       {
 457          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 458          if (M.end() == false && Pkg != M)
 459          {
 460             pkgCache::DepIterator D = M.RevDependsList();
 461             Dynamic<pkgCache::DepIterator> DynD(D);
 462             for (; D.end() == false; ++D)
 463             {
 464                if ((D->Type != pkgCache::Dep::Conflicts &&
 465                     D->Type != pkgCache::Dep::DpkgBreaks &&
 466                     D->Type != pkgCache::Dep::Replaces) ||
 467                    D.ParentPkg().Group() == Grp)
 468                   continue;
 469
 470                map_pointer_t *OldDepLast = NULL;
 471                pkgCache::VerIterator ConVersion = D.ParentVer();
 472                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 473                // duplicate the Conflicts/Breaks/Replaces for :none arch
 474                NewDepends(Pkg, ConVersion, D->Version,
 475                      D->CompareOp, D->Type, OldDepLast);
 476             }
 477          }
 478       }
 479    }
 480    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482                            Pkg.Name(), "AddImplicitDepends", 2);
 483
 484    // Read only a single record and return
 485    if (OutVer != 0)
 486    {
 487       *OutVer = Ver;
 488       return true;
 489    }
 490
 491    /* Record the Description(s) based on their master md5sum */
 492    MD5SumValue CurMd5 = List.Description_md5();
 493    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 494       return true;
 495
 496    /* Before we add a new description we first search in the group for
 497       a version with a description of the same MD5 - if so we reuse this
 498       description group instead of creating our own for this version */
 499    for (pkgCache::PkgIterator P = Grp.PackageList();
 500         P.end() == false; P = Grp.NextPkg(P))
 501    {
 502       for (pkgCache::VerIterator V = P.VersionList();
 503            V.end() == false; ++V)
 504       {
 505          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 506             continue;
 507          Ver->DescriptionList = V->DescriptionList;
 508       }
 509    }
 510
 511    // We haven't found reusable descriptions, so add the first description(s)
 512    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 513    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 514    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 515       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 516          return false;
 517    return true;
 518 }
 519                                                                         /*}}}*/
 520 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 521 {
 522    pkgCache::DescIterator Desc;
 523    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 524
 525    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 526    if (unlikely(descindex == 0 && _error->PendingError()))
 527       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 528             Ver.ParentPkg().Name(), "NewDescription", 1);
 529
 530    md5idx = Desc->md5sum;
 531    Desc->ParentPkg = Ver.ParentPkg().Index();
 532
 533    // we add at the end, so that the start is constant as we need
 534    // that to be able to efficiently share these lists
 535    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 536    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 537    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 538    *LastNextDesc = descindex;
 539
 540    if (NewFileDesc(Desc,List) == false)
 541       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 542             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 543
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547                                                                         /*}}}*/
 548 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* If we found any file depends while parsing the main list we need to
 551    resolve them. Since it is undesired to load the entire list of files
 552    into the cache as virtual packages we do a two stage effort. MergeList
 553    identifies the file depends and this creates Provdies for them by
 554    re-parsing all the indexs. */
 555 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 556 {
 557    List.Owner = this;
 558
 559    unsigned int Counter = 0;
 560    while (List.Step() == true)
 561    {
 562       string PackageName = List.Package();
 563       if (PackageName.empty() == true)
 564          return false;
 565       string Version = List.Version();
 566       if (Version.empty() == true)
 567          continue;
 568
 569       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 570       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 571       if (Pkg.end() == true)
 572          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 573                                 PackageName.c_str(), "FindPkg", 1);
 574       Counter++;
 575       if (Counter % 100 == 0 && Progress != 0)
 576          Progress->Progress(List.Offset());
 577
 578       unsigned short Hash = List.VersionHash();
 579       pkgCache::VerIterator Ver = Pkg.VersionList();
 580       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 581       for (; Ver.end() == false; ++Ver)
 582       {
 583          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 584          {
 585             if (List.CollectFileProvides(Cache,Ver) == false)
 586                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 587                                     PackageName.c_str(), "CollectFileProvides", 1);
 588             break;
 589          }
 590       }
 591
 592       if (Ver.end() == true)
 593          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 594    }
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This creates a new group structure and adds it to the hash table */
 602 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 603 {
 604    Grp = Cache.FindGrp(Name);
 605    if (Grp.end() == false)
 606       return true;
 607
 608    // Get a structure
 609    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 610    if (unlikely(Group == 0))
 611       return false;
 612
 613    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 614    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 615    if (unlikely(idxName == 0))
 616       return false;
 617    Grp->Name = idxName;
 618
 619    // Insert it into the hash table
 620    unsigned long const Hash = Cache.Hash(Name);
 621    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 622    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 623       insertAt = &(Cache.GrpP + *insertAt)->Next;
 624    Grp->Next = *insertAt;
 625    *insertAt = Group;
 626
 627    Grp->ID = Cache.HeaderP->GroupCount++;
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* This creates a new package structure and adds it to the hash table */
 634 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 635                                         const string &Arch) {
 636    pkgCache::GrpIterator Grp;
 637    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 638    if (unlikely(NewGroup(Grp, Name) == false))
 639       return false;
 640
 641    Pkg = Grp.FindPkg(Arch);
 642       if (Pkg.end() == false)
 643          return true;
 644
 645    // Get a structure
 646    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 647    if (unlikely(Package == 0))
 648       return false;
 649    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 650
 651    // Set the name, arch and the ID
 652    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 653    Pkg->Group = Grp.Index();
 654    // all is mapped to the native architecture
 655    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 656    if (unlikely(idxArch == 0))
 657       return false;
 658    Pkg->Arch = idxArch;
 659    Pkg->ID = Cache.HeaderP->PackageCount++;
 660
 661    // Insert the package into our package list
 662    if (Grp->FirstPackage == 0) // the group is new
 663    {
 664       Grp->FirstPackage = Package;
 665       // Insert it into the hash table
 666       map_id_t const Hash = Cache.Hash(Name);
 667       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 668       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 669          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 670       Pkg->NextPackage = *insertAt;
 671       *insertAt = Package;
 672    }
 673    else // Group the Packages together
 674    {
 675       // but first get implicit provides done
 676       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 677       {
 678          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 679          if (M.end() == false)
 680             for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 681             {
 682                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 683                   continue;
 684                pkgCache::VerIterator Ver = Prv.OwnerVer();
 685                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 686                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 687                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 688                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 689                      return false;
 690             }
 691
 692          for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 693             for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
 694                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 695                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 696                      return false;
 697       }
 698
 699       // this package is the new last package
 700       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 701       Pkg->NextPackage = LastPkg->NextPackage;
 702       LastPkg->NextPackage = Package;
 703    }
 704    Grp->LastPackage = Package;
 705    return true;
 706 }
 707                                                                         /*}}}*/
 708 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 709 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 710                                            pkgCache::PkgIterator &P,
 711                                            pkgCache::VerIterator &V)
 712 {
 713    // copy P.Arch() into a string here as a cache remap
 714    // in NewDepends() later may alter the pointer location
 715    string Arch = P.Arch() == NULL ? "" : P.Arch();
 716    map_pointer_t *OldDepLast = NULL;
 717    /* MultiArch handling introduces a lot of implicit Dependencies:
 718       - MultiArch: same → Co-Installable if they have the same version
 719       - All others conflict with all other group members */
 720    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 721    pkgCache::PkgIterator D = G.PackageList();
 722    Dynamic<pkgCache::PkgIterator> DynD(D);
 723    map_stringitem_t const VerStrIdx = V->VerStr;
 724    for (; D.end() != true; D = G.NextPkg(D))
 725    {
 726       if (Arch == D.Arch() || D->VersionList == 0)
 727          continue;
 728       /* We allow only one installed arch at the time
 729          per group, therefore each group member conflicts
 730          with all other group members */
 731       if (coInstall == true)
 732       {
 733          // Replaces: ${self}:other ( << ${binary:Version})
 734          NewDepends(D, V, VerStrIdx,
 735                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 736                     OldDepLast);
 737          // Breaks: ${self}:other (!= ${binary:Version})
 738          NewDepends(D, V, VerStrIdx,
 739                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 740                     OldDepLast);
 741       } else {
 742          // Conflicts: ${self}:other
 743          NewDepends(D, V, 0,
 744                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 745                     OldDepLast);
 746       }
 747    }
 748    return true;
 749 }
 750 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 751                                            pkgCache::PkgIterator &D)
 752 {
 753    /* MultiArch handling introduces a lot of implicit Dependencies:
 754       - MultiArch: same → Co-Installable if they have the same version
 755       - All others conflict with all other group members */
 756    map_pointer_t *OldDepLast = NULL;
 757    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 758    if (coInstall == true)
 759    {
 760       map_stringitem_t const VerStrIdx = V->VerStr;
 761       // Replaces: ${self}:other ( << ${binary:Version})
 762       NewDepends(D, V, VerStrIdx,
 763                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 764                  OldDepLast);
 765       // Breaks: ${self}:other (!= ${binary:Version})
 766       NewDepends(D, V, VerStrIdx,
 767                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 768                  OldDepLast);
 769    } else {
 770       // Conflicts: ${self}:other
 771       NewDepends(D, V, 0,
 772                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 773                  OldDepLast);
 774    }
 775    return true;
 776 }
 777
 778                                                                         /*}}}*/
 779 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 780 // ---------------------------------------------------------------------
 781 /* */
 782 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 783                                    ListParser &List)
 784 {
 785    if (CurrentFile == 0)
 786       return true;
 787
 788    // Get a structure
 789    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 790    if (VerFile == 0)
 791       return false;
 792
 793    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 794    VF->File = CurrentFile - Cache.PkgFileP;
 795
 796    // Link it to the end of the list
 797    map_pointer_t *Last = &Ver->FileList;
 798    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 799       Last = &V->NextFile;
 800    VF->NextFile = *Last;
 801    *Last = VF.Index();
 802
 803    VF->Offset = List.Offset();
 804    VF->Size = List.Size();
 805    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 806       Cache.HeaderP->MaxVerFileSize = VF->Size;
 807    Cache.HeaderP->VerFileCount++;
 808
 809    return true;
 810 }
 811                                                                         /*}}}*/
 812 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 813 // ---------------------------------------------------------------------
 814 /* This puts a version structure in the linked list */
 815 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 816                                             const string &VerStr,
 817                                             map_pointer_t const ParentPkg,
 818                                             unsigned short const Hash,
 819                                             map_pointer_t const Next)
 820 {
 821    // Get a structure
 822    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 823    if (Version == 0)
 824       return 0;
 825
 826    // Fill it in
 827    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 828    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 829    Ver->NextVer = Next;
 830    Ver->ParentPkg = ParentPkg;
 831    Ver->Hash = Hash;
 832    Ver->ID = Cache.HeaderP->VersionCount++;
 833
 834    // try to find the version string in the group for reuse
 835    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 836    pkgCache::GrpIterator Grp = Pkg.Group();
 837    if (Pkg.end() == false && Grp.end() == false)
 838    {
 839       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 840       {
 841          if (Pkg == P)
 842             continue;
 843          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 844          {
 845             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 846             if (cmp == 0)
 847             {
 848                Ver->VerStr = V->VerStr;
 849                return Version;
 850             }
 851             else if (cmp < 0)
 852                break;
 853          }
 854       }
 855    }
 856    // haven't found the version string, so create
 857    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 858    if (unlikely(idxVerStr == 0))
 859       return 0;
 860    Ver->VerStr = idxVerStr;
 861    return Version;
 862 }
 863                                                                         /*}}}*/
 864 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 865 // ---------------------------------------------------------------------
 866 /* */
 867 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 868                                    ListParser &List)
 869 {
 870    if (CurrentFile == 0)
 871       return true;
 872
 873    // Get a structure
 874    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 875    if (DescFile == 0)
 876       return false;
 877
 878    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 879    DF->File = CurrentFile - Cache.PkgFileP;
 880
 881    // Link it to the end of the list
 882    map_pointer_t *Last = &Desc->FileList;
 883    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 884       Last = &D->NextFile;
 885
 886    DF->NextFile = *Last;
 887    *Last = DF.Index();
 888
 889    DF->Offset = List.Offset();
 890    DF->Size = List.Size();
 891    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 892       Cache.HeaderP->MaxDescFileSize = DF->Size;
 893    Cache.HeaderP->DescFileCount++;
 894
 895    return true;
 896 }
 897                                                                         /*}}}*/
 898 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 899 // ---------------------------------------------------------------------
 900 /* This puts a description structure in the linked list */
 901 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 902                                             const string &Lang,
 903                                             const MD5SumValue &md5sum,
 904                                             map_stringitem_t const idxmd5str)
 905 {
 906    // Get a structure
 907    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 908    if (Description == 0)
 909       return 0;
 910
 911    // Fill it in
 912    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 913    Desc->ID = Cache.HeaderP->DescriptionCount++;
 914    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 915    if (unlikely(idxlanguage_code == 0))
 916       return 0;
 917    Desc->language_code = idxlanguage_code;
 918
 919    if (idxmd5str != 0)
 920       Desc->md5sum = idxmd5str;
 921    else
 922    {
 923       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 924       if (unlikely(idxmd5sum == 0))
 925          return 0;
 926       Desc->md5sum = idxmd5sum;
 927    }
 928
 929    return Description;
 930 }
 931                                                                         /*}}}*/
 932 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 933 // ---------------------------------------------------------------------
 934 /* This creates a dependency element in the tree. It is linked to the
 935    version and to the package that it is pointing to. */
 936 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 937                                    pkgCache::VerIterator &Ver,
 938                                    string const &Version,
 939                                    uint8_t const Op,
 940                                    uint8_t const Type,
 941                                    map_stringitem_t* &OldDepLast)
 942 {
 943    map_stringitem_t index = 0;
 944    if (Version.empty() == false)
 945    {
 946       int const CmpOp = Op & 0x0F;
 947       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 948       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 949          index = Ver->VerStr;
 950
 951       if (index == 0)
 952       {
 953          void const * const oldMap = Map.Data();
 954          index = StoreString(VERSIONNUMBER, Version);
 955          if (unlikely(index == 0))
 956             return false;
 957          if (OldDepLast != 0 && oldMap != Map.Data())
 958             OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 959       }
 960    }
 961    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 962 }
 963 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 964                                    pkgCache::VerIterator &Ver,
 965                                    map_pointer_t const Version,
 966                                    uint8_t const Op,
 967                                    uint8_t const Type,
 968                                    map_pointer_t* &OldDepLast)
 969 {
 970    void const * const oldMap = Map.Data();
 971    // Get a structure
 972    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 973    if (unlikely(Dependency == 0))
 974       return false;
 975
 976    bool isDuplicate = false;
 977    map_pointer_t DependencyData = 0;
 978    map_pointer_t PreviousData = 0;
 979    if (Pkg->RevDepends != 0)
 980    {
 981       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 982       DependencyData = L->DependencyData;
 983       do {
 984          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 985          if (Version > D->Version)
 986             break;
 987          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 988          {
 989             isDuplicate = true;
 990             break;
 991          }
 992          PreviousData = DependencyData;
 993          DependencyData = D->NextData;
 994       } while (DependencyData != 0);
 995    }
 996
 997    if (isDuplicate == false)
 998    {
 999       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
1000       if (unlikely(DependencyData == 0))
1001         return false;
1002    }
1003
1004    pkgCache::Dependency * Link = Cache.DepP + Dependency;
1005    Link->ParentVer = Ver.Index();
1006    Link->DependencyData = DependencyData;
1007    Link->ID = Cache.HeaderP->DependsCount++;
1008
1009    pkgCache::DepIterator Dep(Cache, Link);
1010    if (isDuplicate == false)
1011    {
1012       Dep->Type = Type;
1013       Dep->CompareOp = Op;
1014       Dep->Version = Version;
1015       Dep->Package = Pkg.Index();
1016       ++Cache.HeaderP->DependsDataCount;
1017       if (PreviousData != 0)
1018       {
1019          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1020          Dep->NextData = D->NextData;
1021          D->NextData = DependencyData;
1022       }
1023       else if (Pkg->RevDepends != 0)
1024       {
1025          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1026          Dep->NextData = D->DependencyData;
1027       }
1028    }
1029
1030    if (isDuplicate == true || PreviousData != 0)
1031    {
1032       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1033       Link->NextRevDepends = L->NextRevDepends;
1034       L->NextRevDepends = Dependency;
1035    }
1036    else
1037    {
1038       Link->NextRevDepends = Pkg->RevDepends;
1039       Pkg->RevDepends = Dependency;
1040    }
1041
1042
1043    // Do we know where to link the Dependency to?
1044    if (OldDepLast == NULL)
1045    {
1046       OldDepLast = &Ver->DependsList;
1047       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1048          OldDepLast = &D->NextDepends;
1049    } else if (oldMap != Map.Data())
1050       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1051
1052    Dep->NextDepends = *OldDepLast;
1053    *OldDepLast = Dependency;
1054    OldDepLast = &Dep->NextDepends;
1055    return true;
1056 }
1057                                                                         /*}}}*/
1058 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1059 // ---------------------------------------------------------------------
1060 /* This creates a Group and the Package to link this dependency to if
1061    needed and handles also the caching of the old endpoint */
1062 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
1063                                                const string &PackageName,
1064                                                const string &Arch,
1065                                                const string &Version,
1066                                                uint8_t const Op,
1067                                                uint8_t const Type)
1068 {
1069    pkgCache::GrpIterator Grp;
1070    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1071    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1072       return false;
1073
1074    // Locate the target package
1075    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
1076    // we don't create 'none' packages and their dependencies if we can avoid it …
1077    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
1078       return true;
1079    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1080    if (Pkg.end() == true) {
1081       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1082          return false;
1083    }
1084
1085    // Is it a file dependency?
1086    if (unlikely(PackageName[0] == '/'))
1087       FoundFileDeps = true;
1088
1089    /* Caching the old end point speeds up generation substantially */
1090    if (OldDepVer != Ver) {
1091       OldDepLast = NULL;
1092       OldDepVer = Ver;
1093    }
1094
1095    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1096 }
1097                                                                         /*}}}*/
1098 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1099 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1100                                                 const string &PkgName,
1101                                                 const string &PkgArch,
1102                                                 const string &Version,
1103                                                 uint8_t const Flags)
1104 {
1105    pkgCache const &Cache = Owner->Cache;
1106
1107    // We do not add self referencing provides
1108    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1109         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1110       return true;
1111
1112    // Locate the target package
1113    pkgCache::PkgIterator Pkg;
1114    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1115    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1116       return false;
1117
1118    map_stringitem_t idxProvideVersion = 0;
1119    if (Version.empty() == false) {
1120       idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1121       if (unlikely(idxProvideVersion == 0))
1122          return false;
1123    }
1124    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1125 }
1126 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1127                                     pkgCache::PkgIterator &Pkg,
1128                                     map_pointer_t const ProvideVersion,
1129                                     uint8_t const Flags)
1130 {
1131    // Get a structure
1132    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1133    if (unlikely(Provides == 0))
1134       return false;
1135    ++Cache.HeaderP->ProvidesCount;
1136
1137    // Fill it in
1138    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1139    Prv->Version = Ver.Index();
1140    Prv->ProvideVersion = ProvideVersion;
1141    Prv->Flags = Flags;
1142    Prv->NextPkgProv = Ver->ProvidesList;
1143    Ver->ProvidesList = Prv.Index();
1144
1145    // Link it to the package
1146    Prv->ParentPkg = Pkg.Index();
1147    Prv->NextProvides = Pkg->ProvidesList;
1148    Pkg->ProvidesList = Prv.Index();
1149    return true;
1150 }
1151                                                                         /*}}}*/
1152 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1153 bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1154                                 string const &Version, uint8_t const Flags) {
1155    pkgCache &Cache = Owner->Cache;
1156    pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1157    if (Grp.end() == true)
1158       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1159    else
1160    {
1161       map_stringitem_t idxProvideVersion = 0;
1162       if (Version.empty() == false) {
1163          idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1164          if (unlikely(idxProvideVersion == 0))
1165             return false;
1166       }
1167
1168       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1169       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1170       pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1171       for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1172       {
1173          if (isImplicit && OwnerPkg == Pkg)
1174             continue;
1175          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1176             continue;
1177          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1178             return false;
1179       }
1180    }
1181    return true;
1182 }
1183                                                                         /*}}}*/
1184 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1185       pkgCache::VerIterator const &Ver)
1186 {
1187    return Hash == Ver->Hash;
1188 }
1189                                                                         /*}}}*/
1190 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1191 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1192                                    unsigned long Flags)
1193 {
1194    if (File.empty() && Site.empty())
1195    {
1196       CurrentRlsFile = NULL;
1197       return true;
1198    }
1199
1200    // Get some space for the structure
1201    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1202    if (unlikely(idxFile == 0))
1203       return false;
1204    CurrentRlsFile = Cache.RlsFileP + idxFile;
1205
1206    // Fill it in
1207    map_stringitem_t const idxFileName = WriteStringInMap(File);
1208    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1209    if (unlikely(idxFileName == 0 || idxSite == 0))
1210       return false;
1211    CurrentRlsFile->FileName = idxFileName;
1212    CurrentRlsFile->Site = idxSite;
1213    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1214    CurrentRlsFile->Flags = Flags;
1215    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1216    RlsFileName = File;
1217    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1218    Cache.HeaderP->ReleaseFileCount++;
1219
1220    return true;
1221 }
1222                                                                         /*}}}*/
1223 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1224 // ---------------------------------------------------------------------
1225 /* This is used to select which file is to be associated with all newly
1226    added versions. The caller is responsible for setting the IMS fields. */
1227 bool pkgCacheGenerator::SelectFile(std::string const &File,
1228                                    pkgIndexFile const &Index,
1229                                    std::string const &Architecture,
1230                                    std::string const &Component,
1231                                    unsigned long const Flags)
1232 {
1233    // Get some space for the structure
1234    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1235    if (unlikely(idxFile == 0))
1236       return false;
1237    CurrentFile = Cache.PkgFileP + idxFile;
1238
1239    // Fill it in
1240    map_stringitem_t const idxFileName = WriteStringInMap(File);
1241    if (unlikely(idxFileName == 0))
1242       return false;
1243    CurrentFile->FileName = idxFileName;
1244    CurrentFile->NextFile = Cache.HeaderP->FileList;
1245    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1246    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1247    if (unlikely(idxIndexType == 0))
1248       return false;
1249    CurrentFile->IndexType = idxIndexType;
1250    if (Architecture.empty())
1251       CurrentFile->Architecture = 0;
1252    else
1253    {
1254       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1255       if (unlikely(arch == 0))
1256          return false;
1257       CurrentFile->Architecture = arch;
1258    }
1259    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1260    if (unlikely(component == 0))
1261       return false;
1262    CurrentFile->Component = component;
1263    CurrentFile->Flags = Flags;
1264    if (CurrentRlsFile != NULL)
1265       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1266    else
1267       CurrentFile->Release = 0;
1268    PkgFileName = File;
1269    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1270    Cache.HeaderP->PackageFileCount++;
1271
1272    if (Progress != 0)
1273       Progress->SubProgress(Index.Size());
1274    return true;
1275 }
1276                                                                         /*}}}*/
1277 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1278 // ---------------------------------------------------------------------
1279 /* This is used to create handles to strings. Given the same text it
1280    always returns the same number */
1281 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1282                                                  unsigned int Size)
1283 {
1284    std::string const key(S, Size);
1285
1286    std::map<std::string,map_stringitem_t> * strings;
1287    switch(type) {
1288       case MIXED: strings = &strMixed; break;
1289       case PKGNAME: strings = &strPkgNames; break;
1290       case VERSIONNUMBER: strings = &strVersions; break;
1291       case SECTION: strings = &strSections; break;
1292       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1293    }
1294
1295    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1296    if (item != strings->end())
1297       return item->second;
1298
1299    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1300    strings->insert(std::make_pair(key, idxString));
1301    return idxString;
1302 }
1303                                                                         /*}}}*/
1304 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1305 // ---------------------------------------------------------------------
1306 /* This just verifies that each file in the list of index files exists,
1307    has matching attributes with the cache and the cache does not have
1308    any extra files. */
1309 static bool CheckValidity(const string &CacheFile,
1310                           pkgSourceList &List,
1311                           FileIterator Start,
1312                           FileIterator End,
1313                           MMap **OutMap = 0)
1314 {
1315    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1316    // No file, certainly invalid
1317    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1318    {
1319       if (Debug == true)
1320          std::clog << "CacheFile doesn't exist" << std::endl;
1321       return false;
1322    }
1323
1324    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1325    {
1326       if (Debug == true)
1327          std::clog << "sources.list is newer than the cache" << std::endl;
1328       return false;
1329    }
1330
1331    // Map it
1332    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1333    SPtr<MMap> Map = new MMap(CacheF,0);
1334    pkgCache Cache(Map);
1335    if (_error->PendingError() == true || Map->Size() == 0)
1336    {
1337       if (Debug == true)
1338          std::clog << "Errors are pending or Map is empty()" << std::endl;
1339       _error->Discard();
1340       return false;
1341    }
1342
1343    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1344    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1345    std::vector<pkgIndexFile *> Files;
1346    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1347    {
1348       if (Debug == true)
1349          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1350       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1351       if (RlsFile.end() == true)
1352       {
1353          if (Debug == true)
1354             std::clog << "FindInCache returned end-Pointer" << std::endl;
1355          return false;
1356       }
1357
1358       RlsVisited[RlsFile->ID] = true;
1359       if (Debug == true)
1360          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1361
1362       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1363       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1364          if ((*j)->HasPackages())
1365             Files.push_back (*j);
1366    }
1367    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1368       if (RlsVisited[I] == false)
1369       {
1370          if (Debug == true)
1371             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1372          return false;
1373       }
1374
1375    for (; Start != End; ++Start)
1376       Files.push_back(*Start);
1377
1378    /* Now we check every index file, see if it is in the cache,
1379       verify the IMS data and check that it is on the disk too.. */
1380    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1381    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1382    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1383    {
1384       if (Debug == true)
1385          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1386       if ((*PkgFile)->Exists() == false)
1387       {
1388          if (Debug == true)
1389             std::clog << "file doesn't exist" << std::endl;
1390          continue;
1391       }
1392
1393       // FindInCache is also expected to do an IMS check.
1394       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1395       if (File.end() == true)
1396       {
1397          if (Debug == true)
1398             std::clog << "FindInCache returned end-Pointer" << std::endl;
1399          return false;
1400       }
1401
1402       Visited[File->ID] = true;
1403       if (Debug == true)
1404          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1405    }
1406
1407    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1408       if (Visited[I] == false)
1409       {
1410          if (Debug == true)
1411             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1412          return false;
1413       }
1414
1415    if (_error->PendingError() == true)
1416    {
1417       if (Debug == true)
1418       {
1419          std::clog << "Validity failed because of pending errors:" << std::endl;
1420          _error->DumpErrors();
1421       }
1422       _error->Discard();
1423       return false;
1424    }
1425
1426    if (OutMap != 0)
1427       *OutMap = Map.UnGuard();
1428    return true;
1429 }
1430                                                                         /*}}}*/
1431 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1432 // ---------------------------------------------------------------------
1433 /* Size is kind of an abstract notion that is only used for the progress
1434    meter */
1435 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1436 {
1437    map_filesize_t TotalSize = 0;
1438    if (List !=  NULL)
1439    {
1440       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1441       {
1442          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1443          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1444             if ((*j)->HasPackages() == true)
1445                TotalSize += (*j)->Size();
1446       }
1447    }
1448
1449    for (; Start < End; ++Start)
1450    {
1451       if ((*Start)->HasPackages() == false)
1452          continue;
1453       TotalSize += (*Start)->Size();
1454    }
1455    return TotalSize;
1456 }
1457                                                                         /*}}}*/
1458 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1459 // ---------------------------------------------------------------------
1460 /* */
1461 static bool BuildCache(pkgCacheGenerator &Gen,
1462                        OpProgress *Progress,
1463                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1464                        pkgSourceList const * const List,
1465                        FileIterator Start, FileIterator End)
1466 {
1467    std::vector<pkgIndexFile *> Files;
1468    bool const HasFileDeps = Gen.HasFileDeps();
1469
1470    if (List !=  NULL)
1471    {
1472       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1473       {
1474          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1475          {
1476             _error->Warning("Duplicate sources.list entry %s",
1477                   (*i)->Describe().c_str());
1478             continue;
1479          }
1480
1481          if ((*i)->Merge(Gen, Progress) == false)
1482             return false;
1483
1484          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1485          for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
1486          {
1487             if (HasFileDeps)
1488                Files.push_back(*I);
1489
1490             if ((*I)->HasPackages() == false)
1491                continue;
1492
1493             if ((*I)->Exists() == false)
1494                continue;
1495
1496             if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1497             {
1498                _error->Warning("Duplicate sources.list entry %s",
1499                      (*I)->Describe().c_str());
1500                continue;
1501             }
1502
1503             map_filesize_t Size = (*I)->Size();
1504             if (Progress != NULL)
1505                Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1506             CurrentSize += Size;
1507
1508             if ((*I)->Merge(Gen,Progress) == false)
1509                return false;
1510          }
1511       }
1512    }
1513
1514    Gen.SelectReleaseFile("", "");
1515    FileIterator I;
1516    for (I = Start; I != End; ++I)
1517    {
1518       if (HasFileDeps)
1519          Files.push_back(*I);
1520
1521       if ((*I)->HasPackages() == false)
1522          continue;
1523
1524       if ((*I)->Exists() == false)
1525          continue;
1526
1527       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1528       {
1529          _error->Warning("Duplicate sources.list entry %s",
1530                          (*I)->Describe().c_str());
1531          continue;
1532       }
1533
1534       map_filesize_t Size = (*I)->Size();
1535       if (Progress != NULL)
1536          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1537       CurrentSize += Size;
1538
1539       if ((*I)->Merge(Gen,Progress) == false)
1540          return false;
1541    }
1542
1543    if (HasFileDeps == true)
1544    {
1545       if (Progress != NULL)
1546          Progress->Done();
1547       TotalSize = ComputeSize(List, Start, End);
1548       CurrentSize = 0;
1549       for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
1550       {
1551          map_filesize_t Size = (*I)->Size();
1552          if (Progress != NULL)
1553             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1554          CurrentSize += Size;
1555          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1556             return false;
1557       }
1558    }
1559
1560    return true;
1561 }
1562                                                                         /*}}}*/
1563 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1564 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1565    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1566    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1567    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1568    Flags |= MMap::Moveable;
1569    if (_config->FindB("APT::Cache-Fallback", false) == true)
1570       Flags |= MMap::Fallback;
1571    if (CacheF != NULL)
1572       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1573    else
1574       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1575 }
1576                                                                         /*}}}*/
1577 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1578 // ---------------------------------------------------------------------
1579 /* This makes sure that the status cache (the cache that has all
1580    index files from the sources list and all local ones) is ready
1581    to be mmaped. If OutMap is not zero then a MMap object representing
1582    the cache will be stored there. This is pretty much mandetory if you
1583    are using AllowMem. AllowMem lets the function be run as non-root
1584    where it builds the cache 'fast' into a memory buffer. */
1585 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1586                         MMap **OutMap, bool AllowMem)
1587    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1588 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1589                         MMap **OutMap,bool AllowMem)
1590 {
1591    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1592
1593    std::vector<pkgIndexFile *> Files;
1594    /*
1595    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1596         i != List.end();
1597         ++i)
1598    {
1599       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1600       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1601            j != Indexes->end();
1602            ++j)
1603          Files.push_back (*j);
1604    }
1605 */
1606    if (_system->AddStatusFiles(Files) == false)
1607       return false;
1608
1609    // Decide if we can write to the files..
1610    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1611    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1612
1613    // ensure the cache directory exists
1614    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1615    {
1616       string dir = _config->FindDir("Dir::Cache");
1617       size_t const len = dir.size();
1618       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1619          dir = dir.substr(0, len - 5);
1620       if (CacheFile.empty() == false)
1621          CreateDirectory(dir, flNotFile(CacheFile));
1622       if (SrcCacheFile.empty() == false)
1623          CreateDirectory(dir, flNotFile(SrcCacheFile));
1624    }
1625
1626    // Decide if we can write to the cache
1627    bool Writeable = false;
1628    if (CacheFile.empty() == false)
1629       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1630    else
1631       if (SrcCacheFile.empty() == false)
1632          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1633    if (Debug == true)
1634       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1635
1636    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1637       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1638
1639    if (Progress != NULL)
1640       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1641
1642    // Cache is OK, Fin.
1643    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1644    {
1645       if (Progress != NULL)
1646          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1647       if (Debug == true)
1648          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1649       return true;
1650    }
1651    else if (Debug == true)
1652          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1653
1654    /* At this point we know we need to reconstruct the package cache,
1655       begin. */
1656    SPtr<FileFd> CacheF;
1657    SPtr<DynamicMMap> Map;
1658    if (Writeable == true && CacheFile.empty() == false)
1659    {
1660       _error->PushToStack();
1661       unlink(CacheFile.c_str());
1662       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1663       fchmod(CacheF->Fd(),0644);
1664       Map = CreateDynamicMMap(CacheF, MMap::Public);
1665       if (_error->PendingError() == true)
1666       {
1667          delete CacheF.UnGuard();
1668          delete Map.UnGuard();
1669          if (Debug == true)
1670             std::clog << "Open filebased MMap FAILED" << std::endl;
1671          Writeable = false;
1672          if (AllowMem == false)
1673          {
1674             _error->MergeWithStack();
1675             return false;
1676          }
1677          _error->RevertToStack();
1678       }
1679       else
1680       {
1681          _error->MergeWithStack();
1682          if (Debug == true)
1683             std::clog << "Open filebased MMap" << std::endl;
1684       }
1685    }
1686    if (Writeable == false || CacheFile.empty() == true)
1687    {
1688       // Just build it in memory..
1689       Map = CreateDynamicMMap(NULL);
1690       if (Debug == true)
1691          std::clog << "Open memory Map (not filebased)" << std::endl;
1692    }
1693
1694    // Lets try the source cache.
1695    map_filesize_t CurrentSize = 0;
1696    map_filesize_t TotalSize = 0;
1697    if (CheckValidity(SrcCacheFile, List, Files.end(),
1698                      Files.end()) == true)
1699    {
1700       if (Debug == true)
1701          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1702       // Preload the map with the source cache
1703       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1704       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1705       if ((alloc == 0 && _error->PendingError())
1706                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1707                                 SCacheF.Size()) == false)
1708          return false;
1709
1710       TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1711
1712       // Build the status cache
1713       pkgCacheGenerator Gen(Map.Get(),Progress);
1714       if (_error->PendingError() == true)
1715          return false;
1716       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1717                      Files.begin(),Files.end()) == false)
1718          return false;
1719    }
1720    else
1721    {
1722       if (Debug == true)
1723          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1724       TotalSize = ComputeSize(&List, Files.begin(),Files.end());
1725
1726       // Build the source cache
1727       pkgCacheGenerator Gen(Map.Get(),Progress);
1728       if (_error->PendingError() == true)
1729          return false;
1730       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
1731                      Files.end(),Files.end()) == false)
1732          return false;
1733
1734       // Write it back
1735       if (Writeable == true && SrcCacheFile.empty() == false)
1736       {
1737          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1738          if (_error->PendingError() == true)
1739             return false;
1740
1741          fchmod(SCacheF.Fd(),0644);
1742
1743          // Write out the main data
1744          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1745             return _error->Error(_("IO Error saving source cache"));
1746          SCacheF.Sync();
1747
1748          // Write out the proper header
1749          Gen.GetCache().HeaderP->Dirty = false;
1750          if (SCacheF.Seek(0) == false ||
1751              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1752             return _error->Error(_("IO Error saving source cache"));
1753          Gen.GetCache().HeaderP->Dirty = true;
1754          SCacheF.Sync();
1755       }
1756
1757       // Build the status cache
1758       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1759                      Files.begin(), Files.end()) == false)
1760          return false;
1761    }
1762    if (Debug == true)
1763       std::clog << "Caches are ready for shipping" << std::endl;
1764
1765    if (_error->PendingError() == true)
1766       return false;
1767    if (OutMap != 0)
1768    {
1769       if (CacheF != 0)
1770       {
1771          delete Map.UnGuard();
1772          *OutMap = new MMap(*CacheF,0);
1773       }
1774       else
1775       {
1776          *OutMap = Map.UnGuard();
1777       }
1778    }
1779
1780    return true;
1781 }
1782                                                                         /*}}}*/
1783 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1784 // ---------------------------------------------------------------------
1785 /* */
1786 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1787    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1788 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1789 {
1790    std::vector<pkgIndexFile *> Files;
1791    if (_system->AddStatusFiles(Files) == false)
1792       return false;
1793
1794    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1795    map_filesize_t CurrentSize = 0;
1796    map_filesize_t TotalSize = 0;
1797
1798    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1799
1800    // Build the status cache
1801    if (Progress != NULL)
1802       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1803    pkgCacheGenerator Gen(Map.Get(),Progress);
1804    if (_error->PendingError() == true)
1805       return false;
1806    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1807                   Files.begin(), Files.end()) == false)
1808       return false;
1809
1810    if (_error->PendingError() == true)
1811       return false;
1812    *OutMap = Map.UnGuard();
1813
1814    return true;
1815 }
1816                                                                         /*}}}*/
1817 // IsDuplicateDescription                                               /*{{{*/
1818 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1819                             MD5SumValue const &CurMd5, std::string const &CurLang)
1820 {
1821    // Descriptions in the same link-list have all the same md5
1822    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1823       return false;
1824    for (; Desc.end() == false; ++Desc)
1825       if (Desc.LanguageCode() == CurLang)
1826          return true;
1827    return false;
1828 }
1829                                                                         /*}}}*/
1830 // CacheGenerator::FinishCache                                          /*{{{*/
1831 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1832 {
1833    return true;
1834 }
1835                                                                         /*}}}*/
1836
1837 pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
1838 pkgCacheGenerator::ListParser::~ListParser() {}