apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                      CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
  58 {
  59    if (_error->PendingError() == true)
  60       return;
  61
  62    if (Map.Size() == 0)
  63    {
  64       // Setup the map interface..
  65       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  66       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  67          return;
  68
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70
  71       // Starting header
  72       *Cache.HeaderP = pkgCache::Header();
  73
  74       // make room for the hashtables for packages and groups
  75       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  76          return;
  77
  78       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  79       if (unlikely(idxVerSysName == 0))
  80          return;
  81       Cache.HeaderP->VerSysName = idxVerSysName;
  82       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  83       if (unlikely(idxArchitecture == 0))
  84          return;
  85       Cache.HeaderP->Architecture = idxArchitecture;
  86
  87       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  88       if (archs.size() > 1)
  89       {
  90          std::vector<std::string>::const_iterator a = archs.begin();
  91          std::string list = *a;
  92          for (++a; a != archs.end(); ++a)
  93             list.append(",").append(*a);
  94          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  95          if (unlikely(idxArchitectures == 0))
  96             return;
  97          Cache.HeaderP->SetArchitectures(idxArchitectures);
  98       }
  99       else
 100          Cache.HeaderP->SetArchitectures(idxArchitecture);
 101
 102       Cache.ReMap();
 103    }
 104    else
 105    {
 106       // Map directly from the existing file
 107       Cache.ReMap();
 108       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 109       if (Cache.VS != _system->VS)
 110       {
 111          _error->Error(_("Cache has an incompatible versioning system"));
 112          return;
 113       }
 114    }
 115
 116    Cache.HeaderP->Dirty = true;
 117    Map.Sync(0,sizeof(pkgCache::Header));
 118 }
 119                                                                         /*}}}*/
 120 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 121 // ---------------------------------------------------------------------
 122 /* We sync the data then unset the dirty flag in two steps so as to
 123    advoid a problem during a crash */
 124 pkgCacheGenerator::~pkgCacheGenerator()
 125 {
 126    if (_error->PendingError() == true)
 127       return;
 128    if (Map.Sync() == false)
 129       return;
 130
 131    Cache.HeaderP->Dirty = false;
 132    Cache.HeaderP->CacheFileSize = Map.Size();
 133    Map.Sync(0,sizeof(pkgCache::Header));
 134 }
 135                                                                         /*}}}*/
 136 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 137    if (oldMap == newMap)
 138       return;
 139
 140    if (_config->FindB("Debug::pkgCacheGen", false))
 141       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 142
 143    Cache.ReMap(false);
 144
 145    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 146    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 147
 148    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 149         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 150       (*i)->ReMap(oldMap, newMap);
 151    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 152         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 153       (*i)->ReMap(oldMap, newMap);
 154    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 155         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 156       (*i)->ReMap(oldMap, newMap);
 157    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 158         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 159       (*i)->ReMap(oldMap, newMap);
 160    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 161         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 162       (*i)->ReMap(oldMap, newMap);
 163    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 164         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 165       (*i)->ReMap(oldMap, newMap);
 166    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 168       (*i)->ReMap(oldMap, newMap);
 169    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 170         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 171       (*i)->ReMap(oldMap, newMap);
 172 }                                                                       /*}}}*/
 173 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 174 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 175                                         const unsigned long &Len) {
 176    void const * const oldMap = Map.Data();
 177    map_stringitem_t const index = Map.WriteString(String, Len);
 178    if (index != 0)
 179       ReMap(oldMap, Map.Data());
 180    return index;
 181 }
 182                                                                         /*}}}*/
 183 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 184 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 185    void const * const oldMap = Map.Data();
 186    map_stringitem_t const index = Map.WriteString(String);
 187    if (index != 0)
 188       ReMap(oldMap, Map.Data());
 189    return index;
 190 }
 191                                                                         /*}}}*/
 192 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 193    void const * const oldMap = Map.Data();
 194    map_pointer_t const index = Map.Allocate(size);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This provides the generation of the entries in the cache. Each loop
 203    goes through a single package record from the underlying parse engine. */
 204 bool pkgCacheGenerator::MergeList(ListParser &List,
 205                                   pkgCache::VerIterator *OutVer)
 206 {
 207    List.Owner = this;
 208
 209    unsigned int Counter = 0;
 210    while (List.Step() == true)
 211    {
 212       string const PackageName = List.Package();
 213       if (PackageName.empty() == true)
 214          return false;
 215
 216       Counter++;
 217       if (Counter % 100 == 0 && Progress != 0)
 218          Progress->Progress(List.Offset());
 219
 220       string Arch = List.Architecture();
 221       string const Version = List.Version();
 222       if (Version.empty() == true && Arch.empty() == true)
 223       {
 224          // package descriptions
 225          if (MergeListGroup(List, PackageName) == false)
 226             return false;
 227          continue;
 228       }
 229
 230       if (Arch.empty() == true)
 231       {
 232          // use the pseudo arch 'none' for arch-less packages
 233          Arch = "none";
 234          /* We might built a SingleArchCache here, which we don't want to blow up
 235             just for these :none packages to a proper MultiArchCache, so just ensure
 236             that we have always a native package structure first for SingleArch */
 237          pkgCache::PkgIterator NP;
 238          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 239          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 240          // TRANSLATOR: The first placeholder is a package name,
 241          // the other two should be copied verbatim as they include debug info
 242          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 243                               PackageName.c_str(), "NewPackage", 0);
 244       }
 245
 246       // Get a pointer to the package structure
 247       pkgCache::PkgIterator Pkg;
 248       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 249       if (NewPackage(Pkg, PackageName, Arch) == false)
 250          // TRANSLATOR: The first placeholder is a package name,
 251          // the other two should be copied verbatim as they include debug info
 252          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 253                               PackageName.c_str(), "NewPackage", 1);
 254
 255
 256       if (Version.empty() == true)
 257       {
 258          if (MergeListPackage(List, Pkg) == false)
 259             return false;
 260       }
 261       else
 262       {
 263          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 264             return false;
 265       }
 266
 267       if (OutVer != 0)
 268       {
 269          FoundFileDeps |= List.HasFileDeps();
 270          return true;
 271       }
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    FoundFileDeps |= List.HasFileDeps();
 288    return true;
 289 }
 290 // CacheGenerator::MergeListGroup                                       /*{{{*/
 291 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 292 {
 293    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 294    // a group has no data on it's own, only packages have it but these
 295    // stanzas like this come from Translation- files to add descriptions,
 296    // but without a version we don't need a description for it…
 297    if (Grp.end() == true)
 298       return true;
 299    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 300
 301    pkgCache::PkgIterator Pkg;
 302    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 303    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 304       if (MergeListPackage(List, Pkg) == false)
 305          return false;
 306
 307    return true;
 308 }
 309                                                                         /*}}}*/
 310 // CacheGenerator::MergeListPackage                                     /*{{{*/
 311 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 312 {
 313    // we first process the package, then the descriptions
 314    // (for deb this package processing is in fact a no-op)
 315    pkgCache::VerIterator Ver(Cache);
 316    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 317    if (List.UsePackage(Pkg, Ver) == false)
 318       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 319                            Pkg.Name(), "UsePackage", 1);
 320
 321    // Find the right version to write the description
 322    MD5SumValue CurMd5 = List.Description_md5();
 323    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 324       return true;
 325    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 326    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 327    {
 328       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 329
 330       // a version can only have one md5 describing it
 331       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 332          continue;
 333
 334       map_stringitem_t md5idx = VerDesc->md5sum;
 335       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 336       {
 337          // don't add a new description if we have one for the given
 338          // md5 && language
 339          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 340             continue;
 341
 342          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 343       }
 344
 345       // we can stop here as all "same" versions will share the description
 346       break;
 347    }
 348
 349    return true;
 350 }
 351                                                                         /*}}}*/
 352 // CacheGenerator::MergeListVersion                                     /*{{{*/
 353 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 354                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 355 {
 356    pkgCache::VerIterator Ver = Pkg.VersionList();
 357    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 358    map_pointer_t *LastVer = &Pkg->VersionList;
 359    void const * oldMap = Map.Data();
 360
 361    unsigned short const Hash = List.VersionHash();
 362    if (Ver.end() == false)
 363    {
 364       /* We know the list is sorted so we use that fact in the search.
 365          Insertion of new versions is done with correct sorting */
 366       int Res = 1;
 367       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 368       {
 369          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 370          // Version is higher as current version - insert here
 371          if (Res > 0)
 372             break;
 373          // Versionstrings are equal - is hash also equal?
 374          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 375             break;
 376          // proceed with the next till we have either the right
 377          // or we found another version (which will be lower)
 378       }
 379
 380       /* We already have a version for this item, record that we saw it */
 381       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 382       {
 383          if (List.UsePackage(Pkg,Ver) == false)
 384             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 385                                  Pkg.Name(), "UsePackage", 2);
 386
 387          if (NewFileVer(Ver,List) == false)
 388             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 389                                  Pkg.Name(), "NewFileVer", 1);
 390
 391          // Read only a single record and return
 392          if (OutVer != 0)
 393          {
 394             *OutVer = Ver;
 395             return true;
 396          }
 397
 398          return true;
 399       }
 400    }
 401
 402    // Add a new version
 403    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 404    if (verindex == 0 && _error->PendingError())
 405       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 406                            Pkg.Name(), "NewVersion", 1);
 407
 408    if (oldMap != Map.Data())
 409          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 410    *LastVer = verindex;
 411
 412    if (unlikely(List.NewVersion(Ver) == false))
 413       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 414                            Pkg.Name(), "NewVersion", 2);
 415
 416    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 417       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 418                            Pkg.Name(), "UsePackage", 3);
 419
 420    if (unlikely(NewFileVer(Ver,List) == false))
 421       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 422                            Pkg.Name(), "NewFileVer", 2);
 423
 424    pkgCache::GrpIterator Grp = Pkg.Group();
 425    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 426
 427    /* If it is the first version of this package we need to add implicit
 428       Multi-Arch dependencies to all other package versions in the group now -
 429       otherwise we just add them for this new version */
 430    if (Pkg.VersionList()->NextVer == 0)
 431    {
 432       pkgCache::PkgIterator P = Grp.PackageList();
 433       Dynamic<pkgCache::PkgIterator> DynP(P);
 434       for (; P.end() != true; P = Grp.NextPkg(P))
 435       {
 436          if (P->ID == Pkg->ID)
 437             continue;
 438          pkgCache::VerIterator V = P.VersionList();
 439          Dynamic<pkgCache::VerIterator> DynV(V);
 440          for (; V.end() != true; ++V)
 441             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 442                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 443                                     Pkg.Name(), "AddImplicitDepends", 1);
 444       }
 445       /* :none packages are packages without an architecture. They are forbidden by
 446          debian-policy, so usually they will only be in (old) dpkg status files -
 447          and dpkg will complain about them - and are pretty rare. We therefore do
 448          usually not create conflicts while the parent is created, but only if a :none
 449          package (= the target) appears. This creates incorrect dependencies on :none
 450          for architecture-specific dependencies on the package we copy from, but we
 451          will ignore this bug as architecture-specific dependencies are only allowed
 452          in jessie and until then the :none packages should be extinct (hopefully).
 453          In other words: This should work long enough to allow graceful removal of
 454          these packages, it is not supposed to allow users to keep using them … */
 455       if (strcmp(Pkg.Arch(), "none") == 0)
 456       {
 457          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 458          if (M.end() == false && Pkg != M)
 459          {
 460             pkgCache::DepIterator D = M.RevDependsList();
 461             Dynamic<pkgCache::DepIterator> DynD(D);
 462             for (; D.end() == false; ++D)
 463             {
 464                if ((D->Type != pkgCache::Dep::Conflicts &&
 465                     D->Type != pkgCache::Dep::DpkgBreaks &&
 466                     D->Type != pkgCache::Dep::Replaces) ||
 467                    D.ParentPkg().Group() == Grp)
 468                   continue;
 469
 470                map_pointer_t *OldDepLast = NULL;
 471                pkgCache::VerIterator ConVersion = D.ParentVer();
 472                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 473                // duplicate the Conflicts/Breaks/Replaces for :none arch
 474                NewDepends(Pkg, ConVersion, D->Version,
 475                      D->CompareOp, D->Type, OldDepLast);
 476             }
 477          }
 478       }
 479    }
 480    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482                            Pkg.Name(), "AddImplicitDepends", 2);
 483
 484    // Read only a single record and return
 485    if (OutVer != 0)
 486    {
 487       *OutVer = Ver;
 488       return true;
 489    }
 490
 491    /* Record the Description(s) based on their master md5sum */
 492    MD5SumValue CurMd5 = List.Description_md5();
 493    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 494       return true;
 495
 496    /* Before we add a new description we first search in the group for
 497       a version with a description of the same MD5 - if so we reuse this
 498       description group instead of creating our own for this version */
 499    for (pkgCache::PkgIterator P = Grp.PackageList();
 500         P.end() == false; P = Grp.NextPkg(P))
 501    {
 502       for (pkgCache::VerIterator V = P.VersionList();
 503            V.end() == false; ++V)
 504       {
 505          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 506             continue;
 507          Ver->DescriptionList = V->DescriptionList;
 508       }
 509    }
 510
 511    // We haven't found reusable descriptions, so add the first description(s)
 512    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 513    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 514    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 515       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 516          return false;
 517    return true;
 518 }
 519                                                                         /*}}}*/
 520 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 521 {
 522    pkgCache::DescIterator Desc;
 523    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 524
 525    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 526    if (unlikely(descindex == 0 && _error->PendingError()))
 527       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 528             Ver.ParentPkg().Name(), "NewDescription", 1);
 529
 530    md5idx = Desc->md5sum;
 531    Desc->ParentPkg = Ver.ParentPkg().Index();
 532
 533    // we add at the end, so that the start is constant as we need
 534    // that to be able to efficiently share these lists
 535    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 536    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 537    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 538    *LastNextDesc = descindex;
 539
 540    if (NewFileDesc(Desc,List) == false)
 541       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 542             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 543
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547                                                                         /*}}}*/
 548 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* If we found any file depends while parsing the main list we need to
 551    resolve them. Since it is undesired to load the entire list of files
 552    into the cache as virtual packages we do a two stage effort. MergeList
 553    identifies the file depends and this creates Provdies for them by
 554    re-parsing all the indexs. */
 555 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 556 {
 557    List.Owner = this;
 558
 559    unsigned int Counter = 0;
 560    while (List.Step() == true)
 561    {
 562       string PackageName = List.Package();
 563       if (PackageName.empty() == true)
 564          return false;
 565       string Version = List.Version();
 566       if (Version.empty() == true)
 567          continue;
 568
 569       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 570       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 571       if (Pkg.end() == true)
 572          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 573                                 PackageName.c_str(), "FindPkg", 1);
 574       Counter++;
 575       if (Counter % 100 == 0 && Progress != 0)
 576          Progress->Progress(List.Offset());
 577
 578       unsigned short Hash = List.VersionHash();
 579       pkgCache::VerIterator Ver = Pkg.VersionList();
 580       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 581       for (; Ver.end() == false; ++Ver)
 582       {
 583          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 584          {
 585             if (List.CollectFileProvides(Cache,Ver) == false)
 586                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 587                                     PackageName.c_str(), "CollectFileProvides", 1);
 588             break;
 589          }
 590       }
 591
 592       if (Ver.end() == true)
 593          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 594    }
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This creates a new group structure and adds it to the hash table */
 602 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 603 {
 604    Grp = Cache.FindGrp(Name);
 605    if (Grp.end() == false)
 606       return true;
 607
 608    // Get a structure
 609    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 610    if (unlikely(Group == 0))
 611       return false;
 612
 613    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 614    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 615    if (unlikely(idxName == 0))
 616       return false;
 617    Grp->Name = idxName;
 618
 619    // Insert it into the hash table
 620    unsigned long const Hash = Cache.Hash(Name);
 621    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 622    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 623       insertAt = &(Cache.GrpP + *insertAt)->Next;
 624    Grp->Next = *insertAt;
 625    *insertAt = Group;
 626
 627    Grp->ID = Cache.HeaderP->GroupCount++;
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* This creates a new package structure and adds it to the hash table */
 634 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 635                                         const string &Arch) {
 636    pkgCache::GrpIterator Grp;
 637    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 638    if (unlikely(NewGroup(Grp, Name) == false))
 639       return false;
 640
 641    Pkg = Grp.FindPkg(Arch);
 642       if (Pkg.end() == false)
 643          return true;
 644
 645    // Get a structure
 646    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 647    if (unlikely(Package == 0))
 648       return false;
 649    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 650
 651    // Insert the package into our package list
 652    if (Grp->FirstPackage == 0) // the group is new
 653    {
 654       Grp->FirstPackage = Package;
 655       // Insert it into the hash table
 656       map_id_t const Hash = Cache.Hash(Name);
 657       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 658       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 659          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 660       Pkg->NextPackage = *insertAt;
 661       *insertAt = Package;
 662    }
 663    else // Group the Packages together
 664    {
 665       // this package is the new last package
 666       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 667       Pkg->NextPackage = LastPkg->NextPackage;
 668       LastPkg->NextPackage = Package;
 669    }
 670    Grp->LastPackage = Package;
 671
 672    // Set the name, arch and the ID
 673    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 674    Pkg->Group = Grp.Index();
 675    // all is mapped to the native architecture
 676    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 677    if (unlikely(idxArch == 0))
 678       return false;
 679    Pkg->Arch = idxArch;
 680    Pkg->ID = Cache.HeaderP->PackageCount++;
 681
 682    return true;
 683 }
 684                                                                         /*}}}*/
 685 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 686 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 687                                            pkgCache::PkgIterator &P,
 688                                            pkgCache::VerIterator &V)
 689 {
 690    // copy P.Arch() into a string here as a cache remap
 691    // in NewDepends() later may alter the pointer location
 692    string Arch = P.Arch() == NULL ? "" : P.Arch();
 693    map_pointer_t *OldDepLast = NULL;
 694    /* MultiArch handling introduces a lot of implicit Dependencies:
 695       - MultiArch: same → Co-Installable if they have the same version
 696       - All others conflict with all other group members */
 697    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 698    pkgCache::PkgIterator D = G.PackageList();
 699    Dynamic<pkgCache::PkgIterator> DynD(D);
 700    map_stringitem_t const VerStrIdx = V->VerStr;
 701    for (; D.end() != true; D = G.NextPkg(D))
 702    {
 703       if (Arch == D.Arch() || D->VersionList == 0)
 704          continue;
 705       /* We allow only one installed arch at the time
 706          per group, therefore each group member conflicts
 707          with all other group members */
 708       if (coInstall == true)
 709       {
 710          // Replaces: ${self}:other ( << ${binary:Version})
 711          NewDepends(D, V, VerStrIdx,
 712                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 713                     OldDepLast);
 714          // Breaks: ${self}:other (!= ${binary:Version})
 715          NewDepends(D, V, VerStrIdx,
 716                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 717                     OldDepLast);
 718       } else {
 719          // Conflicts: ${self}:other
 720          NewDepends(D, V, 0,
 721                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 722                     OldDepLast);
 723       }
 724    }
 725    return true;
 726 }
 727 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 728                                            pkgCache::PkgIterator &D)
 729 {
 730    /* MultiArch handling introduces a lot of implicit Dependencies:
 731       - MultiArch: same → Co-Installable if they have the same version
 732       - All others conflict with all other group members */
 733    map_pointer_t *OldDepLast = NULL;
 734    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 735    if (coInstall == true)
 736    {
 737       map_stringitem_t const VerStrIdx = V->VerStr;
 738       // Replaces: ${self}:other ( << ${binary:Version})
 739       NewDepends(D, V, VerStrIdx,
 740                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 741                  OldDepLast);
 742       // Breaks: ${self}:other (!= ${binary:Version})
 743       NewDepends(D, V, VerStrIdx,
 744                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 745                  OldDepLast);
 746    } else {
 747       // Conflicts: ${self}:other
 748       NewDepends(D, V, 0,
 749                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 750                  OldDepLast);
 751    }
 752    return true;
 753 }
 754
 755                                                                         /*}}}*/
 756 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 757 // ---------------------------------------------------------------------
 758 /* */
 759 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 760                                    ListParser &List)
 761 {
 762    if (CurrentFile == 0)
 763       return true;
 764
 765    // Get a structure
 766    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 767    if (VerFile == 0)
 768       return false;
 769
 770    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 771    VF->File = CurrentFile - Cache.PkgFileP;
 772
 773    // Link it to the end of the list
 774    map_pointer_t *Last = &Ver->FileList;
 775    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 776       Last = &V->NextFile;
 777    VF->NextFile = *Last;
 778    *Last = VF.Index();
 779
 780    VF->Offset = List.Offset();
 781    VF->Size = List.Size();
 782    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 783       Cache.HeaderP->MaxVerFileSize = VF->Size;
 784    Cache.HeaderP->VerFileCount++;
 785
 786    return true;
 787 }
 788                                                                         /*}}}*/
 789 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 790 // ---------------------------------------------------------------------
 791 /* This puts a version structure in the linked list */
 792 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 793                                             const string &VerStr,
 794                                             map_pointer_t const ParentPkg,
 795                                             unsigned short const Hash,
 796                                             map_pointer_t const Next)
 797 {
 798    // Get a structure
 799    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 800    if (Version == 0)
 801       return 0;
 802
 803    // Fill it in
 804    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 805    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 806    Ver->NextVer = Next;
 807    Ver->ParentPkg = ParentPkg;
 808    Ver->Hash = Hash;
 809    Ver->ID = Cache.HeaderP->VersionCount++;
 810
 811    // try to find the version string in the group for reuse
 812    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 813    pkgCache::GrpIterator Grp = Pkg.Group();
 814    if (Pkg.end() == false && Grp.end() == false)
 815    {
 816       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 817       {
 818          if (Pkg == P)
 819             continue;
 820          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 821          {
 822             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 823             if (cmp == 0)
 824             {
 825                Ver->VerStr = V->VerStr;
 826                return Version;
 827             }
 828             else if (cmp < 0)
 829                break;
 830          }
 831       }
 832    }
 833    // haven't found the version string, so create
 834    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 835    if (unlikely(idxVerStr == 0))
 836       return 0;
 837    Ver->VerStr = idxVerStr;
 838    return Version;
 839 }
 840                                                                         /*}}}*/
 841 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* */
 844 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 845                                    ListParser &List)
 846 {
 847    if (CurrentFile == 0)
 848       return true;
 849
 850    // Get a structure
 851    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 852    if (DescFile == 0)
 853       return false;
 854
 855    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 856    DF->File = CurrentFile - Cache.PkgFileP;
 857
 858    // Link it to the end of the list
 859    map_pointer_t *Last = &Desc->FileList;
 860    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 861       Last = &D->NextFile;
 862
 863    DF->NextFile = *Last;
 864    *Last = DF.Index();
 865
 866    DF->Offset = List.Offset();
 867    DF->Size = List.Size();
 868    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 869       Cache.HeaderP->MaxDescFileSize = DF->Size;
 870    Cache.HeaderP->DescFileCount++;
 871
 872    return true;
 873 }
 874                                                                         /*}}}*/
 875 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 876 // ---------------------------------------------------------------------
 877 /* This puts a description structure in the linked list */
 878 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 879                                             const string &Lang,
 880                                             const MD5SumValue &md5sum,
 881                                             map_stringitem_t const idxmd5str)
 882 {
 883    // Get a structure
 884    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 885    if (Description == 0)
 886       return 0;
 887
 888    // Fill it in
 889    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 890    Desc->ID = Cache.HeaderP->DescriptionCount++;
 891    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 892    if (unlikely(idxlanguage_code == 0))
 893       return 0;
 894    Desc->language_code = idxlanguage_code;
 895
 896    if (idxmd5str != 0)
 897       Desc->md5sum = idxmd5str;
 898    else
 899    {
 900       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 901       if (unlikely(idxmd5sum == 0))
 902          return 0;
 903       Desc->md5sum = idxmd5sum;
 904    }
 905
 906    return Description;
 907 }
 908                                                                         /*}}}*/
 909 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 910 // ---------------------------------------------------------------------
 911 /* This creates a dependency element in the tree. It is linked to the
 912    version and to the package that it is pointing to. */
 913 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 914                                    pkgCache::VerIterator &Ver,
 915                                    string const &Version,
 916                                    unsigned int const &Op,
 917                                    unsigned int const &Type,
 918                                    map_stringitem_t* &OldDepLast)
 919 {
 920    map_stringitem_t index = 0;
 921    if (Version.empty() == false)
 922    {
 923       int const CmpOp = Op & 0x0F;
 924       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 925       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 926          index = Ver->VerStr;
 927
 928       if (index == 0)
 929       {
 930          void const * const oldMap = Map.Data();
 931          index = StoreString(VERSIONNUMBER, Version);
 932          if (unlikely(index == 0))
 933             return false;
 934          if (OldDepLast != 0 && oldMap != Map.Data())
 935             OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 936       }
 937    }
 938    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 939 }
 940 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 941                                    pkgCache::VerIterator &Ver,
 942                                    map_pointer_t const Version,
 943                                    unsigned int const &Op,
 944                                    unsigned int const &Type,
 945                                    map_pointer_t* &OldDepLast)
 946 {
 947    void const * const oldMap = Map.Data();
 948    // Get a structure
 949    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 950    if (unlikely(Dependency == 0))
 951       return false;
 952
 953    bool isDuplicate = false;
 954    map_pointer_t DependencyData = 0;
 955    map_pointer_t * PkgRevDepends = &Pkg->RevDepends;
 956    map_pointer_t previous_id = 0;
 957
 958    while (*PkgRevDepends != 0)
 959    {
 960       pkgCache::Dependency * const L = Cache.DepP + *PkgRevDepends;
 961       PkgRevDepends = &L->NextRevDepends;
 962       if (L->DependencyData == previous_id)
 963          break;
 964       previous_id = L->DependencyData;
 965       pkgCache::DependencyData const * const D = Cache.DepDataP + previous_id;
 966       if (D->Type == Type && D->CompareOp == Op && D->Version == Version)
 967       {
 968          DependencyData = previous_id;
 969          isDuplicate = true;
 970          break;
 971       }
 972    }
 973
 974    if (isDuplicate == false)
 975    {
 976       void const * const oldMap2 = Map.Data();
 977       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 978       if (unlikely(DependencyData == 0))
 979         return false;
 980       if (oldMap2 != Map.Data())
 981          PkgRevDepends += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap2;
 982    }
 983
 984    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 985    Link->ParentVer = Ver.Index();
 986    Link->DependencyData = DependencyData;
 987    Link->ID = Cache.HeaderP->DependsCount++;
 988
 989    pkgCache::DepIterator Dep(Cache, Link);
 990    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 991    if (isDuplicate == false)
 992    {
 993       Dep->Type = Type;
 994       Dep->CompareOp = Op;
 995       Dep->Version = Version;
 996       Dep->Package = Pkg.Index();
 997       ++Cache.HeaderP->DependsDataCount;
 998       Link->NextRevDepends = Pkg->RevDepends;
 999       Pkg->RevDepends = Dependency;
1000    }
1001    else
1002    {
1003       // dependency data is already fine, we just set the reverse link
1004       // and in such a way that the loop above can finish fast, so we keep
1005       // non-duplicates at the front and for the duplicates we:
1006       // a) move to the end of the list, b) insert before another own duplicate
1007       // or c) find two duplicates behind each other.
1008       map_pointer_t const own_id = Link->DependencyData;
1009       while (*PkgRevDepends != 0)
1010       {
1011         pkgCache::Dependency * const L = Cache.DepP + *PkgRevDepends;
1012         PkgRevDepends = &L->NextRevDepends;
1013         if (L->DependencyData == own_id || L->DependencyData == previous_id)
1014         {
1015            Link->NextRevDepends = L->NextRevDepends;
1016            break;
1017         }
1018         previous_id = L->DependencyData;
1019       }
1020       *PkgRevDepends = Dependency;
1021    }
1022
1023    // Do we know where to link the Dependency to?
1024    if (OldDepLast == NULL)
1025    {
1026       OldDepLast = &Ver->DependsList;
1027       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1028          OldDepLast = &D->NextDepends;
1029    } else if (oldMap != Map.Data())
1030       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1031
1032    Dep->NextDepends = *OldDepLast;
1033    *OldDepLast = Dependency;
1034    OldDepLast = &Dep->NextDepends;
1035    return true;
1036 }
1037                                                                         /*}}}*/
1038 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1039 // ---------------------------------------------------------------------
1040 /* This creates a Group and the Package to link this dependency to if
1041    needed and handles also the caching of the old endpoint */
1042 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
1043                                                const string &PackageName,
1044                                                const string &Arch,
1045                                                const string &Version,
1046                                                unsigned int Op,
1047                                                unsigned int Type)
1048 {
1049    pkgCache::GrpIterator Grp;
1050    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1051    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1052       return false;
1053
1054    // Locate the target package
1055    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
1056    // we don't create 'none' packages and their dependencies if we can avoid it …
1057    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
1058       return true;
1059    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1060    if (Pkg.end() == true) {
1061       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1062          return false;
1063    }
1064
1065    // Is it a file dependency?
1066    if (unlikely(PackageName[0] == '/'))
1067       FoundFileDeps = true;
1068
1069    /* Caching the old end point speeds up generation substantially */
1070    if (OldDepVer != Ver) {
1071       OldDepLast = NULL;
1072       OldDepVer = Ver;
1073    }
1074
1075    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1076 }
1077                                                                         /*}}}*/
1078 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1079 // ---------------------------------------------------------------------
1080 /* */
1081 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1082                                                 const string &PkgName,
1083                                                 const string &PkgArch,
1084                                                 const string &Version)
1085 {
1086    pkgCache &Cache = Owner->Cache;
1087
1088    // We do not add self referencing provides
1089    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1090         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1091       return true;
1092
1093    // Get a structure
1094    map_pointer_t const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
1095    if (unlikely(Provides == 0))
1096       return false;
1097    Cache.HeaderP->ProvidesCount++;
1098
1099    // Fill it in
1100    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1101    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
1102    Prv->Version = Ver.Index();
1103    Prv->NextPkgProv = Ver->ProvidesList;
1104    Ver->ProvidesList = Prv.Index();
1105    if (Version.empty() == false) {
1106       map_stringitem_t const idxProvideVersion = WriteString(Version);
1107       Prv->ProvideVersion = idxProvideVersion;
1108       if (unlikely(idxProvideVersion == 0))
1109          return false;
1110    }
1111
1112    // Locate the target package
1113    pkgCache::PkgIterator Pkg;
1114    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1115    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1116       return false;
1117
1118    // Link it to the package
1119    Prv->ParentPkg = Pkg.Index();
1120    Prv->NextProvides = Pkg->ProvidesList;
1121    Pkg->ProvidesList = Prv.Index();
1122
1123    return true;
1124 }
1125                                                                         /*}}}*/
1126 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1127       pkgCache::VerIterator const &Ver)
1128 {
1129    return Hash == Ver->Hash;
1130 }
1131                                                                         /*}}}*/
1132 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1133 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1134                                    unsigned long Flags)
1135 {
1136    if (File.empty() && Site.empty())
1137    {
1138       CurrentRlsFile = NULL;
1139       return true;
1140    }
1141
1142    // Get some space for the structure
1143    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1144    if (unlikely(idxFile == 0))
1145       return false;
1146    CurrentRlsFile = Cache.RlsFileP + idxFile;
1147
1148    // Fill it in
1149    map_stringitem_t const idxFileName = WriteStringInMap(File);
1150    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1151    if (unlikely(idxFileName == 0 || idxSite == 0))
1152       return false;
1153    CurrentRlsFile->FileName = idxFileName;
1154    CurrentRlsFile->Site = idxSite;
1155    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1156    CurrentRlsFile->Flags = Flags;
1157    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1158    RlsFileName = File;
1159    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1160    Cache.HeaderP->ReleaseFileCount++;
1161
1162    return true;
1163 }
1164                                                                         /*}}}*/
1165 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1166 // ---------------------------------------------------------------------
1167 /* This is used to select which file is to be associated with all newly
1168    added versions. The caller is responsible for setting the IMS fields. */
1169 bool pkgCacheGenerator::SelectFile(std::string const &File,
1170                                    pkgIndexFile const &Index,
1171                                    std::string const &Architecture,
1172                                    std::string const &Component,
1173                                    unsigned long const Flags)
1174 {
1175    // Get some space for the structure
1176    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1177    if (unlikely(idxFile == 0))
1178       return false;
1179    CurrentFile = Cache.PkgFileP + idxFile;
1180
1181    // Fill it in
1182    map_stringitem_t const idxFileName = WriteStringInMap(File);
1183    if (unlikely(idxFileName == 0))
1184       return false;
1185    CurrentFile->FileName = idxFileName;
1186    CurrentFile->NextFile = Cache.HeaderP->FileList;
1187    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1188    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1189    if (unlikely(idxIndexType == 0))
1190       return false;
1191    CurrentFile->IndexType = idxIndexType;
1192    if (Architecture.empty())
1193       CurrentFile->Architecture = 0;
1194    else
1195    {
1196       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1197       if (unlikely(arch == 0))
1198          return false;
1199       CurrentFile->Architecture = arch;
1200    }
1201    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1202    if (unlikely(component == 0))
1203       return false;
1204    CurrentFile->Component = component;
1205    CurrentFile->Flags = Flags;
1206    if (CurrentRlsFile != NULL)
1207       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1208    else
1209       CurrentFile->Release = 0;
1210    PkgFileName = File;
1211    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1212    Cache.HeaderP->PackageFileCount++;
1213
1214    if (Progress != 0)
1215       Progress->SubProgress(Index.Size());
1216    return true;
1217 }
1218                                                                         /*}}}*/
1219 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1220 // ---------------------------------------------------------------------
1221 /* This is used to create handles to strings. Given the same text it
1222    always returns the same number */
1223 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1224                                                  unsigned int Size)
1225 {
1226    std::string const key(S, Size);
1227
1228    std::map<std::string,map_stringitem_t> * strings;
1229    switch(type) {
1230       case MIXED: strings = &strMixed; break;
1231       case PKGNAME: strings = &strPkgNames; break;
1232       case VERSIONNUMBER: strings = &strVersions; break;
1233       case SECTION: strings = &strSections; break;
1234       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1235    }
1236
1237    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1238    if (item != strings->end())
1239       return item->second;
1240
1241    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1242    strings->insert(std::make_pair(key, idxString));
1243    return idxString;
1244 }
1245                                                                         /*}}}*/
1246 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1247 // ---------------------------------------------------------------------
1248 /* This just verifies that each file in the list of index files exists,
1249    has matching attributes with the cache and the cache does not have
1250    any extra files. */
1251 static bool CheckValidity(const string &CacheFile,
1252                           pkgSourceList &List,
1253                           FileIterator Start,
1254                           FileIterator End,
1255                           MMap **OutMap = 0)
1256 {
1257    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1258    // No file, certainly invalid
1259    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1260    {
1261       if (Debug == true)
1262          std::clog << "CacheFile doesn't exist" << std::endl;
1263       return false;
1264    }
1265
1266    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1267    {
1268       if (Debug == true)
1269          std::clog << "sources.list is newer than the cache" << std::endl;
1270       return false;
1271    }
1272
1273    // Map it
1274    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1275    SPtr<MMap> Map = new MMap(CacheF,0);
1276    pkgCache Cache(Map);
1277    if (_error->PendingError() == true || Map->Size() == 0)
1278    {
1279       if (Debug == true)
1280          std::clog << "Errors are pending or Map is empty()" << std::endl;
1281       _error->Discard();
1282       return false;
1283    }
1284
1285    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1286    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1287    std::vector<pkgIndexFile *> Files;
1288    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1289    {
1290       if (Debug == true)
1291          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1292       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1293       if (RlsFile.end() == true)
1294       {
1295          if (Debug == true)
1296             std::clog << "FindInCache returned end-Pointer" << std::endl;
1297          return false;
1298       }
1299
1300       RlsVisited[RlsFile->ID] = true;
1301       if (Debug == true)
1302          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1303
1304       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1305       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1306          if ((*j)->HasPackages())
1307             Files.push_back (*j);
1308    }
1309    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1310       if (RlsVisited[I] == false)
1311       {
1312          if (Debug == true)
1313             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1314          return false;
1315       }
1316
1317    for (; Start != End; ++Start)
1318       Files.push_back(*Start);
1319
1320    /* Now we check every index file, see if it is in the cache,
1321       verify the IMS data and check that it is on the disk too.. */
1322    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1323    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1324    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1325    {
1326       if (Debug == true)
1327          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1328       if ((*PkgFile)->Exists() == false)
1329       {
1330          if (Debug == true)
1331             std::clog << "file doesn't exist" << std::endl;
1332          continue;
1333       }
1334
1335       // FindInCache is also expected to do an IMS check.
1336       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1337       if (File.end() == true)
1338       {
1339          if (Debug == true)
1340             std::clog << "FindInCache returned end-Pointer" << std::endl;
1341          return false;
1342       }
1343
1344       Visited[File->ID] = true;
1345       if (Debug == true)
1346          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1347    }
1348
1349    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1350       if (Visited[I] == false)
1351       {
1352          if (Debug == true)
1353             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1354          return false;
1355       }
1356
1357    if (_error->PendingError() == true)
1358    {
1359       if (Debug == true)
1360       {
1361          std::clog << "Validity failed because of pending errors:" << std::endl;
1362          _error->DumpErrors();
1363       }
1364       _error->Discard();
1365       return false;
1366    }
1367
1368    if (OutMap != 0)
1369       *OutMap = Map.UnGuard();
1370    return true;
1371 }
1372                                                                         /*}}}*/
1373 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1374 // ---------------------------------------------------------------------
1375 /* Size is kind of an abstract notion that is only used for the progress
1376    meter */
1377 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1378 {
1379    map_filesize_t TotalSize = 0;
1380    if (List !=  NULL)
1381    {
1382       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1383       {
1384          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1385          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1386             if ((*j)->HasPackages() == true)
1387                TotalSize += (*j)->Size();
1388       }
1389    }
1390
1391    for (; Start < End; ++Start)
1392    {
1393       if ((*Start)->HasPackages() == false)
1394          continue;
1395       TotalSize += (*Start)->Size();
1396    }
1397    return TotalSize;
1398 }
1399                                                                         /*}}}*/
1400 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1401 // ---------------------------------------------------------------------
1402 /* */
1403 static bool BuildCache(pkgCacheGenerator &Gen,
1404                        OpProgress *Progress,
1405                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1406                        pkgSourceList const * const List,
1407                        FileIterator Start, FileIterator End)
1408 {
1409    std::vector<pkgIndexFile *> Files;
1410    bool const HasFileDeps = Gen.HasFileDeps();
1411
1412    if (List !=  NULL)
1413    {
1414       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1415       {
1416          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1417          {
1418             _error->Warning("Duplicate sources.list entry %s",
1419                   (*i)->Describe().c_str());
1420             continue;
1421          }
1422
1423          if ((*i)->Merge(Gen, Progress) == false)
1424             return false;
1425
1426          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1427          for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
1428          {
1429             if (HasFileDeps)
1430                Files.push_back(*I);
1431
1432             if ((*I)->HasPackages() == false)
1433                continue;
1434
1435             if ((*I)->Exists() == false)
1436                continue;
1437
1438             if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1439             {
1440                _error->Warning("Duplicate sources.list entry %s",
1441                      (*I)->Describe().c_str());
1442                continue;
1443             }
1444
1445             map_filesize_t Size = (*I)->Size();
1446             if (Progress != NULL)
1447                Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1448             CurrentSize += Size;
1449
1450             if ((*I)->Merge(Gen,Progress) == false)
1451                return false;
1452          }
1453       }
1454    }
1455
1456    Gen.SelectReleaseFile("", "");
1457    FileIterator I;
1458    for (I = Start; I != End; ++I)
1459    {
1460       if (HasFileDeps)
1461          Files.push_back(*I);
1462
1463       if ((*I)->HasPackages() == false)
1464          continue;
1465
1466       if ((*I)->Exists() == false)
1467          continue;
1468
1469       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1470       {
1471          _error->Warning("Duplicate sources.list entry %s",
1472                          (*I)->Describe().c_str());
1473          continue;
1474       }
1475
1476       map_filesize_t Size = (*I)->Size();
1477       if (Progress != NULL)
1478          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1479       CurrentSize += Size;
1480
1481       if ((*I)->Merge(Gen,Progress) == false)
1482          return false;
1483    }
1484
1485    if (HasFileDeps == true)
1486    {
1487       if (Progress != NULL)
1488          Progress->Done();
1489       TotalSize = ComputeSize(List, Start, End);
1490       CurrentSize = 0;
1491       for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
1492       {
1493          map_filesize_t Size = (*I)->Size();
1494          if (Progress != NULL)
1495             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1496          CurrentSize += Size;
1497          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1498             return false;
1499       }
1500    }
1501
1502    return true;
1503 }
1504                                                                         /*}}}*/
1505 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1506 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1507    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1508    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1509    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1510    Flags |= MMap::Moveable;
1511    if (_config->FindB("APT::Cache-Fallback", false) == true)
1512       Flags |= MMap::Fallback;
1513    if (CacheF != NULL)
1514       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1515    else
1516       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1517 }
1518                                                                         /*}}}*/
1519 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1520 // ---------------------------------------------------------------------
1521 /* This makes sure that the status cache (the cache that has all
1522    index files from the sources list and all local ones) is ready
1523    to be mmaped. If OutMap is not zero then a MMap object representing
1524    the cache will be stored there. This is pretty much mandetory if you
1525    are using AllowMem. AllowMem lets the function be run as non-root
1526    where it builds the cache 'fast' into a memory buffer. */
1527 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1528                         MMap **OutMap, bool AllowMem)
1529    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1530 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1531                         MMap **OutMap,bool AllowMem)
1532 {
1533    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1534
1535    std::vector<pkgIndexFile *> Files;
1536    /*
1537    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1538         i != List.end();
1539         ++i)
1540    {
1541       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1542       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1543            j != Indexes->end();
1544            ++j)
1545          Files.push_back (*j);
1546    }
1547 */
1548    if (_system->AddStatusFiles(Files) == false)
1549       return false;
1550
1551    // Decide if we can write to the files..
1552    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1553    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1554
1555    // ensure the cache directory exists
1556    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1557    {
1558       string dir = _config->FindDir("Dir::Cache");
1559       size_t const len = dir.size();
1560       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1561          dir = dir.substr(0, len - 5);
1562       if (CacheFile.empty() == false)
1563          CreateDirectory(dir, flNotFile(CacheFile));
1564       if (SrcCacheFile.empty() == false)
1565          CreateDirectory(dir, flNotFile(SrcCacheFile));
1566    }
1567
1568    // Decide if we can write to the cache
1569    bool Writeable = false;
1570    if (CacheFile.empty() == false)
1571       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1572    else
1573       if (SrcCacheFile.empty() == false)
1574          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1575    if (Debug == true)
1576       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1577
1578    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1579       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1580
1581    if (Progress != NULL)
1582       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1583
1584    // Cache is OK, Fin.
1585    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1586    {
1587       if (Progress != NULL)
1588          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1589       if (Debug == true)
1590          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1591       return true;
1592    }
1593    else if (Debug == true)
1594          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1595
1596    /* At this point we know we need to reconstruct the package cache,
1597       begin. */
1598    SPtr<FileFd> CacheF;
1599    SPtr<DynamicMMap> Map;
1600    if (Writeable == true && CacheFile.empty() == false)
1601    {
1602       _error->PushToStack();
1603       unlink(CacheFile.c_str());
1604       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1605       fchmod(CacheF->Fd(),0644);
1606       Map = CreateDynamicMMap(CacheF, MMap::Public);
1607       if (_error->PendingError() == true)
1608       {
1609          delete CacheF.UnGuard();
1610          delete Map.UnGuard();
1611          if (Debug == true)
1612             std::clog << "Open filebased MMap FAILED" << std::endl;
1613          Writeable = false;
1614          if (AllowMem == false)
1615          {
1616             _error->MergeWithStack();
1617             return false;
1618          }
1619          _error->RevertToStack();
1620       }
1621       else
1622       {
1623          _error->MergeWithStack();
1624          if (Debug == true)
1625             std::clog << "Open filebased MMap" << std::endl;
1626       }
1627    }
1628    if (Writeable == false || CacheFile.empty() == true)
1629    {
1630       // Just build it in memory..
1631       Map = CreateDynamicMMap(NULL);
1632       if (Debug == true)
1633          std::clog << "Open memory Map (not filebased)" << std::endl;
1634    }
1635
1636    // Lets try the source cache.
1637    map_filesize_t CurrentSize = 0;
1638    map_filesize_t TotalSize = 0;
1639    if (CheckValidity(SrcCacheFile, List, Files.end(),
1640                      Files.end()) == true)
1641    {
1642       if (Debug == true)
1643          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1644       // Preload the map with the source cache
1645       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1646       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1647       if ((alloc == 0 && _error->PendingError())
1648                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1649                                 SCacheF.Size()) == false)
1650          return false;
1651
1652       TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1653
1654       // Build the status cache
1655       pkgCacheGenerator Gen(Map.Get(),Progress);
1656       if (_error->PendingError() == true)
1657          return false;
1658       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1659                      Files.begin(),Files.end()) == false)
1660          return false;
1661    }
1662    else
1663    {
1664       if (Debug == true)
1665          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1666       TotalSize = ComputeSize(&List, Files.begin(),Files.end());
1667
1668       // Build the source cache
1669       pkgCacheGenerator Gen(Map.Get(),Progress);
1670       if (_error->PendingError() == true)
1671          return false;
1672       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
1673                      Files.end(),Files.end()) == false)
1674          return false;
1675
1676       // Write it back
1677       if (Writeable == true && SrcCacheFile.empty() == false)
1678       {
1679          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1680          if (_error->PendingError() == true)
1681             return false;
1682
1683          fchmod(SCacheF.Fd(),0644);
1684
1685          // Write out the main data
1686          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1687             return _error->Error(_("IO Error saving source cache"));
1688          SCacheF.Sync();
1689
1690          // Write out the proper header
1691          Gen.GetCache().HeaderP->Dirty = false;
1692          if (SCacheF.Seek(0) == false ||
1693              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1694             return _error->Error(_("IO Error saving source cache"));
1695          Gen.GetCache().HeaderP->Dirty = true;
1696          SCacheF.Sync();
1697       }
1698
1699       // Build the status cache
1700       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1701                      Files.begin(), Files.end()) == false)
1702          return false;
1703    }
1704    if (Debug == true)
1705       std::clog << "Caches are ready for shipping" << std::endl;
1706
1707    if (_error->PendingError() == true)
1708       return false;
1709    if (OutMap != 0)
1710    {
1711       if (CacheF != 0)
1712       {
1713          delete Map.UnGuard();
1714          *OutMap = new MMap(*CacheF,0);
1715       }
1716       else
1717       {
1718          *OutMap = Map.UnGuard();
1719       }
1720    }
1721
1722    return true;
1723 }
1724                                                                         /*}}}*/
1725 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1726 // ---------------------------------------------------------------------
1727 /* */
1728 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1729    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1730 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1731 {
1732    std::vector<pkgIndexFile *> Files;
1733    if (_system->AddStatusFiles(Files) == false)
1734       return false;
1735
1736    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1737    map_filesize_t CurrentSize = 0;
1738    map_filesize_t TotalSize = 0;
1739
1740    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1741
1742    // Build the status cache
1743    if (Progress != NULL)
1744       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1745    pkgCacheGenerator Gen(Map.Get(),Progress);
1746    if (_error->PendingError() == true)
1747       return false;
1748    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1749                   Files.begin(), Files.end()) == false)
1750       return false;
1751
1752    if (_error->PendingError() == true)
1753       return false;
1754    *OutMap = Map.UnGuard();
1755
1756    return true;
1757 }
1758                                                                         /*}}}*/
1759 // IsDuplicateDescription                                               /*{{{*/
1760 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1761                             MD5SumValue const &CurMd5, std::string const &CurLang)
1762 {
1763    // Descriptions in the same link-list have all the same md5
1764    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1765       return false;
1766    for (; Desc.end() == false; ++Desc)
1767       if (Desc.LanguageCode() == CurLang)
1768          return true;
1769    return false;
1770 }
1771                                                                         /*}}}*/
1772 // CacheGenerator::FinishCache                                          /*{{{*/
1773 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1774 {
1775    return true;
1776 }
1777                                                                         /*}}}*/
1778
1779 pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
1780 pkgCacheGenerator::ListParser::~ListParser() {}