apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                      CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
  58 {
  59    if (_error->PendingError() == true)
  60       return;
  61
  62    if (Map.Size() == 0)
  63    {
  64       // Setup the map interface..
  65       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  66       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  67          return;
  68
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70
  71       // Starting header
  72       *Cache.HeaderP = pkgCache::Header();
  73
  74       // make room for the hashtables for packages and groups
  75       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  76          return;
  77
  78       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  79       if (unlikely(idxVerSysName == 0))
  80          return;
  81       Cache.HeaderP->VerSysName = idxVerSysName;
  82       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  83       if (unlikely(idxArchitecture == 0))
  84          return;
  85       Cache.HeaderP->Architecture = idxArchitecture;
  86
  87       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  88       if (archs.size() > 1)
  89       {
  90          std::vector<std::string>::const_iterator a = archs.begin();
  91          std::string list = *a;
  92          for (++a; a != archs.end(); ++a)
  93             list.append(",").append(*a);
  94          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  95          if (unlikely(idxArchitectures == 0))
  96             return;
  97          Cache.HeaderP->SetArchitectures(idxArchitectures);
  98       }
  99       else
 100          Cache.HeaderP->SetArchitectures(idxArchitecture);
 101
 102       Cache.ReMap();
 103    }
 104    else
 105    {
 106       // Map directly from the existing file
 107       Cache.ReMap();
 108       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 109       if (Cache.VS != _system->VS)
 110       {
 111          _error->Error(_("Cache has an incompatible versioning system"));
 112          return;
 113       }
 114    }
 115
 116    Cache.HeaderP->Dirty = true;
 117    Map.Sync(0,sizeof(pkgCache::Header));
 118 }
 119                                                                         /*}}}*/
 120 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 121 // ---------------------------------------------------------------------
 122 /* We sync the data then unset the dirty flag in two steps so as to
 123    advoid a problem during a crash */
 124 pkgCacheGenerator::~pkgCacheGenerator()
 125 {
 126    if (_error->PendingError() == true)
 127       return;
 128    if (Map.Sync() == false)
 129       return;
 130
 131    Cache.HeaderP->Dirty = false;
 132    Cache.HeaderP->CacheFileSize = Map.Size();
 133    Map.Sync(0,sizeof(pkgCache::Header));
 134 }
 135                                                                         /*}}}*/
 136 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 137    if (oldMap == newMap)
 138       return;
 139
 140    if (_config->FindB("Debug::pkgCacheGen", false))
 141       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 142
 143    Cache.ReMap(false);
 144
 145    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 146    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 147
 148    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 149         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 150       (*i)->ReMap(oldMap, newMap);
 151    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 152         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 153       (*i)->ReMap(oldMap, newMap);
 154    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 155         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 156       (*i)->ReMap(oldMap, newMap);
 157    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 158         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 159       (*i)->ReMap(oldMap, newMap);
 160    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 161         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 162       (*i)->ReMap(oldMap, newMap);
 163    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 164         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 165       (*i)->ReMap(oldMap, newMap);
 166    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 168       (*i)->ReMap(oldMap, newMap);
 169    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 170         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 171       (*i)->ReMap(oldMap, newMap);
 172 }                                                                       /*}}}*/
 173 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 174 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 175                                         const unsigned long &Len) {
 176    void const * const oldMap = Map.Data();
 177    map_stringitem_t const index = Map.WriteString(String, Len);
 178    if (index != 0)
 179       ReMap(oldMap, Map.Data());
 180    return index;
 181 }
 182                                                                         /*}}}*/
 183 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 184 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 185    void const * const oldMap = Map.Data();
 186    map_stringitem_t const index = Map.WriteString(String);
 187    if (index != 0)
 188       ReMap(oldMap, Map.Data());
 189    return index;
 190 }
 191                                                                         /*}}}*/
 192 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 193    void const * const oldMap = Map.Data();
 194    map_pointer_t const index = Map.Allocate(size);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This provides the generation of the entries in the cache. Each loop
 203    goes through a single package record from the underlying parse engine. */
 204 bool pkgCacheGenerator::MergeList(ListParser &List,
 205                                   pkgCache::VerIterator *OutVer)
 206 {
 207    List.Owner = this;
 208
 209    unsigned int Counter = 0;
 210    while (List.Step() == true)
 211    {
 212       string const PackageName = List.Package();
 213       if (PackageName.empty() == true)
 214          return false;
 215
 216       Counter++;
 217       if (Counter % 100 == 0 && Progress != 0)
 218          Progress->Progress(List.Offset());
 219
 220       string Arch = List.Architecture();
 221       string const Version = List.Version();
 222       if (Version.empty() == true && Arch.empty() == true)
 223       {
 224          // package descriptions
 225          if (MergeListGroup(List, PackageName) == false)
 226             return false;
 227          continue;
 228       }
 229
 230       if (Arch.empty() == true)
 231       {
 232          // use the pseudo arch 'none' for arch-less packages
 233          Arch = "none";
 234          /* We might built a SingleArchCache here, which we don't want to blow up
 235             just for these :none packages to a proper MultiArchCache, so just ensure
 236             that we have always a native package structure first for SingleArch */
 237          pkgCache::PkgIterator NP;
 238          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 239          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 240          // TRANSLATOR: The first placeholder is a package name,
 241          // the other two should be copied verbatim as they include debug info
 242          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 243                               PackageName.c_str(), "NewPackage", 0);
 244       }
 245
 246       // Get a pointer to the package structure
 247       pkgCache::PkgIterator Pkg;
 248       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 249       if (NewPackage(Pkg, PackageName, Arch) == false)
 250          // TRANSLATOR: The first placeholder is a package name,
 251          // the other two should be copied verbatim as they include debug info
 252          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 253                               PackageName.c_str(), "NewPackage", 1);
 254
 255
 256       if (Version.empty() == true)
 257       {
 258          if (MergeListPackage(List, Pkg) == false)
 259             return false;
 260       }
 261       else
 262       {
 263          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 264             return false;
 265       }
 266
 267       if (OutVer != 0)
 268       {
 269          FoundFileDeps |= List.HasFileDeps();
 270          return true;
 271       }
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    FoundFileDeps |= List.HasFileDeps();
 288    return true;
 289 }
 290 // CacheGenerator::MergeListGroup                                       /*{{{*/
 291 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 292 {
 293    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 294    // a group has no data on it's own, only packages have it but these
 295    // stanzas like this come from Translation- files to add descriptions,
 296    // but without a version we don't need a description for it…
 297    if (Grp.end() == true)
 298       return true;
 299    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 300
 301    pkgCache::PkgIterator Pkg;
 302    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 303    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 304       if (MergeListPackage(List, Pkg) == false)
 305          return false;
 306
 307    return true;
 308 }
 309                                                                         /*}}}*/
 310 // CacheGenerator::MergeListPackage                                     /*{{{*/
 311 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 312 {
 313    // we first process the package, then the descriptions
 314    // (for deb this package processing is in fact a no-op)
 315    pkgCache::VerIterator Ver(Cache);
 316    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 317    if (List.UsePackage(Pkg, Ver) == false)
 318       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 319                            Pkg.Name(), "UsePackage", 1);
 320
 321    // Find the right version to write the description
 322    MD5SumValue CurMd5 = List.Description_md5();
 323    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 324       return true;
 325    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 326    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 327    {
 328       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 329
 330       // a version can only have one md5 describing it
 331       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 332          continue;
 333
 334       map_stringitem_t md5idx = VerDesc->md5sum;
 335       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 336       {
 337          // don't add a new description if we have one for the given
 338          // md5 && language
 339          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 340             continue;
 341
 342          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 343       }
 344
 345       // we can stop here as all "same" versions will share the description
 346       break;
 347    }
 348
 349    return true;
 350 }
 351                                                                         /*}}}*/
 352 // CacheGenerator::MergeListVersion                                     /*{{{*/
 353 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 354                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 355 {
 356    pkgCache::VerIterator Ver = Pkg.VersionList();
 357    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 358    map_pointer_t *LastVer = &Pkg->VersionList;
 359    void const * oldMap = Map.Data();
 360
 361    unsigned short const Hash = List.VersionHash();
 362    if (Ver.end() == false)
 363    {
 364       /* We know the list is sorted so we use that fact in the search.
 365          Insertion of new versions is done with correct sorting */
 366       int Res = 1;
 367       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 368       {
 369          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 370          // Version is higher as current version - insert here
 371          if (Res > 0)
 372             break;
 373          // Versionstrings are equal - is hash also equal?
 374          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 375             break;
 376          // proceed with the next till we have either the right
 377          // or we found another version (which will be lower)
 378       }
 379
 380       /* We already have a version for this item, record that we saw it */
 381       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 382       {
 383          if (List.UsePackage(Pkg,Ver) == false)
 384             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 385                                  Pkg.Name(), "UsePackage", 2);
 386
 387          if (NewFileVer(Ver,List) == false)
 388             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 389                                  Pkg.Name(), "NewFileVer", 1);
 390
 391          // Read only a single record and return
 392          if (OutVer != 0)
 393          {
 394             *OutVer = Ver;
 395             return true;
 396          }
 397
 398          return true;
 399       }
 400    }
 401
 402    // Add a new version
 403    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 404    if (verindex == 0 && _error->PendingError())
 405       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 406                            Pkg.Name(), "NewVersion", 1);
 407
 408    if (oldMap != Map.Data())
 409          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 410    *LastVer = verindex;
 411
 412    if (unlikely(List.NewVersion(Ver) == false))
 413       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 414                            Pkg.Name(), "NewVersion", 2);
 415
 416    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 417       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 418                            Pkg.Name(), "UsePackage", 3);
 419
 420    if (unlikely(NewFileVer(Ver,List) == false))
 421       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 422                            Pkg.Name(), "NewFileVer", 2);
 423
 424    pkgCache::GrpIterator Grp = Pkg.Group();
 425    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 426
 427    /* If it is the first version of this package we need to add implicit
 428       Multi-Arch dependencies to all other package versions in the group now -
 429       otherwise we just add them for this new version */
 430    if (Pkg.VersionList()->NextVer == 0)
 431    {
 432       pkgCache::PkgIterator P = Grp.PackageList();
 433       Dynamic<pkgCache::PkgIterator> DynP(P);
 434       for (; P.end() != true; P = Grp.NextPkg(P))
 435       {
 436          if (P->ID == Pkg->ID)
 437             continue;
 438          pkgCache::VerIterator V = P.VersionList();
 439          Dynamic<pkgCache::VerIterator> DynV(V);
 440          for (; V.end() != true; ++V)
 441             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 442                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 443                                     Pkg.Name(), "AddImplicitDepends", 1);
 444       }
 445       /* :none packages are packages without an architecture. They are forbidden by
 446          debian-policy, so usually they will only be in (old) dpkg status files -
 447          and dpkg will complain about them - and are pretty rare. We therefore do
 448          usually not create conflicts while the parent is created, but only if a :none
 449          package (= the target) appears. This creates incorrect dependencies on :none
 450          for architecture-specific dependencies on the package we copy from, but we
 451          will ignore this bug as architecture-specific dependencies are only allowed
 452          in jessie and until then the :none packages should be extinct (hopefully).
 453          In other words: This should work long enough to allow graceful removal of
 454          these packages, it is not supposed to allow users to keep using them … */
 455       if (strcmp(Pkg.Arch(), "none") == 0)
 456       {
 457          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 458          if (M.end() == false && Pkg != M)
 459          {
 460             pkgCache::DepIterator D = M.RevDependsList();
 461             Dynamic<pkgCache::DepIterator> DynD(D);
 462             for (; D.end() == false; ++D)
 463             {
 464                if ((D->Type != pkgCache::Dep::Conflicts &&
 465                     D->Type != pkgCache::Dep::DpkgBreaks &&
 466                     D->Type != pkgCache::Dep::Replaces) ||
 467                    D.ParentPkg().Group() == Grp)
 468                   continue;
 469
 470                map_pointer_t *OldDepLast = NULL;
 471                pkgCache::VerIterator ConVersion = D.ParentVer();
 472                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 473                // duplicate the Conflicts/Breaks/Replaces for :none arch
 474                NewDepends(Pkg, ConVersion, D->Version,
 475                      D->CompareOp, D->Type, OldDepLast);
 476             }
 477          }
 478       }
 479    }
 480    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482                            Pkg.Name(), "AddImplicitDepends", 2);
 483
 484    // Read only a single record and return
 485    if (OutVer != 0)
 486    {
 487       *OutVer = Ver;
 488       return true;
 489    }
 490
 491    /* Record the Description(s) based on their master md5sum */
 492    MD5SumValue CurMd5 = List.Description_md5();
 493    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 494       return true;
 495
 496    /* Before we add a new description we first search in the group for
 497       a version with a description of the same MD5 - if so we reuse this
 498       description group instead of creating our own for this version */
 499    for (pkgCache::PkgIterator P = Grp.PackageList();
 500         P.end() == false; P = Grp.NextPkg(P))
 501    {
 502       for (pkgCache::VerIterator V = P.VersionList();
 503            V.end() == false; ++V)
 504       {
 505          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 506             continue;
 507          Ver->DescriptionList = V->DescriptionList;
 508       }
 509    }
 510
 511    // We haven't found reusable descriptions, so add the first description(s)
 512    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 513    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 514    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 515       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 516          return false;
 517    return true;
 518 }
 519                                                                         /*}}}*/
 520 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 521 {
 522    pkgCache::DescIterator Desc;
 523    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 524
 525    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 526    if (unlikely(descindex == 0 && _error->PendingError()))
 527       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 528             Ver.ParentPkg().Name(), "NewDescription", 1);
 529
 530    md5idx = Desc->md5sum;
 531    Desc->ParentPkg = Ver.ParentPkg().Index();
 532
 533    // we add at the end, so that the start is constant as we need
 534    // that to be able to efficiently share these lists
 535    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 536    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 537    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 538    *LastNextDesc = descindex;
 539
 540    if (NewFileDesc(Desc,List) == false)
 541       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 542             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 543
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547                                                                         /*}}}*/
 548 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* If we found any file depends while parsing the main list we need to
 551    resolve them. Since it is undesired to load the entire list of files
 552    into the cache as virtual packages we do a two stage effort. MergeList
 553    identifies the file depends and this creates Provdies for them by
 554    re-parsing all the indexs. */
 555 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 556 {
 557    List.Owner = this;
 558
 559    unsigned int Counter = 0;
 560    while (List.Step() == true)
 561    {
 562       string PackageName = List.Package();
 563       if (PackageName.empty() == true)
 564          return false;
 565       string Version = List.Version();
 566       if (Version.empty() == true)
 567          continue;
 568
 569       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 570       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 571       if (Pkg.end() == true)
 572          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 573                                 PackageName.c_str(), "FindPkg", 1);
 574       Counter++;
 575       if (Counter % 100 == 0 && Progress != 0)
 576          Progress->Progress(List.Offset());
 577
 578       unsigned short Hash = List.VersionHash();
 579       pkgCache::VerIterator Ver = Pkg.VersionList();
 580       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 581       for (; Ver.end() == false; ++Ver)
 582       {
 583          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 584          {
 585             if (List.CollectFileProvides(Cache,Ver) == false)
 586                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 587                                     PackageName.c_str(), "CollectFileProvides", 1);
 588             break;
 589          }
 590       }
 591
 592       if (Ver.end() == true)
 593          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 594    }
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This creates a new group structure and adds it to the hash table */
 602 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 603 {
 604    Grp = Cache.FindGrp(Name);
 605    if (Grp.end() == false)
 606       return true;
 607
 608    // Get a structure
 609    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 610    if (unlikely(Group == 0))
 611       return false;
 612
 613    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 614    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 615    if (unlikely(idxName == 0))
 616       return false;
 617    Grp->Name = idxName;
 618
 619    // Insert it into the hash table
 620    unsigned long const Hash = Cache.Hash(Name);
 621    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 622    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 623       insertAt = &(Cache.GrpP + *insertAt)->Next;
 624    Grp->Next = *insertAt;
 625    *insertAt = Group;
 626
 627    Grp->ID = Cache.HeaderP->GroupCount++;
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* This creates a new package structure and adds it to the hash table */
 634 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 635                                         const string &Arch) {
 636    pkgCache::GrpIterator Grp;
 637    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 638    if (unlikely(NewGroup(Grp, Name) == false))
 639       return false;
 640
 641    Pkg = Grp.FindPkg(Arch);
 642       if (Pkg.end() == false)
 643          return true;
 644
 645    // Get a structure
 646    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 647    if (unlikely(Package == 0))
 648       return false;
 649    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 650
 651    // Insert the package into our package list
 652    if (Grp->FirstPackage == 0) // the group is new
 653    {
 654       Grp->FirstPackage = Package;
 655       // Insert it into the hash table
 656       map_id_t const Hash = Cache.Hash(Name);
 657       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 658       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 659          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 660       Pkg->NextPackage = *insertAt;
 661       *insertAt = Package;
 662    }
 663    else // Group the Packages together
 664    {
 665       // this package is the new last package
 666       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 667       Pkg->NextPackage = LastPkg->NextPackage;
 668       LastPkg->NextPackage = Package;
 669    }
 670    Grp->LastPackage = Package;
 671
 672    // Set the name, arch and the ID
 673    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 674    Pkg->Group = Grp.Index();
 675    // all is mapped to the native architecture
 676    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 677    if (unlikely(idxArch == 0))
 678       return false;
 679    Pkg->Arch = idxArch;
 680    Pkg->ID = Cache.HeaderP->PackageCount++;
 681
 682    return true;
 683 }
 684                                                                         /*}}}*/
 685 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 686 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 687                                            pkgCache::PkgIterator &P,
 688                                            pkgCache::VerIterator &V)
 689 {
 690    // copy P.Arch() into a string here as a cache remap
 691    // in NewDepends() later may alter the pointer location
 692    string Arch = P.Arch() == NULL ? "" : P.Arch();
 693    map_pointer_t *OldDepLast = NULL;
 694    /* MultiArch handling introduces a lot of implicit Dependencies:
 695       - MultiArch: same → Co-Installable if they have the same version
 696       - All others conflict with all other group members */
 697    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 698    pkgCache::PkgIterator D = G.PackageList();
 699    Dynamic<pkgCache::PkgIterator> DynD(D);
 700    map_stringitem_t const VerStrIdx = V->VerStr;
 701    for (; D.end() != true; D = G.NextPkg(D))
 702    {
 703       if (Arch == D.Arch() || D->VersionList == 0)
 704          continue;
 705       /* We allow only one installed arch at the time
 706          per group, therefore each group member conflicts
 707          with all other group members */
 708       if (coInstall == true)
 709       {
 710          // Replaces: ${self}:other ( << ${binary:Version})
 711          NewDepends(D, V, VerStrIdx,
 712                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 713                     OldDepLast);
 714          // Breaks: ${self}:other (!= ${binary:Version})
 715          NewDepends(D, V, VerStrIdx,
 716                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 717                     OldDepLast);
 718       } else {
 719          // Conflicts: ${self}:other
 720          NewDepends(D, V, 0,
 721                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 722                     OldDepLast);
 723       }
 724    }
 725    return true;
 726 }
 727 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 728                                            pkgCache::PkgIterator &D)
 729 {
 730    /* MultiArch handling introduces a lot of implicit Dependencies:
 731       - MultiArch: same → Co-Installable if they have the same version
 732       - All others conflict with all other group members */
 733    map_pointer_t *OldDepLast = NULL;
 734    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 735    if (coInstall == true)
 736    {
 737       map_stringitem_t const VerStrIdx = V->VerStr;
 738       // Replaces: ${self}:other ( << ${binary:Version})
 739       NewDepends(D, V, VerStrIdx,
 740                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 741                  OldDepLast);
 742       // Breaks: ${self}:other (!= ${binary:Version})
 743       NewDepends(D, V, VerStrIdx,
 744                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 745                  OldDepLast);
 746    } else {
 747       // Conflicts: ${self}:other
 748       NewDepends(D, V, 0,
 749                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 750                  OldDepLast);
 751    }
 752    return true;
 753 }
 754
 755                                                                         /*}}}*/
 756 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 757 // ---------------------------------------------------------------------
 758 /* */
 759 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 760                                    ListParser &List)
 761 {
 762    if (CurrentFile == 0)
 763       return true;
 764
 765    // Get a structure
 766    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 767    if (VerFile == 0)
 768       return false;
 769
 770    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 771    VF->File = CurrentFile - Cache.PkgFileP;
 772
 773    // Link it to the end of the list
 774    map_pointer_t *Last = &Ver->FileList;
 775    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 776       Last = &V->NextFile;
 777    VF->NextFile = *Last;
 778    *Last = VF.Index();
 779
 780    VF->Offset = List.Offset();
 781    VF->Size = List.Size();
 782    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 783       Cache.HeaderP->MaxVerFileSize = VF->Size;
 784    Cache.HeaderP->VerFileCount++;
 785
 786    return true;
 787 }
 788                                                                         /*}}}*/
 789 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 790 // ---------------------------------------------------------------------
 791 /* This puts a version structure in the linked list */
 792 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 793                                             const string &VerStr,
 794                                             map_pointer_t const ParentPkg,
 795                                             unsigned short const Hash,
 796                                             map_pointer_t const Next)
 797 {
 798    // Get a structure
 799    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 800    if (Version == 0)
 801       return 0;
 802
 803    // Fill it in
 804    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 805    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 806    Ver->NextVer = Next;
 807    Ver->ParentPkg = ParentPkg;
 808    Ver->Hash = Hash;
 809    Ver->ID = Cache.HeaderP->VersionCount++;
 810
 811    // try to find the version string in the group for reuse
 812    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 813    pkgCache::GrpIterator Grp = Pkg.Group();
 814    if (Pkg.end() == false && Grp.end() == false)
 815    {
 816       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 817       {
 818          if (Pkg == P)
 819             continue;
 820          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 821          {
 822             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 823             if (cmp == 0)
 824             {
 825                Ver->VerStr = V->VerStr;
 826                return Version;
 827             }
 828             else if (cmp < 0)
 829                break;
 830          }
 831       }
 832    }
 833    // haven't found the version string, so create
 834    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 835    if (unlikely(idxVerStr == 0))
 836       return 0;
 837    Ver->VerStr = idxVerStr;
 838    return Version;
 839 }
 840                                                                         /*}}}*/
 841 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* */
 844 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 845                                    ListParser &List)
 846 {
 847    if (CurrentFile == 0)
 848       return true;
 849
 850    // Get a structure
 851    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 852    if (DescFile == 0)
 853       return false;
 854
 855    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 856    DF->File = CurrentFile - Cache.PkgFileP;
 857
 858    // Link it to the end of the list
 859    map_pointer_t *Last = &Desc->FileList;
 860    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 861       Last = &D->NextFile;
 862
 863    DF->NextFile = *Last;
 864    *Last = DF.Index();
 865
 866    DF->Offset = List.Offset();
 867    DF->Size = List.Size();
 868    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 869       Cache.HeaderP->MaxDescFileSize = DF->Size;
 870    Cache.HeaderP->DescFileCount++;
 871
 872    return true;
 873 }
 874                                                                         /*}}}*/
 875 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 876 // ---------------------------------------------------------------------
 877 /* This puts a description structure in the linked list */
 878 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 879                                             const string &Lang,
 880                                             const MD5SumValue &md5sum,
 881                                             map_stringitem_t const idxmd5str)
 882 {
 883    // Get a structure
 884    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 885    if (Description == 0)
 886       return 0;
 887
 888    // Fill it in
 889    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 890    Desc->ID = Cache.HeaderP->DescriptionCount++;
 891    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 892    if (unlikely(idxlanguage_code == 0))
 893       return 0;
 894    Desc->language_code = idxlanguage_code;
 895
 896    if (idxmd5str != 0)
 897       Desc->md5sum = idxmd5str;
 898    else
 899    {
 900       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 901       if (unlikely(idxmd5sum == 0))
 902          return 0;
 903       Desc->md5sum = idxmd5sum;
 904    }
 905
 906    return Description;
 907 }
 908                                                                         /*}}}*/
 909 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 910 // ---------------------------------------------------------------------
 911 /* This creates a dependency element in the tree. It is linked to the
 912    version and to the package that it is pointing to. */
 913 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 914                                    pkgCache::VerIterator &Ver,
 915                                    string const &Version,
 916                                    uint8_t const Op,
 917                                    uint8_t const Type,
 918                                    map_stringitem_t* &OldDepLast)
 919 {
 920    map_stringitem_t index = 0;
 921    if (Version.empty() == false)
 922    {
 923       int const CmpOp = Op & 0x0F;
 924       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 925       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 926          index = Ver->VerStr;
 927
 928       if (index == 0)
 929       {
 930          void const * const oldMap = Map.Data();
 931          index = StoreString(VERSIONNUMBER, Version);
 932          if (unlikely(index == 0))
 933             return false;
 934          if (OldDepLast != 0 && oldMap != Map.Data())
 935             OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 936       }
 937    }
 938    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 939 }
 940 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 941                                    pkgCache::VerIterator &Ver,
 942                                    map_pointer_t const Version,
 943                                    uint8_t const Op,
 944                                    uint8_t const Type,
 945                                    map_pointer_t* &OldDepLast)
 946 {
 947    void const * const oldMap = Map.Data();
 948    // Get a structure
 949    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 950    if (unlikely(Dependency == 0))
 951       return false;
 952
 953    bool isDuplicate = false;
 954    map_pointer_t DependencyData = 0;
 955    map_pointer_t PreviousData = 0;
 956    if (Pkg->RevDepends != 0)
 957    {
 958       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 959       DependencyData = L->DependencyData;
 960       do {
 961          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 962          if (Version > D->Version)
 963             break;
 964          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 965          {
 966             isDuplicate = true;
 967             break;
 968          }
 969          PreviousData = DependencyData;
 970          DependencyData = D->NextData;
 971       } while (DependencyData != 0);
 972    }
 973
 974    if (isDuplicate == false)
 975    {
 976       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 977       if (unlikely(DependencyData == 0))
 978         return false;
 979    }
 980
 981    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 982    Link->ParentVer = Ver.Index();
 983    Link->DependencyData = DependencyData;
 984    Link->ID = Cache.HeaderP->DependsCount++;
 985
 986    pkgCache::DepIterator Dep(Cache, Link);
 987    if (isDuplicate == false)
 988    {
 989       Dep->Type = Type;
 990       Dep->CompareOp = Op;
 991       Dep->Version = Version;
 992       Dep->Package = Pkg.Index();
 993       ++Cache.HeaderP->DependsDataCount;
 994       if (PreviousData != 0)
 995       {
 996          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
 997          Dep->NextData = D->NextData;
 998          D->NextData = DependencyData;
 999       }
1000       else if (Pkg->RevDepends != 0)
1001       {
1002          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1003          Dep->NextData = D->DependencyData;
1004       }
1005    }
1006
1007    if (isDuplicate == true || PreviousData != 0)
1008    {
1009       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1010       Link->NextRevDepends = L->NextRevDepends;
1011       L->NextRevDepends = Dependency;
1012    }
1013    else
1014    {
1015       Link->NextRevDepends = Pkg->RevDepends;
1016       Pkg->RevDepends = Dependency;
1017    }
1018
1019
1020    // Do we know where to link the Dependency to?
1021    if (OldDepLast == NULL)
1022    {
1023       OldDepLast = &Ver->DependsList;
1024       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1025          OldDepLast = &D->NextDepends;
1026    } else if (oldMap != Map.Data())
1027       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1028
1029    Dep->NextDepends = *OldDepLast;
1030    *OldDepLast = Dependency;
1031    OldDepLast = &Dep->NextDepends;
1032    return true;
1033 }
1034                                                                         /*}}}*/
1035 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1036 // ---------------------------------------------------------------------
1037 /* This creates a Group and the Package to link this dependency to if
1038    needed and handles also the caching of the old endpoint */
1039 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
1040                                                const string &PackageName,
1041                                                const string &Arch,
1042                                                const string &Version,
1043                                                uint8_t const Op,
1044                                                uint8_t const Type)
1045 {
1046    pkgCache::GrpIterator Grp;
1047    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1048    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1049       return false;
1050
1051    // Locate the target package
1052    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
1053    // we don't create 'none' packages and their dependencies if we can avoid it …
1054    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
1055       return true;
1056    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1057    if (Pkg.end() == true) {
1058       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1059          return false;
1060    }
1061
1062    // Is it a file dependency?
1063    if (unlikely(PackageName[0] == '/'))
1064       FoundFileDeps = true;
1065
1066    /* Caching the old end point speeds up generation substantially */
1067    if (OldDepVer != Ver) {
1068       OldDepLast = NULL;
1069       OldDepVer = Ver;
1070    }
1071
1072    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1073 }
1074                                                                         /*}}}*/
1075 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1076 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1077                                                 const string &PkgName,
1078                                                 const string &PkgArch,
1079                                                 const string &Version,
1080                                                 uint8_t const Flags)
1081 {
1082    pkgCache &Cache = Owner->Cache;
1083
1084    // We do not add self referencing provides
1085    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1086         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1087       return true;
1088
1089    // Get a structure
1090    map_pointer_t const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
1091    if (unlikely(Provides == 0))
1092       return false;
1093    Cache.HeaderP->ProvidesCount++;
1094
1095    // Fill it in
1096    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1097    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
1098    Prv->Version = Ver.Index();
1099    Prv->Flags = Flags;
1100    Prv->NextPkgProv = Ver->ProvidesList;
1101    Ver->ProvidesList = Prv.Index();
1102    if (Version.empty() == false) {
1103       map_stringitem_t const idxProvideVersion = WriteString(Version);
1104       Prv->ProvideVersion = idxProvideVersion;
1105       if (unlikely(idxProvideVersion == 0))
1106          return false;
1107    }
1108
1109    // Locate the target package
1110    pkgCache::PkgIterator Pkg;
1111    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1112    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1113       return false;
1114
1115    // Link it to the package
1116    Prv->ParentPkg = Pkg.Index();
1117    Prv->NextProvides = Pkg->ProvidesList;
1118    Pkg->ProvidesList = Prv.Index();
1119
1120    return true;
1121 }
1122                                                                         /*}}}*/
1123 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1124       pkgCache::VerIterator const &Ver)
1125 {
1126    return Hash == Ver->Hash;
1127 }
1128                                                                         /*}}}*/
1129 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1130 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1131                                    unsigned long Flags)
1132 {
1133    if (File.empty() && Site.empty())
1134    {
1135       CurrentRlsFile = NULL;
1136       return true;
1137    }
1138
1139    // Get some space for the structure
1140    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1141    if (unlikely(idxFile == 0))
1142       return false;
1143    CurrentRlsFile = Cache.RlsFileP + idxFile;
1144
1145    // Fill it in
1146    map_stringitem_t const idxFileName = WriteStringInMap(File);
1147    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1148    if (unlikely(idxFileName == 0 || idxSite == 0))
1149       return false;
1150    CurrentRlsFile->FileName = idxFileName;
1151    CurrentRlsFile->Site = idxSite;
1152    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1153    CurrentRlsFile->Flags = Flags;
1154    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1155    RlsFileName = File;
1156    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1157    Cache.HeaderP->ReleaseFileCount++;
1158
1159    return true;
1160 }
1161                                                                         /*}}}*/
1162 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1163 // ---------------------------------------------------------------------
1164 /* This is used to select which file is to be associated with all newly
1165    added versions. The caller is responsible for setting the IMS fields. */
1166 bool pkgCacheGenerator::SelectFile(std::string const &File,
1167                                    pkgIndexFile const &Index,
1168                                    std::string const &Architecture,
1169                                    std::string const &Component,
1170                                    unsigned long const Flags)
1171 {
1172    // Get some space for the structure
1173    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1174    if (unlikely(idxFile == 0))
1175       return false;
1176    CurrentFile = Cache.PkgFileP + idxFile;
1177
1178    // Fill it in
1179    map_stringitem_t const idxFileName = WriteStringInMap(File);
1180    if (unlikely(idxFileName == 0))
1181       return false;
1182    CurrentFile->FileName = idxFileName;
1183    CurrentFile->NextFile = Cache.HeaderP->FileList;
1184    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1185    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1186    if (unlikely(idxIndexType == 0))
1187       return false;
1188    CurrentFile->IndexType = idxIndexType;
1189    if (Architecture.empty())
1190       CurrentFile->Architecture = 0;
1191    else
1192    {
1193       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1194       if (unlikely(arch == 0))
1195          return false;
1196       CurrentFile->Architecture = arch;
1197    }
1198    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1199    if (unlikely(component == 0))
1200       return false;
1201    CurrentFile->Component = component;
1202    CurrentFile->Flags = Flags;
1203    if (CurrentRlsFile != NULL)
1204       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1205    else
1206       CurrentFile->Release = 0;
1207    PkgFileName = File;
1208    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1209    Cache.HeaderP->PackageFileCount++;
1210
1211    if (Progress != 0)
1212       Progress->SubProgress(Index.Size());
1213    return true;
1214 }
1215                                                                         /*}}}*/
1216 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1217 // ---------------------------------------------------------------------
1218 /* This is used to create handles to strings. Given the same text it
1219    always returns the same number */
1220 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1221                                                  unsigned int Size)
1222 {
1223    std::string const key(S, Size);
1224
1225    std::map<std::string,map_stringitem_t> * strings;
1226    switch(type) {
1227       case MIXED: strings = &strMixed; break;
1228       case PKGNAME: strings = &strPkgNames; break;
1229       case VERSIONNUMBER: strings = &strVersions; break;
1230       case SECTION: strings = &strSections; break;
1231       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1232    }
1233
1234    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1235    if (item != strings->end())
1236       return item->second;
1237
1238    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1239    strings->insert(std::make_pair(key, idxString));
1240    return idxString;
1241 }
1242                                                                         /*}}}*/
1243 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1244 // ---------------------------------------------------------------------
1245 /* This just verifies that each file in the list of index files exists,
1246    has matching attributes with the cache and the cache does not have
1247    any extra files. */
1248 static bool CheckValidity(const string &CacheFile,
1249                           pkgSourceList &List,
1250                           FileIterator Start,
1251                           FileIterator End,
1252                           MMap **OutMap = 0)
1253 {
1254    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1255    // No file, certainly invalid
1256    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1257    {
1258       if (Debug == true)
1259          std::clog << "CacheFile doesn't exist" << std::endl;
1260       return false;
1261    }
1262
1263    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1264    {
1265       if (Debug == true)
1266          std::clog << "sources.list is newer than the cache" << std::endl;
1267       return false;
1268    }
1269
1270    // Map it
1271    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1272    SPtr<MMap> Map = new MMap(CacheF,0);
1273    pkgCache Cache(Map);
1274    if (_error->PendingError() == true || Map->Size() == 0)
1275    {
1276       if (Debug == true)
1277          std::clog << "Errors are pending or Map is empty()" << std::endl;
1278       _error->Discard();
1279       return false;
1280    }
1281
1282    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1283    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1284    std::vector<pkgIndexFile *> Files;
1285    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1286    {
1287       if (Debug == true)
1288          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1289       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1290       if (RlsFile.end() == true)
1291       {
1292          if (Debug == true)
1293             std::clog << "FindInCache returned end-Pointer" << std::endl;
1294          return false;
1295       }
1296
1297       RlsVisited[RlsFile->ID] = true;
1298       if (Debug == true)
1299          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1300
1301       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1302       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1303          if ((*j)->HasPackages())
1304             Files.push_back (*j);
1305    }
1306    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1307       if (RlsVisited[I] == false)
1308       {
1309          if (Debug == true)
1310             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1311          return false;
1312       }
1313
1314    for (; Start != End; ++Start)
1315       Files.push_back(*Start);
1316
1317    /* Now we check every index file, see if it is in the cache,
1318       verify the IMS data and check that it is on the disk too.. */
1319    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1320    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1321    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1322    {
1323       if (Debug == true)
1324          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1325       if ((*PkgFile)->Exists() == false)
1326       {
1327          if (Debug == true)
1328             std::clog << "file doesn't exist" << std::endl;
1329          continue;
1330       }
1331
1332       // FindInCache is also expected to do an IMS check.
1333       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1334       if (File.end() == true)
1335       {
1336          if (Debug == true)
1337             std::clog << "FindInCache returned end-Pointer" << std::endl;
1338          return false;
1339       }
1340
1341       Visited[File->ID] = true;
1342       if (Debug == true)
1343          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1344    }
1345
1346    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1347       if (Visited[I] == false)
1348       {
1349          if (Debug == true)
1350             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1351          return false;
1352       }
1353
1354    if (_error->PendingError() == true)
1355    {
1356       if (Debug == true)
1357       {
1358          std::clog << "Validity failed because of pending errors:" << std::endl;
1359          _error->DumpErrors();
1360       }
1361       _error->Discard();
1362       return false;
1363    }
1364
1365    if (OutMap != 0)
1366       *OutMap = Map.UnGuard();
1367    return true;
1368 }
1369                                                                         /*}}}*/
1370 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1371 // ---------------------------------------------------------------------
1372 /* Size is kind of an abstract notion that is only used for the progress
1373    meter */
1374 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1375 {
1376    map_filesize_t TotalSize = 0;
1377    if (List !=  NULL)
1378    {
1379       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1380       {
1381          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1382          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1383             if ((*j)->HasPackages() == true)
1384                TotalSize += (*j)->Size();
1385       }
1386    }
1387
1388    for (; Start < End; ++Start)
1389    {
1390       if ((*Start)->HasPackages() == false)
1391          continue;
1392       TotalSize += (*Start)->Size();
1393    }
1394    return TotalSize;
1395 }
1396                                                                         /*}}}*/
1397 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1398 // ---------------------------------------------------------------------
1399 /* */
1400 static bool BuildCache(pkgCacheGenerator &Gen,
1401                        OpProgress *Progress,
1402                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1403                        pkgSourceList const * const List,
1404                        FileIterator Start, FileIterator End)
1405 {
1406    std::vector<pkgIndexFile *> Files;
1407    bool const HasFileDeps = Gen.HasFileDeps();
1408
1409    if (List !=  NULL)
1410    {
1411       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1412       {
1413          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1414          {
1415             _error->Warning("Duplicate sources.list entry %s",
1416                   (*i)->Describe().c_str());
1417             continue;
1418          }
1419
1420          if ((*i)->Merge(Gen, Progress) == false)
1421             return false;
1422
1423          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1424          for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
1425          {
1426             if (HasFileDeps)
1427                Files.push_back(*I);
1428
1429             if ((*I)->HasPackages() == false)
1430                continue;
1431
1432             if ((*I)->Exists() == false)
1433                continue;
1434
1435             if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1436             {
1437                _error->Warning("Duplicate sources.list entry %s",
1438                      (*I)->Describe().c_str());
1439                continue;
1440             }
1441
1442             map_filesize_t Size = (*I)->Size();
1443             if (Progress != NULL)
1444                Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1445             CurrentSize += Size;
1446
1447             if ((*I)->Merge(Gen,Progress) == false)
1448                return false;
1449          }
1450       }
1451    }
1452
1453    Gen.SelectReleaseFile("", "");
1454    FileIterator I;
1455    for (I = Start; I != End; ++I)
1456    {
1457       if (HasFileDeps)
1458          Files.push_back(*I);
1459
1460       if ((*I)->HasPackages() == false)
1461          continue;
1462
1463       if ((*I)->Exists() == false)
1464          continue;
1465
1466       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1467       {
1468          _error->Warning("Duplicate sources.list entry %s",
1469                          (*I)->Describe().c_str());
1470          continue;
1471       }
1472
1473       map_filesize_t Size = (*I)->Size();
1474       if (Progress != NULL)
1475          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1476       CurrentSize += Size;
1477
1478       if ((*I)->Merge(Gen,Progress) == false)
1479          return false;
1480    }
1481
1482    if (HasFileDeps == true)
1483    {
1484       if (Progress != NULL)
1485          Progress->Done();
1486       TotalSize = ComputeSize(List, Start, End);
1487       CurrentSize = 0;
1488       for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
1489       {
1490          map_filesize_t Size = (*I)->Size();
1491          if (Progress != NULL)
1492             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1493          CurrentSize += Size;
1494          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1495             return false;
1496       }
1497    }
1498
1499    return true;
1500 }
1501                                                                         /*}}}*/
1502 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1503 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1504    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1505    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1506    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1507    Flags |= MMap::Moveable;
1508    if (_config->FindB("APT::Cache-Fallback", false) == true)
1509       Flags |= MMap::Fallback;
1510    if (CacheF != NULL)
1511       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1512    else
1513       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1514 }
1515                                                                         /*}}}*/
1516 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1517 // ---------------------------------------------------------------------
1518 /* This makes sure that the status cache (the cache that has all
1519    index files from the sources list and all local ones) is ready
1520    to be mmaped. If OutMap is not zero then a MMap object representing
1521    the cache will be stored there. This is pretty much mandetory if you
1522    are using AllowMem. AllowMem lets the function be run as non-root
1523    where it builds the cache 'fast' into a memory buffer. */
1524 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1525                         MMap **OutMap, bool AllowMem)
1526    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1527 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1528                         MMap **OutMap,bool AllowMem)
1529 {
1530    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1531
1532    std::vector<pkgIndexFile *> Files;
1533    /*
1534    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1535         i != List.end();
1536         ++i)
1537    {
1538       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1539       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1540            j != Indexes->end();
1541            ++j)
1542          Files.push_back (*j);
1543    }
1544 */
1545    if (_system->AddStatusFiles(Files) == false)
1546       return false;
1547
1548    // Decide if we can write to the files..
1549    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1550    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1551
1552    // ensure the cache directory exists
1553    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1554    {
1555       string dir = _config->FindDir("Dir::Cache");
1556       size_t const len = dir.size();
1557       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1558          dir = dir.substr(0, len - 5);
1559       if (CacheFile.empty() == false)
1560          CreateDirectory(dir, flNotFile(CacheFile));
1561       if (SrcCacheFile.empty() == false)
1562          CreateDirectory(dir, flNotFile(SrcCacheFile));
1563    }
1564
1565    // Decide if we can write to the cache
1566    bool Writeable = false;
1567    if (CacheFile.empty() == false)
1568       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1569    else
1570       if (SrcCacheFile.empty() == false)
1571          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1572    if (Debug == true)
1573       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1574
1575    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1576       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1577
1578    if (Progress != NULL)
1579       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1580
1581    // Cache is OK, Fin.
1582    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1583    {
1584       if (Progress != NULL)
1585          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1586       if (Debug == true)
1587          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1588       return true;
1589    }
1590    else if (Debug == true)
1591          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1592
1593    /* At this point we know we need to reconstruct the package cache,
1594       begin. */
1595    SPtr<FileFd> CacheF;
1596    SPtr<DynamicMMap> Map;
1597    if (Writeable == true && CacheFile.empty() == false)
1598    {
1599       _error->PushToStack();
1600       unlink(CacheFile.c_str());
1601       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1602       fchmod(CacheF->Fd(),0644);
1603       Map = CreateDynamicMMap(CacheF, MMap::Public);
1604       if (_error->PendingError() == true)
1605       {
1606          delete CacheF.UnGuard();
1607          delete Map.UnGuard();
1608          if (Debug == true)
1609             std::clog << "Open filebased MMap FAILED" << std::endl;
1610          Writeable = false;
1611          if (AllowMem == false)
1612          {
1613             _error->MergeWithStack();
1614             return false;
1615          }
1616          _error->RevertToStack();
1617       }
1618       else
1619       {
1620          _error->MergeWithStack();
1621          if (Debug == true)
1622             std::clog << "Open filebased MMap" << std::endl;
1623       }
1624    }
1625    if (Writeable == false || CacheFile.empty() == true)
1626    {
1627       // Just build it in memory..
1628       Map = CreateDynamicMMap(NULL);
1629       if (Debug == true)
1630          std::clog << "Open memory Map (not filebased)" << std::endl;
1631    }
1632
1633    // Lets try the source cache.
1634    map_filesize_t CurrentSize = 0;
1635    map_filesize_t TotalSize = 0;
1636    if (CheckValidity(SrcCacheFile, List, Files.end(),
1637                      Files.end()) == true)
1638    {
1639       if (Debug == true)
1640          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1641       // Preload the map with the source cache
1642       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1643       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1644       if ((alloc == 0 && _error->PendingError())
1645                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1646                                 SCacheF.Size()) == false)
1647          return false;
1648
1649       TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1650
1651       // Build the status cache
1652       pkgCacheGenerator Gen(Map.Get(),Progress);
1653       if (_error->PendingError() == true)
1654          return false;
1655       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1656                      Files.begin(),Files.end()) == false)
1657          return false;
1658    }
1659    else
1660    {
1661       if (Debug == true)
1662          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1663       TotalSize = ComputeSize(&List, Files.begin(),Files.end());
1664
1665       // Build the source cache
1666       pkgCacheGenerator Gen(Map.Get(),Progress);
1667       if (_error->PendingError() == true)
1668          return false;
1669       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
1670                      Files.end(),Files.end()) == false)
1671          return false;
1672
1673       // Write it back
1674       if (Writeable == true && SrcCacheFile.empty() == false)
1675       {
1676          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1677          if (_error->PendingError() == true)
1678             return false;
1679
1680          fchmod(SCacheF.Fd(),0644);
1681
1682          // Write out the main data
1683          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1684             return _error->Error(_("IO Error saving source cache"));
1685          SCacheF.Sync();
1686
1687          // Write out the proper header
1688          Gen.GetCache().HeaderP->Dirty = false;
1689          if (SCacheF.Seek(0) == false ||
1690              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1691             return _error->Error(_("IO Error saving source cache"));
1692          Gen.GetCache().HeaderP->Dirty = true;
1693          SCacheF.Sync();
1694       }
1695
1696       // Build the status cache
1697       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1698                      Files.begin(), Files.end()) == false)
1699          return false;
1700    }
1701    if (Debug == true)
1702       std::clog << "Caches are ready for shipping" << std::endl;
1703
1704    if (_error->PendingError() == true)
1705       return false;
1706    if (OutMap != 0)
1707    {
1708       if (CacheF != 0)
1709       {
1710          delete Map.UnGuard();
1711          *OutMap = new MMap(*CacheF,0);
1712       }
1713       else
1714       {
1715          *OutMap = Map.UnGuard();
1716       }
1717    }
1718
1719    return true;
1720 }
1721                                                                         /*}}}*/
1722 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1723 // ---------------------------------------------------------------------
1724 /* */
1725 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1726    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1727 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1728 {
1729    std::vector<pkgIndexFile *> Files;
1730    if (_system->AddStatusFiles(Files) == false)
1731       return false;
1732
1733    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1734    map_filesize_t CurrentSize = 0;
1735    map_filesize_t TotalSize = 0;
1736
1737    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1738
1739    // Build the status cache
1740    if (Progress != NULL)
1741       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1742    pkgCacheGenerator Gen(Map.Get(),Progress);
1743    if (_error->PendingError() == true)
1744       return false;
1745    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1746                   Files.begin(), Files.end()) == false)
1747       return false;
1748
1749    if (_error->PendingError() == true)
1750       return false;
1751    *OutMap = Map.UnGuard();
1752
1753    return true;
1754 }
1755                                                                         /*}}}*/
1756 // IsDuplicateDescription                                               /*{{{*/
1757 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1758                             MD5SumValue const &CurMd5, std::string const &CurLang)
1759 {
1760    // Descriptions in the same link-list have all the same md5
1761    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1762       return false;
1763    for (; Desc.end() == false; ++Desc)
1764       if (Desc.LanguageCode() == CurLang)
1765          return true;
1766    return false;
1767 }
1768                                                                         /*}}}*/
1769 // CacheGenerator::FinishCache                                          /*{{{*/
1770 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1771 {
1772    return true;
1773 }
1774                                                                         /*}}}*/
1775
1776 pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
1777 pkgCacheGenerator::ListParser::~ListParser() {}