apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                      CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0), d(NULL)
  58 {
  59    if (_error->PendingError() == true)
  60       return;
  61
  62    if (Map.Size() == 0)
  63    {
  64       // Setup the map interface..
  65       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  66       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  67          return;
  68
  69       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  70
  71       // Starting header
  72       *Cache.HeaderP = pkgCache::Header();
  73
  74       // make room for the hashtables for packages and groups
  75       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  76          return;
  77
  78       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  79       if (unlikely(idxVerSysName == 0))
  80          return;
  81       Cache.HeaderP->VerSysName = idxVerSysName;
  82       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  83       if (unlikely(idxArchitecture == 0))
  84          return;
  85       Cache.HeaderP->Architecture = idxArchitecture;
  86
  87       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  88       if (archs.size() > 1)
  89       {
  90          std::vector<std::string>::const_iterator a = archs.begin();
  91          std::string list = *a;
  92          for (++a; a != archs.end(); ++a)
  93             list.append(",").append(*a);
  94          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  95          if (unlikely(idxArchitectures == 0))
  96             return;
  97          Cache.HeaderP->SetArchitectures(idxArchitectures);
  98       }
  99       else
 100          Cache.HeaderP->SetArchitectures(idxArchitecture);
 101
 102       Cache.ReMap();
 103    }
 104    else
 105    {
 106       // Map directly from the existing file
 107       Cache.ReMap();
 108       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 109       if (Cache.VS != _system->VS)
 110       {
 111          _error->Error(_("Cache has an incompatible versioning system"));
 112          return;
 113       }
 114    }
 115
 116    Cache.HeaderP->Dirty = true;
 117    Map.Sync(0,sizeof(pkgCache::Header));
 118 }
 119                                                                         /*}}}*/
 120 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 121 // ---------------------------------------------------------------------
 122 /* We sync the data then unset the dirty flag in two steps so as to
 123    advoid a problem during a crash */
 124 pkgCacheGenerator::~pkgCacheGenerator()
 125 {
 126    if (_error->PendingError() == true)
 127       return;
 128    if (Map.Sync() == false)
 129       return;
 130
 131    Cache.HeaderP->Dirty = false;
 132    Cache.HeaderP->CacheFileSize = Map.Size();
 133    Map.Sync(0,sizeof(pkgCache::Header));
 134 }
 135                                                                         /*}}}*/
 136 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 137    if (oldMap == newMap)
 138       return;
 139
 140    if (_config->FindB("Debug::pkgCacheGen", false))
 141       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 142
 143    Cache.ReMap(false);
 144
 145    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 146    CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 147
 148    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 149         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 150       (*i)->ReMap(oldMap, newMap);
 151    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 152         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 153       (*i)->ReMap(oldMap, newMap);
 154    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 155         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 156       (*i)->ReMap(oldMap, newMap);
 157    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 158         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 159       (*i)->ReMap(oldMap, newMap);
 160    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 161         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 162       (*i)->ReMap(oldMap, newMap);
 163    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 164         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 165       (*i)->ReMap(oldMap, newMap);
 166    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 167         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 168       (*i)->ReMap(oldMap, newMap);
 169    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
 170         i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
 171       (*i)->ReMap(oldMap, newMap);
 172 }                                                                       /*}}}*/
 173 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 174 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 175                                         const unsigned long &Len) {
 176    void const * const oldMap = Map.Data();
 177    map_stringitem_t const index = Map.WriteString(String, Len);
 178    if (index != 0)
 179       ReMap(oldMap, Map.Data());
 180    return index;
 181 }
 182                                                                         /*}}}*/
 183 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 184 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 185    void const * const oldMap = Map.Data();
 186    map_stringitem_t const index = Map.WriteString(String);
 187    if (index != 0)
 188       ReMap(oldMap, Map.Data());
 189    return index;
 190 }
 191                                                                         /*}}}*/
 192 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 193    void const * const oldMap = Map.Data();
 194    map_pointer_t const index = Map.Allocate(size);
 195    if (index != 0)
 196       ReMap(oldMap, Map.Data());
 197    return index;
 198 }
 199                                                                         /*}}}*/
 200 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This provides the generation of the entries in the cache. Each loop
 203    goes through a single package record from the underlying parse engine. */
 204 bool pkgCacheGenerator::MergeList(ListParser &List,
 205                                   pkgCache::VerIterator *OutVer)
 206 {
 207    List.Owner = this;
 208
 209    unsigned int Counter = 0;
 210    while (List.Step() == true)
 211    {
 212       string const PackageName = List.Package();
 213       if (PackageName.empty() == true)
 214          return false;
 215
 216       Counter++;
 217       if (Counter % 100 == 0 && Progress != 0)
 218          Progress->Progress(List.Offset());
 219
 220       string Arch = List.Architecture();
 221       string const Version = List.Version();
 222       if (Version.empty() == true && Arch.empty() == true)
 223       {
 224          // package descriptions
 225          if (MergeListGroup(List, PackageName) == false)
 226             return false;
 227          continue;
 228       }
 229
 230       if (Arch.empty() == true)
 231       {
 232          // use the pseudo arch 'none' for arch-less packages
 233          Arch = "none";
 234          /* We might built a SingleArchCache here, which we don't want to blow up
 235             just for these :none packages to a proper MultiArchCache, so just ensure
 236             that we have always a native package structure first for SingleArch */
 237          pkgCache::PkgIterator NP;
 238          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 239          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 240          // TRANSLATOR: The first placeholder is a package name,
 241          // the other two should be copied verbatim as they include debug info
 242          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 243                               PackageName.c_str(), "NewPackage", 0);
 244       }
 245
 246       // Get a pointer to the package structure
 247       pkgCache::PkgIterator Pkg;
 248       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 249       if (NewPackage(Pkg, PackageName, Arch) == false)
 250          // TRANSLATOR: The first placeholder is a package name,
 251          // the other two should be copied verbatim as they include debug info
 252          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 253                               PackageName.c_str(), "NewPackage", 1);
 254
 255
 256       if (Version.empty() == true)
 257       {
 258          if (MergeListPackage(List, Pkg) == false)
 259             return false;
 260       }
 261       else
 262       {
 263          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 264             return false;
 265       }
 266
 267       if (OutVer != 0)
 268       {
 269          FoundFileDeps |= List.HasFileDeps();
 270          return true;
 271       }
 272    }
 273
 274    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 275       return _error->Error(_("Wow, you exceeded the number of package "
 276                              "names this APT is capable of."));
 277    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 278       return _error->Error(_("Wow, you exceeded the number of versions "
 279                              "this APT is capable of."));
 280    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 281       return _error->Error(_("Wow, you exceeded the number of descriptions "
 282                              "this APT is capable of."));
 283    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 284       return _error->Error(_("Wow, you exceeded the number of dependencies "
 285                              "this APT is capable of."));
 286
 287    FoundFileDeps |= List.HasFileDeps();
 288    return true;
 289 }
 290 // CacheGenerator::MergeListGroup                                       /*{{{*/
 291 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 292 {
 293    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 294    // a group has no data on it's own, only packages have it but these
 295    // stanzas like this come from Translation- files to add descriptions,
 296    // but without a version we don't need a description for it…
 297    if (Grp.end() == true)
 298       return true;
 299    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 300
 301    pkgCache::PkgIterator Pkg;
 302    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 303    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 304       if (MergeListPackage(List, Pkg) == false)
 305          return false;
 306
 307    return true;
 308 }
 309                                                                         /*}}}*/
 310 // CacheGenerator::MergeListPackage                                     /*{{{*/
 311 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 312 {
 313    // we first process the package, then the descriptions
 314    // (for deb this package processing is in fact a no-op)
 315    pkgCache::VerIterator Ver(Cache);
 316    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 317    if (List.UsePackage(Pkg, Ver) == false)
 318       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 319                            Pkg.Name(), "UsePackage", 1);
 320
 321    // Find the right version to write the description
 322    MD5SumValue CurMd5 = List.Description_md5();
 323    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 324       return true;
 325    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 326    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 327    {
 328       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 329
 330       // a version can only have one md5 describing it
 331       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 332          continue;
 333
 334       map_stringitem_t md5idx = VerDesc->md5sum;
 335       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 336       {
 337          // don't add a new description if we have one for the given
 338          // md5 && language
 339          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 340             continue;
 341
 342          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 343       }
 344
 345       // we can stop here as all "same" versions will share the description
 346       break;
 347    }
 348
 349    return true;
 350 }
 351                                                                         /*}}}*/
 352 // CacheGenerator::MergeListVersion                                     /*{{{*/
 353 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 354                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 355 {
 356    pkgCache::VerIterator Ver = Pkg.VersionList();
 357    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 358    map_pointer_t *LastVer = &Pkg->VersionList;
 359    void const * oldMap = Map.Data();
 360
 361    unsigned short const Hash = List.VersionHash();
 362    if (Ver.end() == false)
 363    {
 364       /* We know the list is sorted so we use that fact in the search.
 365          Insertion of new versions is done with correct sorting */
 366       int Res = 1;
 367       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 368       {
 369          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 370          // Version is higher as current version - insert here
 371          if (Res > 0)
 372             break;
 373          // Versionstrings are equal - is hash also equal?
 374          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 375             break;
 376          // proceed with the next till we have either the right
 377          // or we found another version (which will be lower)
 378       }
 379
 380       /* We already have a version for this item, record that we saw it */
 381       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 382       {
 383          if (List.UsePackage(Pkg,Ver) == false)
 384             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 385                                  Pkg.Name(), "UsePackage", 2);
 386
 387          if (NewFileVer(Ver,List) == false)
 388             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 389                                  Pkg.Name(), "NewFileVer", 1);
 390
 391          // Read only a single record and return
 392          if (OutVer != 0)
 393          {
 394             *OutVer = Ver;
 395             return true;
 396          }
 397
 398          return true;
 399       }
 400    }
 401
 402    // Add a new version
 403    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 404    if (verindex == 0 && _error->PendingError())
 405       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 406                            Pkg.Name(), "NewVersion", 1);
 407
 408    if (oldMap != Map.Data())
 409          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 410    *LastVer = verindex;
 411
 412    if (unlikely(List.NewVersion(Ver) == false))
 413       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 414                            Pkg.Name(), "NewVersion", 2);
 415
 416    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 417       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 418                            Pkg.Name(), "UsePackage", 3);
 419
 420    if (unlikely(NewFileVer(Ver,List) == false))
 421       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 422                            Pkg.Name(), "NewFileVer", 2);
 423
 424    pkgCache::GrpIterator Grp = Pkg.Group();
 425    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 426
 427    /* If it is the first version of this package we need to add implicit
 428       Multi-Arch dependencies to all other package versions in the group now -
 429       otherwise we just add them for this new version */
 430    if (Pkg.VersionList()->NextVer == 0)
 431    {
 432       pkgCache::PkgIterator P = Grp.PackageList();
 433       Dynamic<pkgCache::PkgIterator> DynP(P);
 434       for (; P.end() != true; P = Grp.NextPkg(P))
 435       {
 436          if (P->ID == Pkg->ID)
 437             continue;
 438          pkgCache::VerIterator V = P.VersionList();
 439          Dynamic<pkgCache::VerIterator> DynV(V);
 440          for (; V.end() != true; ++V)
 441             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 442                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 443                                     Pkg.Name(), "AddImplicitDepends", 1);
 444       }
 445       /* :none packages are packages without an architecture. They are forbidden by
 446          debian-policy, so usually they will only be in (old) dpkg status files -
 447          and dpkg will complain about them - and are pretty rare. We therefore do
 448          usually not create conflicts while the parent is created, but only if a :none
 449          package (= the target) appears. This creates incorrect dependencies on :none
 450          for architecture-specific dependencies on the package we copy from, but we
 451          will ignore this bug as architecture-specific dependencies are only allowed
 452          in jessie and until then the :none packages should be extinct (hopefully).
 453          In other words: This should work long enough to allow graceful removal of
 454          these packages, it is not supposed to allow users to keep using them … */
 455       if (strcmp(Pkg.Arch(), "none") == 0)
 456       {
 457          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 458          if (M.end() == false && Pkg != M)
 459          {
 460             pkgCache::DepIterator D = M.RevDependsList();
 461             Dynamic<pkgCache::DepIterator> DynD(D);
 462             for (; D.end() == false; ++D)
 463             {
 464                if ((D->Type != pkgCache::Dep::Conflicts &&
 465                     D->Type != pkgCache::Dep::DpkgBreaks &&
 466                     D->Type != pkgCache::Dep::Replaces) ||
 467                    D.ParentPkg().Group() == Grp)
 468                   continue;
 469
 470                map_pointer_t *OldDepLast = NULL;
 471                pkgCache::VerIterator ConVersion = D.ParentVer();
 472                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 473                // duplicate the Conflicts/Breaks/Replaces for :none arch
 474                NewDepends(Pkg, ConVersion, D->Version,
 475                      D->CompareOp, D->Type, OldDepLast);
 476             }
 477          }
 478       }
 479    }
 480    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 481       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 482                            Pkg.Name(), "AddImplicitDepends", 2);
 483
 484    // Read only a single record and return
 485    if (OutVer != 0)
 486    {
 487       *OutVer = Ver;
 488       return true;
 489    }
 490
 491    /* Record the Description(s) based on their master md5sum */
 492    MD5SumValue CurMd5 = List.Description_md5();
 493    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 494       return true;
 495
 496    /* Before we add a new description we first search in the group for
 497       a version with a description of the same MD5 - if so we reuse this
 498       description group instead of creating our own for this version */
 499    for (pkgCache::PkgIterator P = Grp.PackageList();
 500         P.end() == false; P = Grp.NextPkg(P))
 501    {
 502       for (pkgCache::VerIterator V = P.VersionList();
 503            V.end() == false; ++V)
 504       {
 505          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 506             continue;
 507          Ver->DescriptionList = V->DescriptionList;
 508       }
 509    }
 510
 511    // We haven't found reusable descriptions, so add the first description(s)
 512    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 513    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 514    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 515       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 516          return false;
 517    return true;
 518 }
 519                                                                         /*}}}*/
 520 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 521 {
 522    pkgCache::DescIterator Desc;
 523    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 524
 525    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 526    if (unlikely(descindex == 0 && _error->PendingError()))
 527       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 528             Ver.ParentPkg().Name(), "NewDescription", 1);
 529
 530    md5idx = Desc->md5sum;
 531    Desc->ParentPkg = Ver.ParentPkg().Index();
 532
 533    // we add at the end, so that the start is constant as we need
 534    // that to be able to efficiently share these lists
 535    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 536    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 537    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 538    *LastNextDesc = descindex;
 539
 540    if (NewFileDesc(Desc,List) == false)
 541       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 542             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 543
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547                                                                         /*}}}*/
 548 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* If we found any file depends while parsing the main list we need to
 551    resolve them. Since it is undesired to load the entire list of files
 552    into the cache as virtual packages we do a two stage effort. MergeList
 553    identifies the file depends and this creates Provdies for them by
 554    re-parsing all the indexs. */
 555 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 556 {
 557    List.Owner = this;
 558
 559    unsigned int Counter = 0;
 560    while (List.Step() == true)
 561    {
 562       string PackageName = List.Package();
 563       if (PackageName.empty() == true)
 564          return false;
 565       string Version = List.Version();
 566       if (Version.empty() == true)
 567          continue;
 568
 569       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 570       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 571       if (Pkg.end() == true)
 572          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 573                                 PackageName.c_str(), "FindPkg", 1);
 574       Counter++;
 575       if (Counter % 100 == 0 && Progress != 0)
 576          Progress->Progress(List.Offset());
 577
 578       unsigned short Hash = List.VersionHash();
 579       pkgCache::VerIterator Ver = Pkg.VersionList();
 580       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 581       for (; Ver.end() == false; ++Ver)
 582       {
 583          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 584          {
 585             if (List.CollectFileProvides(Cache,Ver) == false)
 586                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 587                                     PackageName.c_str(), "CollectFileProvides", 1);
 588             break;
 589          }
 590       }
 591
 592       if (Ver.end() == true)
 593          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 594    }
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This creates a new group structure and adds it to the hash table */
 602 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 603 {
 604    Grp = Cache.FindGrp(Name);
 605    if (Grp.end() == false)
 606       return true;
 607
 608    // Get a structure
 609    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 610    if (unlikely(Group == 0))
 611       return false;
 612
 613    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 614    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 615    if (unlikely(idxName == 0))
 616       return false;
 617    Grp->Name = idxName;
 618
 619    // Insert it into the hash table
 620    unsigned long const Hash = Cache.Hash(Name);
 621    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 622    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 623       insertAt = &(Cache.GrpP + *insertAt)->Next;
 624    Grp->Next = *insertAt;
 625    *insertAt = Group;
 626
 627    Grp->ID = Cache.HeaderP->GroupCount++;
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* This creates a new package structure and adds it to the hash table */
 634 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 635                                         const string &Arch) {
 636    pkgCache::GrpIterator Grp;
 637    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 638    if (unlikely(NewGroup(Grp, Name) == false))
 639       return false;
 640
 641    Pkg = Grp.FindPkg(Arch);
 642       if (Pkg.end() == false)
 643          return true;
 644
 645    // Get a structure
 646    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 647    if (unlikely(Package == 0))
 648       return false;
 649    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 650
 651    // Set the name, arch and the ID
 652    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 653    Pkg->Group = Grp.Index();
 654    // all is mapped to the native architecture
 655    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 656    if (unlikely(idxArch == 0))
 657       return false;
 658    Pkg->Arch = idxArch;
 659    Pkg->ID = Cache.HeaderP->PackageCount++;
 660
 661    // Insert the package into our package list
 662    if (Grp->FirstPackage == 0) // the group is new
 663    {
 664       Grp->FirstPackage = Package;
 665       // Insert it into the hash table
 666       map_id_t const Hash = Cache.Hash(Name);
 667       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 668       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 669          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 670       Pkg->NextPackage = *insertAt;
 671       *insertAt = Package;
 672    }
 673    else // Group the Packages together
 674    {
 675       // but first get implicit provides done
 676       if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
 677       {
 678          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
 679          if (M.end() == false)
 680             for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
 681             {
 682                if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
 683                   continue;
 684                pkgCache::VerIterator Ver = Prv.OwnerVer();
 685                if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
 686                    ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
 687                         (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
 688                   if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
 689                      return false;
 690             }
 691
 692          for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
 693             for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
 694                if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
 695                   if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
 696                      return false;
 697       }
 698       // and negative dependencies, don't forget negative dependencies
 699       {
 700          pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
 701          if (M.end() == false)
 702             for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
 703             {
 704                if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
 705                   continue;
 706                if (Dep->Type != pkgCache::Dep::DpkgBreaks && Dep->Type != pkgCache::Dep::Conflicts &&
 707                      Dep->Type != pkgCache::Dep::Replaces)
 708                   continue;
 709                pkgCache::VerIterator Ver = Dep.ParentVer();
 710                map_pointer_t * unused = NULL;
 711                if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
 712                   return false;
 713             }
 714       }
 715
 716       // this package is the new last package
 717       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 718       Pkg->NextPackage = LastPkg->NextPackage;
 719       LastPkg->NextPackage = Package;
 720    }
 721    Grp->LastPackage = Package;
 722    return true;
 723 }
 724                                                                         /*}}}*/
 725 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 726 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 727                                            pkgCache::PkgIterator &P,
 728                                            pkgCache::VerIterator &V)
 729 {
 730    // copy P.Arch() into a string here as a cache remap
 731    // in NewDepends() later may alter the pointer location
 732    string Arch = P.Arch() == NULL ? "" : P.Arch();
 733    map_pointer_t *OldDepLast = NULL;
 734    /* MultiArch handling introduces a lot of implicit Dependencies:
 735       - MultiArch: same → Co-Installable if they have the same version
 736       - All others conflict with all other group members */
 737    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 738    pkgCache::PkgIterator D = G.PackageList();
 739    Dynamic<pkgCache::PkgIterator> DynD(D);
 740    map_stringitem_t const VerStrIdx = V->VerStr;
 741    for (; D.end() != true; D = G.NextPkg(D))
 742    {
 743       if (Arch == D.Arch() || D->VersionList == 0)
 744          continue;
 745       /* We allow only one installed arch at the time
 746          per group, therefore each group member conflicts
 747          with all other group members */
 748       if (coInstall == true)
 749       {
 750          // Replaces: ${self}:other ( << ${binary:Version})
 751          NewDepends(D, V, VerStrIdx,
 752                     pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 753                     OldDepLast);
 754          // Breaks: ${self}:other (!= ${binary:Version})
 755          NewDepends(D, V, VerStrIdx,
 756                     pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 757                     OldDepLast);
 758       } else {
 759          // Conflicts: ${self}:other
 760          NewDepends(D, V, 0,
 761                     pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 762                     OldDepLast);
 763       }
 764    }
 765    return true;
 766 }
 767 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 768                                            pkgCache::PkgIterator &D)
 769 {
 770    /* MultiArch handling introduces a lot of implicit Dependencies:
 771       - MultiArch: same → Co-Installable if they have the same version
 772       - All others conflict with all other group members */
 773    map_pointer_t *OldDepLast = NULL;
 774    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 775    if (coInstall == true)
 776    {
 777       map_stringitem_t const VerStrIdx = V->VerStr;
 778       // Replaces: ${self}:other ( << ${binary:Version})
 779       NewDepends(D, V, VerStrIdx,
 780                  pkgCache::Dep::Less | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Replaces,
 781                  OldDepLast);
 782       // Breaks: ${self}:other (!= ${binary:Version})
 783       NewDepends(D, V, VerStrIdx,
 784                  pkgCache::Dep::NotEquals | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::DpkgBreaks,
 785                  OldDepLast);
 786    } else {
 787       // Conflicts: ${self}:other
 788       NewDepends(D, V, 0,
 789                  pkgCache::Dep::NoOp | pkgCache::Dep::MultiArchImplicit, pkgCache::Dep::Conflicts,
 790                  OldDepLast);
 791    }
 792    return true;
 793 }
 794
 795                                                                         /*}}}*/
 796 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 797 // ---------------------------------------------------------------------
 798 /* */
 799 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 800                                    ListParser &List)
 801 {
 802    if (CurrentFile == 0)
 803       return true;
 804
 805    // Get a structure
 806    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 807    if (VerFile == 0)
 808       return false;
 809
 810    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 811    VF->File = CurrentFile - Cache.PkgFileP;
 812
 813    // Link it to the end of the list
 814    map_pointer_t *Last = &Ver->FileList;
 815    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 816       Last = &V->NextFile;
 817    VF->NextFile = *Last;
 818    *Last = VF.Index();
 819
 820    VF->Offset = List.Offset();
 821    VF->Size = List.Size();
 822    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 823       Cache.HeaderP->MaxVerFileSize = VF->Size;
 824    Cache.HeaderP->VerFileCount++;
 825
 826    return true;
 827 }
 828                                                                         /*}}}*/
 829 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 830 // ---------------------------------------------------------------------
 831 /* This puts a version structure in the linked list */
 832 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 833                                             const string &VerStr,
 834                                             map_pointer_t const ParentPkg,
 835                                             unsigned short const Hash,
 836                                             map_pointer_t const Next)
 837 {
 838    // Get a structure
 839    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 840    if (Version == 0)
 841       return 0;
 842
 843    // Fill it in
 844    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 845    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 846    Ver->NextVer = Next;
 847    Ver->ParentPkg = ParentPkg;
 848    Ver->Hash = Hash;
 849    Ver->ID = Cache.HeaderP->VersionCount++;
 850
 851    // try to find the version string in the group for reuse
 852    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 853    pkgCache::GrpIterator Grp = Pkg.Group();
 854    if (Pkg.end() == false && Grp.end() == false)
 855    {
 856       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 857       {
 858          if (Pkg == P)
 859             continue;
 860          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 861          {
 862             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 863             if (cmp == 0)
 864             {
 865                Ver->VerStr = V->VerStr;
 866                return Version;
 867             }
 868             else if (cmp < 0)
 869                break;
 870          }
 871       }
 872    }
 873    // haven't found the version string, so create
 874    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 875    if (unlikely(idxVerStr == 0))
 876       return 0;
 877    Ver->VerStr = idxVerStr;
 878    return Version;
 879 }
 880                                                                         /*}}}*/
 881 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 882 // ---------------------------------------------------------------------
 883 /* */
 884 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 885                                    ListParser &List)
 886 {
 887    if (CurrentFile == 0)
 888       return true;
 889
 890    // Get a structure
 891    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 892    if (DescFile == 0)
 893       return false;
 894
 895    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 896    DF->File = CurrentFile - Cache.PkgFileP;
 897
 898    // Link it to the end of the list
 899    map_pointer_t *Last = &Desc->FileList;
 900    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 901       Last = &D->NextFile;
 902
 903    DF->NextFile = *Last;
 904    *Last = DF.Index();
 905
 906    DF->Offset = List.Offset();
 907    DF->Size = List.Size();
 908    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 909       Cache.HeaderP->MaxDescFileSize = DF->Size;
 910    Cache.HeaderP->DescFileCount++;
 911
 912    return true;
 913 }
 914                                                                         /*}}}*/
 915 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* This puts a description structure in the linked list */
 918 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 919                                             const string &Lang,
 920                                             const MD5SumValue &md5sum,
 921                                             map_stringitem_t const idxmd5str)
 922 {
 923    // Get a structure
 924    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 925    if (Description == 0)
 926       return 0;
 927
 928    // Fill it in
 929    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 930    Desc->ID = Cache.HeaderP->DescriptionCount++;
 931    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 932    if (unlikely(idxlanguage_code == 0))
 933       return 0;
 934    Desc->language_code = idxlanguage_code;
 935
 936    if (idxmd5str != 0)
 937       Desc->md5sum = idxmd5str;
 938    else
 939    {
 940       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 941       if (unlikely(idxmd5sum == 0))
 942          return 0;
 943       Desc->md5sum = idxmd5sum;
 944    }
 945
 946    return Description;
 947 }
 948                                                                         /*}}}*/
 949 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 950 // ---------------------------------------------------------------------
 951 /* This creates a dependency element in the tree. It is linked to the
 952    version and to the package that it is pointing to. */
 953 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 954                                    pkgCache::VerIterator &Ver,
 955                                    map_pointer_t const Version,
 956                                    uint8_t const Op,
 957                                    uint8_t const Type,
 958                                    map_pointer_t* &OldDepLast)
 959 {
 960    void const * const oldMap = Map.Data();
 961    // Get a structure
 962    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 963    if (unlikely(Dependency == 0))
 964       return false;
 965
 966    bool isDuplicate = false;
 967    map_pointer_t DependencyData = 0;
 968    map_pointer_t PreviousData = 0;
 969    if (Pkg->RevDepends != 0)
 970    {
 971       pkgCache::Dependency const * const L = Cache.DepP + Pkg->RevDepends;
 972       DependencyData = L->DependencyData;
 973       do {
 974          pkgCache::DependencyData const * const D = Cache.DepDataP + DependencyData;
 975          if (Version > D->Version)
 976             break;
 977          if (D->Version == Version && D->Type == Type && D->CompareOp == Op)
 978          {
 979             isDuplicate = true;
 980             break;
 981          }
 982          PreviousData = DependencyData;
 983          DependencyData = D->NextData;
 984       } while (DependencyData != 0);
 985    }
 986
 987    if (isDuplicate == false)
 988    {
 989       DependencyData = AllocateInMap(sizeof(pkgCache::DependencyData));
 990       if (unlikely(DependencyData == 0))
 991         return false;
 992    }
 993
 994    pkgCache::Dependency * Link = Cache.DepP + Dependency;
 995    Link->ParentVer = Ver.Index();
 996    Link->DependencyData = DependencyData;
 997    Link->ID = Cache.HeaderP->DependsCount++;
 998
 999    pkgCache::DepIterator Dep(Cache, Link);
1000    if (isDuplicate == false)
1001    {
1002       Dep->Type = Type;
1003       Dep->CompareOp = Op;
1004       Dep->Version = Version;
1005       Dep->Package = Pkg.Index();
1006       ++Cache.HeaderP->DependsDataCount;
1007       if (PreviousData != 0)
1008       {
1009          pkgCache::DependencyData * const D = Cache.DepDataP + PreviousData;
1010          Dep->NextData = D->NextData;
1011          D->NextData = DependencyData;
1012       }
1013       else if (Pkg->RevDepends != 0)
1014       {
1015          pkgCache::Dependency const * const D = Cache.DepP + Pkg->RevDepends;
1016          Dep->NextData = D->DependencyData;
1017       }
1018    }
1019
1020    if (isDuplicate == true || PreviousData != 0)
1021    {
1022       pkgCache::Dependency * const L = Cache.DepP + Pkg->RevDepends;
1023       Link->NextRevDepends = L->NextRevDepends;
1024       L->NextRevDepends = Dependency;
1025    }
1026    else
1027    {
1028       Link->NextRevDepends = Pkg->RevDepends;
1029       Pkg->RevDepends = Dependency;
1030    }
1031
1032
1033    // Do we know where to link the Dependency to?
1034    if (OldDepLast == NULL)
1035    {
1036       OldDepLast = &Ver->DependsList;
1037       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
1038          OldDepLast = &D->NextDepends;
1039    } else if (oldMap != Map.Data())
1040       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1041
1042    Dep->NextDepends = *OldDepLast;
1043    *OldDepLast = Dependency;
1044    OldDepLast = &Dep->NextDepends;
1045    return true;
1046 }
1047                                                                         /*}}}*/
1048 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
1049 // ---------------------------------------------------------------------
1050 /* This creates a Group and the Package to link this dependency to if
1051    needed and handles also the caching of the old endpoint */
1052 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
1053                                                const string &PackageName,
1054                                                const string &Arch,
1055                                                const string &Version,
1056                                                uint8_t const Op,
1057                                                uint8_t const Type)
1058 {
1059    pkgCache::GrpIterator Grp;
1060    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1061    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1062       return false;
1063
1064    // Is it a file dependency?
1065    if (unlikely(PackageName[0] == '/'))
1066       FoundFileDeps = true;
1067
1068    map_stringitem_t idxVersion = 0;
1069    if (Version.empty() == false)
1070    {
1071       int const CmpOp = Op & 0x0F;
1072       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
1073       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
1074          idxVersion = Ver->VerStr;
1075
1076       if (idxVersion == 0)
1077       {
1078          idxVersion = StoreString(VERSIONNUMBER, Version);
1079          if (unlikely(idxVersion == 0))
1080             return false;
1081       }
1082    }
1083
1084    bool const isNegative = (Type == pkgCache::Dep::DpkgBreaks ||
1085          Type == pkgCache::Dep::Conflicts ||
1086          Type == pkgCache::Dep::Replaces);
1087
1088    pkgCache::PkgIterator Pkg;
1089    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1090    if (isNegative == false || (Op & pkgCache::Dep::ArchSpecific) == pkgCache::Dep::ArchSpecific || Grp->FirstPackage == 0)
1091    {
1092       // Locate the target package
1093       Pkg = Grp.FindPkg(Arch);
1094       if (Pkg.end() == true) {
1095          if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1096             return false;
1097       }
1098
1099       /* Caching the old end point speeds up generation substantially */
1100       if (OldDepVer != Ver) {
1101          OldDepLast = NULL;
1102          OldDepVer = Ver;
1103       }
1104
1105       return Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast);
1106    }
1107    else
1108    {
1109       /* Caching the old end point speeds up generation substantially */
1110       if (OldDepVer != Ver) {
1111          OldDepLast = NULL;
1112          OldDepVer = Ver;
1113       }
1114
1115       for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1116       {
1117          if (Owner->NewDepends(Pkg, Ver, idxVersion, Op, Type, OldDepLast) == false)
1118                return false;
1119       }
1120    }
1121    return true;
1122 }
1123                                                                         /*}}}*/
1124 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1125 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1126                                                 const string &PkgName,
1127                                                 const string &PkgArch,
1128                                                 const string &Version,
1129                                                 uint8_t const Flags)
1130 {
1131    pkgCache const &Cache = Owner->Cache;
1132
1133    // We do not add self referencing provides
1134    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1135         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1136       return true;
1137
1138    // Locate the target package
1139    pkgCache::PkgIterator Pkg;
1140    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1141    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1142       return false;
1143
1144    map_stringitem_t idxProvideVersion = 0;
1145    if (Version.empty() == false) {
1146       idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1147       if (unlikely(idxProvideVersion == 0))
1148          return false;
1149    }
1150    return Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags);
1151 }
1152 bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
1153                                     pkgCache::PkgIterator &Pkg,
1154                                     map_pointer_t const ProvideVersion,
1155                                     uint8_t const Flags)
1156 {
1157    // Get a structure
1158    map_pointer_t const Provides = AllocateInMap(sizeof(pkgCache::Provides));
1159    if (unlikely(Provides == 0))
1160       return false;
1161    ++Cache.HeaderP->ProvidesCount;
1162
1163    // Fill it in
1164    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1165    Prv->Version = Ver.Index();
1166    Prv->ProvideVersion = ProvideVersion;
1167    Prv->Flags = Flags;
1168    Prv->NextPkgProv = Ver->ProvidesList;
1169    Ver->ProvidesList = Prv.Index();
1170
1171    // Link it to the package
1172    Prv->ParentPkg = Pkg.Index();
1173    Prv->NextProvides = Pkg->ProvidesList;
1174    Pkg->ProvidesList = Prv.Index();
1175    return true;
1176 }
1177                                                                         /*}}}*/
1178 // ListParser::NewProvidesAllArch - add provides for all architectures  /*{{{*/
1179 bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
1180                                 string const &Version, uint8_t const Flags) {
1181    pkgCache &Cache = Owner->Cache;
1182    pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
1183    if (Grp.end() == true)
1184       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
1185    else
1186    {
1187       map_stringitem_t idxProvideVersion = 0;
1188       if (Version.empty() == false) {
1189          idxProvideVersion = StoreString(VERSIONNUMBER, Version);
1190          if (unlikely(idxProvideVersion == 0))
1191             return false;
1192       }
1193
1194       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
1195       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
1196       pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
1197       for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
1198       {
1199          if (isImplicit && OwnerPkg == Pkg)
1200             continue;
1201          if (isArchSpecific == false && APT::Configuration::checkArchitecture(OwnerPkg.Arch()) == false)
1202             continue;
1203          if (Owner->NewProvides(Ver, Pkg, idxProvideVersion, Flags) == false)
1204             return false;
1205       }
1206    }
1207    return true;
1208 }
1209                                                                         /*}}}*/
1210 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1211       pkgCache::VerIterator const &Ver)
1212 {
1213    return Hash == Ver->Hash;
1214 }
1215                                                                         /*}}}*/
1216 // CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to    /*{{{*/
1217 bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
1218                                    unsigned long Flags)
1219 {
1220    if (File.empty() && Site.empty())
1221    {
1222       CurrentRlsFile = NULL;
1223       return true;
1224    }
1225
1226    // Get some space for the structure
1227    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
1228    if (unlikely(idxFile == 0))
1229       return false;
1230    CurrentRlsFile = Cache.RlsFileP + idxFile;
1231
1232    // Fill it in
1233    map_stringitem_t const idxFileName = WriteStringInMap(File);
1234    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1235    if (unlikely(idxFileName == 0 || idxSite == 0))
1236       return false;
1237    CurrentRlsFile->FileName = idxFileName;
1238    CurrentRlsFile->Site = idxSite;
1239    CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
1240    CurrentRlsFile->Flags = Flags;
1241    CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
1242    RlsFileName = File;
1243    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
1244    Cache.HeaderP->ReleaseFileCount++;
1245
1246    return true;
1247 }
1248                                                                         /*}}}*/
1249 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1250 // ---------------------------------------------------------------------
1251 /* This is used to select which file is to be associated with all newly
1252    added versions. The caller is responsible for setting the IMS fields. */
1253 bool pkgCacheGenerator::SelectFile(std::string const &File,
1254                                    pkgIndexFile const &Index,
1255                                    std::string const &Architecture,
1256                                    std::string const &Component,
1257                                    unsigned long const Flags)
1258 {
1259    // Get some space for the structure
1260    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1261    if (unlikely(idxFile == 0))
1262       return false;
1263    CurrentFile = Cache.PkgFileP + idxFile;
1264
1265    // Fill it in
1266    map_stringitem_t const idxFileName = WriteStringInMap(File);
1267    if (unlikely(idxFileName == 0))
1268       return false;
1269    CurrentFile->FileName = idxFileName;
1270    CurrentFile->NextFile = Cache.HeaderP->FileList;
1271    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1272    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1273    if (unlikely(idxIndexType == 0))
1274       return false;
1275    CurrentFile->IndexType = idxIndexType;
1276    if (Architecture.empty())
1277       CurrentFile->Architecture = 0;
1278    else
1279    {
1280       map_stringitem_t const arch = StoreString(pkgCacheGenerator::MIXED, Architecture);
1281       if (unlikely(arch == 0))
1282          return false;
1283       CurrentFile->Architecture = arch;
1284    }
1285    map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
1286    if (unlikely(component == 0))
1287       return false;
1288    CurrentFile->Component = component;
1289    CurrentFile->Flags = Flags;
1290    if (CurrentRlsFile != NULL)
1291       CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
1292    else
1293       CurrentFile->Release = 0;
1294    PkgFileName = File;
1295    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1296    Cache.HeaderP->PackageFileCount++;
1297
1298    if (Progress != 0)
1299       Progress->SubProgress(Index.Size());
1300    return true;
1301 }
1302                                                                         /*}}}*/
1303 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1304 // ---------------------------------------------------------------------
1305 /* This is used to create handles to strings. Given the same text it
1306    always returns the same number */
1307 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1308                                                  unsigned int Size)
1309 {
1310    std::string const key(S, Size);
1311
1312    std::map<std::string,map_stringitem_t> * strings;
1313    switch(type) {
1314       case MIXED: strings = &strMixed; break;
1315       case PKGNAME: strings = &strPkgNames; break;
1316       case VERSIONNUMBER: strings = &strVersions; break;
1317       case SECTION: strings = &strSections; break;
1318       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1319    }
1320
1321    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1322    if (item != strings->end())
1323       return item->second;
1324
1325    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1326    strings->insert(std::make_pair(key, idxString));
1327    return idxString;
1328 }
1329                                                                         /*}}}*/
1330 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1331 // ---------------------------------------------------------------------
1332 /* This just verifies that each file in the list of index files exists,
1333    has matching attributes with the cache and the cache does not have
1334    any extra files. */
1335 static bool CheckValidity(const string &CacheFile,
1336                           pkgSourceList &List,
1337                           FileIterator Start,
1338                           FileIterator End,
1339                           MMap **OutMap = 0)
1340 {
1341    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1342    // No file, certainly invalid
1343    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1344    {
1345       if (Debug == true)
1346          std::clog << "CacheFile doesn't exist" << std::endl;
1347       return false;
1348    }
1349
1350    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1351    {
1352       if (Debug == true)
1353          std::clog << "sources.list is newer than the cache" << std::endl;
1354       return false;
1355    }
1356
1357    // Map it
1358    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1359    SPtr<MMap> Map = new MMap(CacheF,0);
1360    pkgCache Cache(Map);
1361    if (_error->PendingError() == true || Map->Size() == 0)
1362    {
1363       if (Debug == true)
1364          std::clog << "Errors are pending or Map is empty()" << std::endl;
1365       _error->Discard();
1366       return false;
1367    }
1368
1369    SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
1370    memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
1371    std::vector<pkgIndexFile *> Files;
1372    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
1373    {
1374       if (Debug == true)
1375          std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
1376       pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
1377       if (RlsFile.end() == true)
1378       {
1379          if (Debug == true)
1380             std::clog << "FindInCache returned end-Pointer" << std::endl;
1381          return false;
1382       }
1383
1384       RlsVisited[RlsFile->ID] = true;
1385       if (Debug == true)
1386          std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
1387
1388       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1389       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1390          if ((*j)->HasPackages())
1391             Files.push_back (*j);
1392    }
1393    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
1394       if (RlsVisited[I] == false)
1395       {
1396          if (Debug == true)
1397             std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
1398          return false;
1399       }
1400
1401    for (; Start != End; ++Start)
1402       Files.push_back(*Start);
1403
1404    /* Now we check every index file, see if it is in the cache,
1405       verify the IMS data and check that it is on the disk too.. */
1406    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1407    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1408    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
1409    {
1410       if (Debug == true)
1411          std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
1412       if ((*PkgFile)->Exists() == false)
1413       {
1414          if (Debug == true)
1415             std::clog << "file doesn't exist" << std::endl;
1416          continue;
1417       }
1418
1419       // FindInCache is also expected to do an IMS check.
1420       pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
1421       if (File.end() == true)
1422       {
1423          if (Debug == true)
1424             std::clog << "FindInCache returned end-Pointer" << std::endl;
1425          return false;
1426       }
1427
1428       Visited[File->ID] = true;
1429       if (Debug == true)
1430          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1431    }
1432
1433    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1434       if (Visited[I] == false)
1435       {
1436          if (Debug == true)
1437             std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
1438          return false;
1439       }
1440
1441    if (_error->PendingError() == true)
1442    {
1443       if (Debug == true)
1444       {
1445          std::clog << "Validity failed because of pending errors:" << std::endl;
1446          _error->DumpErrors();
1447       }
1448       _error->Discard();
1449       return false;
1450    }
1451
1452    if (OutMap != 0)
1453       *OutMap = Map.UnGuard();
1454    return true;
1455 }
1456                                                                         /*}}}*/
1457 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1458 // ---------------------------------------------------------------------
1459 /* Size is kind of an abstract notion that is only used for the progress
1460    meter */
1461 static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
1462 {
1463    map_filesize_t TotalSize = 0;
1464    if (List !=  NULL)
1465    {
1466       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1467       {
1468          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1469          for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
1470             if ((*j)->HasPackages() == true)
1471                TotalSize += (*j)->Size();
1472       }
1473    }
1474
1475    for (; Start < End; ++Start)
1476    {
1477       if ((*Start)->HasPackages() == false)
1478          continue;
1479       TotalSize += (*Start)->Size();
1480    }
1481    return TotalSize;
1482 }
1483                                                                         /*}}}*/
1484 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1485 // ---------------------------------------------------------------------
1486 /* */
1487 static bool BuildCache(pkgCacheGenerator &Gen,
1488                        OpProgress *Progress,
1489                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1490                        pkgSourceList const * const List,
1491                        FileIterator Start, FileIterator End)
1492 {
1493    std::vector<pkgIndexFile *> Files;
1494    bool const HasFileDeps = Gen.HasFileDeps();
1495
1496    if (List !=  NULL)
1497    {
1498       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
1499       {
1500          if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
1501          {
1502             _error->Warning("Duplicate sources.list entry %s",
1503                   (*i)->Describe().c_str());
1504             continue;
1505          }
1506
1507          if ((*i)->Merge(Gen, Progress) == false)
1508             return false;
1509
1510          std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1511          for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
1512          {
1513             if (HasFileDeps)
1514                Files.push_back(*I);
1515
1516             if ((*I)->HasPackages() == false)
1517                continue;
1518
1519             if ((*I)->Exists() == false)
1520                continue;
1521
1522             if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1523             {
1524                _error->Warning("Duplicate sources.list entry %s",
1525                      (*I)->Describe().c_str());
1526                continue;
1527             }
1528
1529             map_filesize_t Size = (*I)->Size();
1530             if (Progress != NULL)
1531                Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1532             CurrentSize += Size;
1533
1534             if ((*I)->Merge(Gen,Progress) == false)
1535                return false;
1536          }
1537       }
1538    }
1539
1540    Gen.SelectReleaseFile("", "");
1541    FileIterator I;
1542    for (I = Start; I != End; ++I)
1543    {
1544       if (HasFileDeps)
1545          Files.push_back(*I);
1546
1547       if ((*I)->HasPackages() == false)
1548          continue;
1549
1550       if ((*I)->Exists() == false)
1551          continue;
1552
1553       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1554       {
1555          _error->Warning("Duplicate sources.list entry %s",
1556                          (*I)->Describe().c_str());
1557          continue;
1558       }
1559
1560       map_filesize_t Size = (*I)->Size();
1561       if (Progress != NULL)
1562          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1563       CurrentSize += Size;
1564
1565       if ((*I)->Merge(Gen,Progress) == false)
1566          return false;
1567    }
1568
1569    if (HasFileDeps == true)
1570    {
1571       if (Progress != NULL)
1572          Progress->Done();
1573       TotalSize = ComputeSize(List, Start, End);
1574       CurrentSize = 0;
1575       for (std::vector<pkgIndexFile *>::const_iterator I = Files.begin(); I != Files.end(); ++I)
1576       {
1577          map_filesize_t Size = (*I)->Size();
1578          if (Progress != NULL)
1579             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1580          CurrentSize += Size;
1581          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1582             return false;
1583       }
1584    }
1585
1586    return true;
1587 }
1588                                                                         /*}}}*/
1589 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1590 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1591    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1592    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1593    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1594    Flags |= MMap::Moveable;
1595    if (_config->FindB("APT::Cache-Fallback", false) == true)
1596       Flags |= MMap::Fallback;
1597    if (CacheF != NULL)
1598       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1599    else
1600       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1601 }
1602                                                                         /*}}}*/
1603 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1604 // ---------------------------------------------------------------------
1605 /* This makes sure that the status cache (the cache that has all
1606    index files from the sources list and all local ones) is ready
1607    to be mmaped. If OutMap is not zero then a MMap object representing
1608    the cache will be stored there. This is pretty much mandetory if you
1609    are using AllowMem. AllowMem lets the function be run as non-root
1610    where it builds the cache 'fast' into a memory buffer. */
1611 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1612                         MMap **OutMap, bool AllowMem)
1613    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1614 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1615                         MMap **OutMap,bool AllowMem)
1616 {
1617    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1618
1619    std::vector<pkgIndexFile *> Files;
1620    /*
1621    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1622         i != List.end();
1623         ++i)
1624    {
1625       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1626       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1627            j != Indexes->end();
1628            ++j)
1629          Files.push_back (*j);
1630    }
1631 */
1632    if (_system->AddStatusFiles(Files) == false)
1633       return false;
1634
1635    // Decide if we can write to the files..
1636    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1637    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1638
1639    // ensure the cache directory exists
1640    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1641    {
1642       string dir = _config->FindDir("Dir::Cache");
1643       size_t const len = dir.size();
1644       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1645          dir = dir.substr(0, len - 5);
1646       if (CacheFile.empty() == false)
1647          CreateDirectory(dir, flNotFile(CacheFile));
1648       if (SrcCacheFile.empty() == false)
1649          CreateDirectory(dir, flNotFile(SrcCacheFile));
1650    }
1651
1652    // Decide if we can write to the cache
1653    bool Writeable = false;
1654    if (CacheFile.empty() == false)
1655       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1656    else
1657       if (SrcCacheFile.empty() == false)
1658          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1659    if (Debug == true)
1660       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1661
1662    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1663       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1664
1665    if (Progress != NULL)
1666       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1667
1668    // Cache is OK, Fin.
1669    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1670    {
1671       if (Progress != NULL)
1672          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1673       if (Debug == true)
1674          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1675       return true;
1676    }
1677    else if (Debug == true)
1678          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1679
1680    /* At this point we know we need to reconstruct the package cache,
1681       begin. */
1682    SPtr<FileFd> CacheF;
1683    SPtr<DynamicMMap> Map;
1684    if (Writeable == true && CacheFile.empty() == false)
1685    {
1686       _error->PushToStack();
1687       unlink(CacheFile.c_str());
1688       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1689       fchmod(CacheF->Fd(),0644);
1690       Map = CreateDynamicMMap(CacheF, MMap::Public);
1691       if (_error->PendingError() == true)
1692       {
1693          delete CacheF.UnGuard();
1694          delete Map.UnGuard();
1695          if (Debug == true)
1696             std::clog << "Open filebased MMap FAILED" << std::endl;
1697          Writeable = false;
1698          if (AllowMem == false)
1699          {
1700             _error->MergeWithStack();
1701             return false;
1702          }
1703          _error->RevertToStack();
1704       }
1705       else
1706       {
1707          _error->MergeWithStack();
1708          if (Debug == true)
1709             std::clog << "Open filebased MMap" << std::endl;
1710       }
1711    }
1712    if (Writeable == false || CacheFile.empty() == true)
1713    {
1714       // Just build it in memory..
1715       Map = CreateDynamicMMap(NULL);
1716       if (Debug == true)
1717          std::clog << "Open memory Map (not filebased)" << std::endl;
1718    }
1719
1720    // Lets try the source cache.
1721    map_filesize_t CurrentSize = 0;
1722    map_filesize_t TotalSize = 0;
1723    if (CheckValidity(SrcCacheFile, List, Files.end(),
1724                      Files.end()) == true)
1725    {
1726       if (Debug == true)
1727          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1728       // Preload the map with the source cache
1729       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1730       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1731       if ((alloc == 0 && _error->PendingError())
1732                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1733                                 SCacheF.Size()) == false)
1734          return false;
1735
1736       TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1737
1738       // Build the status cache
1739       pkgCacheGenerator Gen(Map.Get(),Progress);
1740       if (_error->PendingError() == true)
1741          return false;
1742       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1743                      Files.begin(),Files.end()) == false)
1744          return false;
1745    }
1746    else
1747    {
1748       if (Debug == true)
1749          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1750       TotalSize = ComputeSize(&List, Files.begin(),Files.end());
1751
1752       // Build the source cache
1753       pkgCacheGenerator Gen(Map.Get(),Progress);
1754       if (_error->PendingError() == true)
1755          return false;
1756       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
1757                      Files.end(),Files.end()) == false)
1758          return false;
1759
1760       // Write it back
1761       if (Writeable == true && SrcCacheFile.empty() == false)
1762       {
1763          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1764          if (_error->PendingError() == true)
1765             return false;
1766
1767          fchmod(SCacheF.Fd(),0644);
1768
1769          // Write out the main data
1770          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1771             return _error->Error(_("IO Error saving source cache"));
1772          SCacheF.Sync();
1773
1774          // Write out the proper header
1775          Gen.GetCache().HeaderP->Dirty = false;
1776          if (SCacheF.Seek(0) == false ||
1777              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1778             return _error->Error(_("IO Error saving source cache"));
1779          Gen.GetCache().HeaderP->Dirty = true;
1780          SCacheF.Sync();
1781       }
1782
1783       // Build the status cache
1784       if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
1785                      Files.begin(), Files.end()) == false)
1786          return false;
1787    }
1788    if (Debug == true)
1789       std::clog << "Caches are ready for shipping" << std::endl;
1790
1791    if (_error->PendingError() == true)
1792       return false;
1793    if (OutMap != 0)
1794    {
1795       if (CacheF != 0)
1796       {
1797          delete Map.UnGuard();
1798          *OutMap = new MMap(*CacheF,0);
1799       }
1800       else
1801       {
1802          *OutMap = Map.UnGuard();
1803       }
1804    }
1805
1806    return true;
1807 }
1808                                                                         /*}}}*/
1809 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1810 // ---------------------------------------------------------------------
1811 /* */
1812 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1813    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1814 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1815 {
1816    std::vector<pkgIndexFile *> Files;
1817    if (_system->AddStatusFiles(Files) == false)
1818       return false;
1819
1820    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1821    map_filesize_t CurrentSize = 0;
1822    map_filesize_t TotalSize = 0;
1823
1824    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
1825
1826    // Build the status cache
1827    if (Progress != NULL)
1828       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1829    pkgCacheGenerator Gen(Map.Get(),Progress);
1830    if (_error->PendingError() == true)
1831       return false;
1832    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
1833                   Files.begin(), Files.end()) == false)
1834       return false;
1835
1836    if (_error->PendingError() == true)
1837       return false;
1838    *OutMap = Map.UnGuard();
1839
1840    return true;
1841 }
1842                                                                         /*}}}*/
1843 // IsDuplicateDescription                                               /*{{{*/
1844 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1845                             MD5SumValue const &CurMd5, std::string const &CurLang)
1846 {
1847    // Descriptions in the same link-list have all the same md5
1848    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1849       return false;
1850    for (; Desc.end() == false; ++Desc)
1851       if (Desc.LanguageCode() == CurLang)
1852          return true;
1853    return false;
1854 }
1855                                                                         /*}}}*/
1856 // CacheGenerator::FinishCache                                          /*{{{*/
1857 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1858 {
1859    return true;
1860 }
1861                                                                         /*}}}*/
1862
1863 pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {}
1864 pkgCacheGenerator::ListParser::~ListParser() {}