apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                     FoundFileDeps(0)
  58 {
  59    CurrentFile = 0;
  60
  61    if (_error->PendingError() == true)
  62       return;
  63
  64    if (Map.Size() == 0)
  65    {
  66       // Setup the map interface..
  67       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  68       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  69          return;
  70
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72
  73       // Starting header
  74       *Cache.HeaderP = pkgCache::Header();
  75
  76       // make room for the hashtables for packages and groups
  77       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
  78          return;
  79
  80       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  81       if (unlikely(idxVerSysName == 0))
  82          return;
  83       Cache.HeaderP->VerSysName = idxVerSysName;
  84       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
  85       if (unlikely(idxArchitecture == 0))
  86          return;
  87       Cache.HeaderP->Architecture = idxArchitecture;
  88
  89       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  90       if (archs.size() > 1)
  91       {
  92          std::vector<std::string>::const_iterator a = archs.begin();
  93          std::string list = *a;
  94          for (++a; a != archs.end(); ++a)
  95             list.append(",").append(*a);
  96          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
  97          if (unlikely(idxArchitectures == 0))
  98             return;
  99          Cache.HeaderP->SetArchitectures(idxArchitectures);
 100       }
 101       else
 102          Cache.HeaderP->SetArchitectures(idxArchitecture);
 103
 104       Cache.ReMap();
 105    }
 106    else
 107    {
 108       // Map directly from the existing file
 109       Cache.ReMap();
 110       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 111       if (Cache.VS != _system->VS)
 112       {
 113          _error->Error(_("Cache has an incompatible versioning system"));
 114          return;
 115       }
 116    }
 117
 118    Cache.HeaderP->Dirty = true;
 119    Map.Sync(0,sizeof(pkgCache::Header));
 120 }
 121                                                                         /*}}}*/
 122 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* We sync the data then unset the dirty flag in two steps so as to
 125    advoid a problem during a crash */
 126 pkgCacheGenerator::~pkgCacheGenerator()
 127 {
 128    if (_error->PendingError() == true)
 129       return;
 130    if (Map.Sync() == false)
 131       return;
 132
 133    Cache.HeaderP->Dirty = false;
 134    Cache.HeaderP->CacheFileSize = Map.Size();
 135    Map.Sync(0,sizeof(pkgCache::Header));
 136 }
 137                                                                         /*}}}*/
 138 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 139    if (oldMap == newMap)
 140       return;
 141
 142    if (_config->FindB("Debug::pkgCacheGen", false))
 143       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 144
 145    Cache.ReMap(false);
 146
 147    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 148
 149    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 150         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 151       (*i)->ReMap(oldMap, newMap);
 152    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 153         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 154       (*i)->ReMap(oldMap, newMap);
 155    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 156         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 157       (*i)->ReMap(oldMap, newMap);
 158    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 159         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 160       (*i)->ReMap(oldMap, newMap);
 161    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 162         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 163       (*i)->ReMap(oldMap, newMap);
 164    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 165         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 166       (*i)->ReMap(oldMap, newMap);
 167    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 168         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 169       (*i)->ReMap(oldMap, newMap);
 170 }                                                                       /*}}}*/
 171 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 172 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 173                                         const unsigned long &Len) {
 174    void const * const oldMap = Map.Data();
 175    map_stringitem_t const index = Map.WriteString(String, Len);
 176    if (index != 0)
 177       ReMap(oldMap, Map.Data());
 178    return index;
 179 }
 180                                                                         /*}}}*/
 181 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 182 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 183    void const * const oldMap = Map.Data();
 184    map_stringitem_t const index = Map.WriteString(String);
 185    if (index != 0)
 186       ReMap(oldMap, Map.Data());
 187    return index;
 188 }
 189                                                                         /*}}}*/
 190 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 191    void const * const oldMap = Map.Data();
 192    map_pointer_t const index = Map.Allocate(size);
 193    if (index != 0)
 194       ReMap(oldMap, Map.Data());
 195    return index;
 196 }
 197                                                                         /*}}}*/
 198 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 199 // ---------------------------------------------------------------------
 200 /* This provides the generation of the entries in the cache. Each loop
 201    goes through a single package record from the underlying parse engine. */
 202 bool pkgCacheGenerator::MergeList(ListParser &List,
 203                                   pkgCache::VerIterator *OutVer)
 204 {
 205    List.Owner = this;
 206
 207    unsigned int Counter = 0;
 208    while (List.Step() == true)
 209    {
 210       string const PackageName = List.Package();
 211       if (PackageName.empty() == true)
 212          return false;
 213
 214       Counter++;
 215       if (Counter % 100 == 0 && Progress != 0)
 216          Progress->Progress(List.Offset());
 217
 218       string Arch = List.Architecture();
 219       string const Version = List.Version();
 220       if (Version.empty() == true && Arch.empty() == true)
 221       {
 222          // package descriptions
 223          if (MergeListGroup(List, PackageName) == false)
 224             return false;
 225          continue;
 226       }
 227
 228       if (Arch.empty() == true)
 229       {
 230          // use the pseudo arch 'none' for arch-less packages
 231          Arch = "none";
 232          /* We might built a SingleArchCache here, which we don't want to blow up
 233             just for these :none packages to a proper MultiArchCache, so just ensure
 234             that we have always a native package structure first for SingleArch */
 235          pkgCache::PkgIterator NP;
 236          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 237          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 238          // TRANSLATOR: The first placeholder is a package name,
 239          // the other two should be copied verbatim as they include debug info
 240          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 241                               PackageName.c_str(), "NewPackage", 0);
 242       }
 243
 244       // Get a pointer to the package structure
 245       pkgCache::PkgIterator Pkg;
 246       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 247       if (NewPackage(Pkg, PackageName, Arch) == false)
 248          // TRANSLATOR: The first placeholder is a package name,
 249          // the other two should be copied verbatim as they include debug info
 250          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 251                               PackageName.c_str(), "NewPackage", 1);
 252
 253
 254       if (Version.empty() == true)
 255       {
 256          if (MergeListPackage(List, Pkg) == false)
 257             return false;
 258       }
 259       else
 260       {
 261          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 262             return false;
 263       }
 264
 265       if (OutVer != 0)
 266       {
 267          FoundFileDeps |= List.HasFileDeps();
 268          return true;
 269       }
 270    }
 271
 272    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 273       return _error->Error(_("Wow, you exceeded the number of package "
 274                              "names this APT is capable of."));
 275    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 276       return _error->Error(_("Wow, you exceeded the number of versions "
 277                              "this APT is capable of."));
 278    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 279       return _error->Error(_("Wow, you exceeded the number of descriptions "
 280                              "this APT is capable of."));
 281    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 282       return _error->Error(_("Wow, you exceeded the number of dependencies "
 283                              "this APT is capable of."));
 284
 285    FoundFileDeps |= List.HasFileDeps();
 286    return true;
 287 }
 288 // CacheGenerator::MergeListGroup                                       /*{{{*/
 289 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 290 {
 291    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 292    // a group has no data on it's own, only packages have it but these
 293    // stanzas like this come from Translation- files to add descriptions,
 294    // but without a version we don't need a description for it…
 295    if (Grp.end() == true)
 296       return true;
 297    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 298
 299    pkgCache::PkgIterator Pkg;
 300    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 301    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 302       if (MergeListPackage(List, Pkg) == false)
 303          return false;
 304
 305    return true;
 306 }
 307                                                                         /*}}}*/
 308 // CacheGenerator::MergeListPackage                                     /*{{{*/
 309 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 310 {
 311    // we first process the package, then the descriptions
 312    // (for deb this package processing is in fact a no-op)
 313    pkgCache::VerIterator Ver(Cache);
 314    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 315    if (List.UsePackage(Pkg, Ver) == false)
 316       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 317                            Pkg.Name(), "UsePackage", 1);
 318
 319    // Find the right version to write the description
 320    MD5SumValue CurMd5 = List.Description_md5();
 321    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 322       return true;
 323    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 324    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 325    {
 326       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 327
 328       // a version can only have one md5 describing it
 329       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 330          continue;
 331
 332       map_stringitem_t md5idx = VerDesc->md5sum;
 333       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 334       {
 335          // don't add a new description if we have one for the given
 336          // md5 && language
 337          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 338             continue;
 339
 340          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 341       }
 342
 343       // we can stop here as all "same" versions will share the description
 344       break;
 345    }
 346
 347    return true;
 348 }
 349                                                                         /*}}}*/
 350 // CacheGenerator::MergeListVersion                                     /*{{{*/
 351 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 352                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 353 {
 354    pkgCache::VerIterator Ver = Pkg.VersionList();
 355    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 356    map_pointer_t *LastVer = &Pkg->VersionList;
 357    void const * oldMap = Map.Data();
 358
 359    unsigned short const Hash = List.VersionHash();
 360    if (Ver.end() == false)
 361    {
 362       /* We know the list is sorted so we use that fact in the search.
 363          Insertion of new versions is done with correct sorting */
 364       int Res = 1;
 365       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
 366       {
 367          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 368          // Version is higher as current version - insert here
 369          if (Res > 0)
 370             break;
 371          // Versionstrings are equal - is hash also equal?
 372          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 373             break;
 374          // proceed with the next till we have either the right
 375          // or we found another version (which will be lower)
 376       }
 377
 378       /* We already have a version for this item, record that we saw it */
 379       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 380       {
 381          if (List.UsePackage(Pkg,Ver) == false)
 382             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 383                                  Pkg.Name(), "UsePackage", 2);
 384
 385          if (NewFileVer(Ver,List) == false)
 386             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 387                                  Pkg.Name(), "NewFileVer", 1);
 388
 389          // Read only a single record and return
 390          if (OutVer != 0)
 391          {
 392             *OutVer = Ver;
 393             return true;
 394          }
 395
 396          return true;
 397       }
 398    }
 399
 400    // Add a new version
 401    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 402    if (verindex == 0 && _error->PendingError())
 403       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 404                            Pkg.Name(), "NewVersion", 1);
 405
 406    if (oldMap != Map.Data())
 407          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 408    *LastVer = verindex;
 409
 410    if (unlikely(List.NewVersion(Ver) == false))
 411       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 412                            Pkg.Name(), "NewVersion", 2);
 413
 414    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 415       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 416                            Pkg.Name(), "UsePackage", 3);
 417
 418    if (unlikely(NewFileVer(Ver,List) == false))
 419       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 420                            Pkg.Name(), "NewFileVer", 2);
 421
 422    pkgCache::GrpIterator Grp = Pkg.Group();
 423    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 424
 425    /* If it is the first version of this package we need to add implicit
 426       Multi-Arch dependencies to all other package versions in the group now -
 427       otherwise we just add them for this new version */
 428    if (Pkg.VersionList()->NextVer == 0)
 429    {
 430       pkgCache::PkgIterator P = Grp.PackageList();
 431       Dynamic<pkgCache::PkgIterator> DynP(P);
 432       for (; P.end() != true; P = Grp.NextPkg(P))
 433       {
 434          if (P->ID == Pkg->ID)
 435             continue;
 436          pkgCache::VerIterator V = P.VersionList();
 437          Dynamic<pkgCache::VerIterator> DynV(V);
 438          for (; V.end() != true; ++V)
 439             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 440                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 441                                     Pkg.Name(), "AddImplicitDepends", 1);
 442       }
 443       /* :none packages are packages without an architecture. They are forbidden by
 444          debian-policy, so usually they will only be in (old) dpkg status files -
 445          and dpkg will complain about them - and are pretty rare. We therefore do
 446          usually not create conflicts while the parent is created, but only if a :none
 447          package (= the target) appears. This creates incorrect dependencies on :none
 448          for architecture-specific dependencies on the package we copy from, but we
 449          will ignore this bug as architecture-specific dependencies are only allowed
 450          in jessie and until then the :none packages should be extinct (hopefully).
 451          In other words: This should work long enough to allow graceful removal of
 452          these packages, it is not supposed to allow users to keep using them … */
 453       if (strcmp(Pkg.Arch(), "none") == 0)
 454       {
 455          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 456          if (M.end() == false && Pkg != M)
 457          {
 458             pkgCache::DepIterator D = M.RevDependsList();
 459             Dynamic<pkgCache::DepIterator> DynD(D);
 460             for (; D.end() == false; ++D)
 461             {
 462                if ((D->Type != pkgCache::Dep::Conflicts &&
 463                     D->Type != pkgCache::Dep::DpkgBreaks &&
 464                     D->Type != pkgCache::Dep::Replaces) ||
 465                    D.ParentPkg().Group() == Grp)
 466                   continue;
 467
 468                map_pointer_t *OldDepLast = NULL;
 469                pkgCache::VerIterator ConVersion = D.ParentVer();
 470                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 471                // duplicate the Conflicts/Breaks/Replaces for :none arch
 472                NewDepends(Pkg, ConVersion, D->Version,
 473                      D->CompareOp, D->Type, OldDepLast);
 474             }
 475          }
 476       }
 477    }
 478    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 479       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 480                            Pkg.Name(), "AddImplicitDepends", 2);
 481
 482    // Read only a single record and return
 483    if (OutVer != 0)
 484    {
 485       *OutVer = Ver;
 486       return true;
 487    }
 488
 489    /* Record the Description(s) based on their master md5sum */
 490    MD5SumValue CurMd5 = List.Description_md5();
 491    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 492       return true;
 493
 494    /* Before we add a new description we first search in the group for
 495       a version with a description of the same MD5 - if so we reuse this
 496       description group instead of creating our own for this version */
 497    for (pkgCache::PkgIterator P = Grp.PackageList();
 498         P.end() == false; P = Grp.NextPkg(P))
 499    {
 500       for (pkgCache::VerIterator V = P.VersionList();
 501            V.end() == false; ++V)
 502       {
 503          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 504             continue;
 505          Ver->DescriptionList = V->DescriptionList;
 506       }
 507    }
 508
 509    // We haven't found reusable descriptions, so add the first description(s)
 510    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 511    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 512    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 513       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 514          return false;
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 519 {
 520    pkgCache::DescIterator Desc;
 521    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 522
 523    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 524    if (unlikely(descindex == 0 && _error->PendingError()))
 525       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 526             Ver.ParentPkg().Name(), "NewDescription", 1);
 527
 528    md5idx = Desc->md5sum;
 529    Desc->ParentPkg = Ver.ParentPkg().Index();
 530
 531    // we add at the end, so that the start is constant as we need
 532    // that to be able to efficiently share these lists
 533    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 534    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 535    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 536    *LastNextDesc = descindex;
 537
 538    if (NewFileDesc(Desc,List) == false)
 539       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 540             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 541
 542    return true;
 543 }
 544                                                                         /*}}}*/
 545                                                                         /*}}}*/
 546 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* If we found any file depends while parsing the main list we need to
 549    resolve them. Since it is undesired to load the entire list of files
 550    into the cache as virtual packages we do a two stage effort. MergeList
 551    identifies the file depends and this creates Provdies for them by
 552    re-parsing all the indexs. */
 553 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 554 {
 555    List.Owner = this;
 556
 557    unsigned int Counter = 0;
 558    while (List.Step() == true)
 559    {
 560       string PackageName = List.Package();
 561       if (PackageName.empty() == true)
 562          return false;
 563       string Version = List.Version();
 564       if (Version.empty() == true)
 565          continue;
 566
 567       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 568       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 569       if (Pkg.end() == true)
 570          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 571                                 PackageName.c_str(), "FindPkg", 1);
 572       Counter++;
 573       if (Counter % 100 == 0 && Progress != 0)
 574          Progress->Progress(List.Offset());
 575
 576       unsigned short Hash = List.VersionHash();
 577       pkgCache::VerIterator Ver = Pkg.VersionList();
 578       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 579       for (; Ver.end() == false; ++Ver)
 580       {
 581          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 582          {
 583             if (List.CollectFileProvides(Cache,Ver) == false)
 584                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 585                                     PackageName.c_str(), "CollectFileProvides", 1);
 586             break;
 587          }
 588       }
 589
 590       if (Ver.end() == true)
 591          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 592    }
 593
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This creates a new group structure and adds it to the hash table */
 600 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 601 {
 602    Grp = Cache.FindGrp(Name);
 603    if (Grp.end() == false)
 604       return true;
 605
 606    // Get a structure
 607    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 608    if (unlikely(Group == 0))
 609       return false;
 610
 611    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 612    map_stringitem_t const idxName = StoreString(PKGNAME, Name);
 613    if (unlikely(idxName == 0))
 614       return false;
 615    Grp->Name = idxName;
 616
 617    // Insert it into the hash table
 618    unsigned long const Hash = Cache.Hash(Name);
 619    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
 620    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 621       insertAt = &(Cache.GrpP + *insertAt)->Next;
 622    Grp->Next = *insertAt;
 623    *insertAt = Group;
 624
 625    Grp->ID = Cache.HeaderP->GroupCount++;
 626    return true;
 627 }
 628                                                                         /*}}}*/
 629 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 630 // ---------------------------------------------------------------------
 631 /* This creates a new package structure and adds it to the hash table */
 632 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 633                                         const string &Arch) {
 634    pkgCache::GrpIterator Grp;
 635    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 636    if (unlikely(NewGroup(Grp, Name) == false))
 637       return false;
 638
 639    Pkg = Grp.FindPkg(Arch);
 640       if (Pkg.end() == false)
 641          return true;
 642
 643    // Get a structure
 644    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 645    if (unlikely(Package == 0))
 646       return false;
 647    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 648
 649    // Insert the package into our package list
 650    if (Grp->FirstPackage == 0) // the group is new
 651    {
 652       Grp->FirstPackage = Package;
 653       // Insert it into the hash table
 654       map_id_t const Hash = Cache.Hash(Name);
 655       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
 656       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
 657          insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
 658       Pkg->NextPackage = *insertAt;
 659       *insertAt = Package;
 660    }
 661    else // Group the Packages together
 662    {
 663       // this package is the new last package
 664       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 665       Pkg->NextPackage = LastPkg->NextPackage;
 666       LastPkg->NextPackage = Package;
 667    }
 668    Grp->LastPackage = Package;
 669
 670    // Set the name, arch and the ID
 671    APT_IGNORE_DEPRECATED(Pkg->Name = Grp->Name;)
 672    Pkg->Group = Grp.Index();
 673    // all is mapped to the native architecture
 674    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : StoreString(MIXED, Arch);
 675    if (unlikely(idxArch == 0))
 676       return false;
 677    Pkg->Arch = idxArch;
 678    Pkg->ID = Cache.HeaderP->PackageCount++;
 679
 680    return true;
 681 }
 682                                                                         /*}}}*/
 683 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 684 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 685                                            pkgCache::PkgIterator &P,
 686                                            pkgCache::VerIterator &V)
 687 {
 688    // copy P.Arch() into a string here as a cache remap
 689    // in NewDepends() later may alter the pointer location
 690    string Arch = P.Arch() == NULL ? "" : P.Arch();
 691    map_pointer_t *OldDepLast = NULL;
 692    /* MultiArch handling introduces a lot of implicit Dependencies:
 693       - MultiArch: same → Co-Installable if they have the same version
 694       - All others conflict with all other group members */
 695    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 696    pkgCache::PkgIterator D = G.PackageList();
 697    Dynamic<pkgCache::PkgIterator> DynD(D);
 698    map_stringitem_t const VerStrIdx = V->VerStr;
 699    for (; D.end() != true; D = G.NextPkg(D))
 700    {
 701       if (Arch == D.Arch() || D->VersionList == 0)
 702          continue;
 703       /* We allow only one installed arch at the time
 704          per group, therefore each group member conflicts
 705          with all other group members */
 706       if (coInstall == true)
 707       {
 708          // Replaces: ${self}:other ( << ${binary:Version})
 709          NewDepends(D, V, VerStrIdx,
 710                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 711                     OldDepLast);
 712          // Breaks: ${self}:other (!= ${binary:Version})
 713          NewDepends(D, V, VerStrIdx,
 714                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 715                     OldDepLast);
 716       } else {
 717          // Conflicts: ${self}:other
 718          NewDepends(D, V, 0,
 719                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 720                     OldDepLast);
 721       }
 722    }
 723    return true;
 724 }
 725 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 726                                            pkgCache::PkgIterator &D)
 727 {
 728    /* MultiArch handling introduces a lot of implicit Dependencies:
 729       - MultiArch: same → Co-Installable if they have the same version
 730       - All others conflict with all other group members */
 731    map_pointer_t *OldDepLast = NULL;
 732    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 733    if (coInstall == true)
 734    {
 735       map_stringitem_t const VerStrIdx = V->VerStr;
 736       // Replaces: ${self}:other ( << ${binary:Version})
 737       NewDepends(D, V, VerStrIdx,
 738                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 739                  OldDepLast);
 740       // Breaks: ${self}:other (!= ${binary:Version})
 741       NewDepends(D, V, VerStrIdx,
 742                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 743                  OldDepLast);
 744    } else {
 745       // Conflicts: ${self}:other
 746       NewDepends(D, V, 0,
 747                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 748                  OldDepLast);
 749    }
 750    return true;
 751 }
 752
 753                                                                         /*}}}*/
 754 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 755 // ---------------------------------------------------------------------
 756 /* */
 757 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 758                                    ListParser &List)
 759 {
 760    if (CurrentFile == 0)
 761       return true;
 762
 763    // Get a structure
 764    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 765    if (VerFile == 0)
 766       return false;
 767
 768    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 769    VF->File = CurrentFile - Cache.PkgFileP;
 770
 771    // Link it to the end of the list
 772    map_pointer_t *Last = &Ver->FileList;
 773    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 774       Last = &V->NextFile;
 775    VF->NextFile = *Last;
 776    *Last = VF.Index();
 777
 778    VF->Offset = List.Offset();
 779    VF->Size = List.Size();
 780    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 781       Cache.HeaderP->MaxVerFileSize = VF->Size;
 782    Cache.HeaderP->VerFileCount++;
 783
 784    return true;
 785 }
 786                                                                         /*}}}*/
 787 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 788 // ---------------------------------------------------------------------
 789 /* This puts a version structure in the linked list */
 790 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 791                                             const string &VerStr,
 792                                             map_pointer_t const ParentPkg,
 793                                             unsigned short const Hash,
 794                                             map_pointer_t const Next)
 795 {
 796    // Get a structure
 797    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 798    if (Version == 0)
 799       return 0;
 800
 801    // Fill it in
 802    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 803    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 804    Ver->NextVer = Next;
 805    Ver->ParentPkg = ParentPkg;
 806    Ver->Hash = Hash;
 807    Ver->ID = Cache.HeaderP->VersionCount++;
 808
 809    // try to find the version string in the group for reuse
 810    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 811    pkgCache::GrpIterator Grp = Pkg.Group();
 812    if (Pkg.end() == false && Grp.end() == false)
 813    {
 814       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 815       {
 816          if (Pkg == P)
 817             continue;
 818          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 819          {
 820             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 821             if (cmp == 0)
 822             {
 823                Ver->VerStr = V->VerStr;
 824                return Version;
 825             }
 826             else if (cmp < 0)
 827                break;
 828          }
 829       }
 830    }
 831    // haven't found the version string, so create
 832    map_stringitem_t const idxVerStr = StoreString(VERSIONNUMBER, VerStr);
 833    if (unlikely(idxVerStr == 0))
 834       return 0;
 835    Ver->VerStr = idxVerStr;
 836    return Version;
 837 }
 838                                                                         /*}}}*/
 839 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 840 // ---------------------------------------------------------------------
 841 /* */
 842 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 843                                    ListParser &List)
 844 {
 845    if (CurrentFile == 0)
 846       return true;
 847
 848    // Get a structure
 849    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 850    if (DescFile == 0)
 851       return false;
 852
 853    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 854    DF->File = CurrentFile - Cache.PkgFileP;
 855
 856    // Link it to the end of the list
 857    map_pointer_t *Last = &Desc->FileList;
 858    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 859       Last = &D->NextFile;
 860
 861    DF->NextFile = *Last;
 862    *Last = DF.Index();
 863
 864    DF->Offset = List.Offset();
 865    DF->Size = List.Size();
 866    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 867       Cache.HeaderP->MaxDescFileSize = DF->Size;
 868    Cache.HeaderP->DescFileCount++;
 869
 870    return true;
 871 }
 872                                                                         /*}}}*/
 873 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 874 // ---------------------------------------------------------------------
 875 /* This puts a description structure in the linked list */
 876 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 877                                             const string &Lang,
 878                                             const MD5SumValue &md5sum,
 879                                             map_stringitem_t const idxmd5str)
 880 {
 881    // Get a structure
 882    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 883    if (Description == 0)
 884       return 0;
 885
 886    // Fill it in
 887    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 888    Desc->ID = Cache.HeaderP->DescriptionCount++;
 889    map_stringitem_t const idxlanguage_code = StoreString(MIXED, Lang);
 890    if (unlikely(idxlanguage_code == 0))
 891       return 0;
 892    Desc->language_code = idxlanguage_code;
 893
 894    if (idxmd5str != 0)
 895       Desc->md5sum = idxmd5str;
 896    else
 897    {
 898       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 899       if (unlikely(idxmd5sum == 0))
 900          return 0;
 901       Desc->md5sum = idxmd5sum;
 902    }
 903
 904    return Description;
 905 }
 906                                                                         /*}}}*/
 907 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 908 // ---------------------------------------------------------------------
 909 /* This creates a dependency element in the tree. It is linked to the
 910    version and to the package that it is pointing to. */
 911 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 912                                    pkgCache::VerIterator &Ver,
 913                                    string const &Version,
 914                                    unsigned int const &Op,
 915                                    unsigned int const &Type,
 916                                    map_stringitem_t* &OldDepLast)
 917 {
 918    map_stringitem_t index = 0;
 919    if (Version.empty() == false)
 920    {
 921       int const CmpOp = Op & 0x0F;
 922       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 923       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 924          index = Ver->VerStr;
 925
 926       if (index == 0)
 927       {
 928          void const * const oldMap = Map.Data();
 929          index = StoreString(VERSIONNUMBER, Version);
 930          if (unlikely(index == 0))
 931             return false;
 932          if (OldDepLast != 0 && oldMap != Map.Data())
 933             OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 934       }
 935    }
 936    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 937 }
 938 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 939                                    pkgCache::VerIterator &Ver,
 940                                    map_pointer_t const Version,
 941                                    unsigned int const &Op,
 942                                    unsigned int const &Type,
 943                                    map_pointer_t* &OldDepLast)
 944 {
 945    void const * const oldMap = Map.Data();
 946    // Get a structure
 947    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 948    if (unlikely(Dependency == 0))
 949       return false;
 950
 951    // Fill it in
 952    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 953    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 954    Dep->ParentVer = Ver.Index();
 955    Dep->Type = Type;
 956    Dep->CompareOp = Op;
 957    Dep->Version = Version;
 958    Dep->ID = Cache.HeaderP->DependsCount++;
 959
 960    // Link it to the package
 961    Dep->Package = Pkg.Index();
 962    Dep->NextRevDepends = Pkg->RevDepends;
 963    Pkg->RevDepends = Dep.Index();
 964
 965    // Do we know where to link the Dependency to?
 966    if (OldDepLast == NULL)
 967    {
 968       OldDepLast = &Ver->DependsList;
 969       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 970          OldDepLast = &D->NextDepends;
 971    } else if (oldMap != Map.Data())
 972       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 973
 974    Dep->NextDepends = *OldDepLast;
 975    *OldDepLast = Dep.Index();
 976    OldDepLast = &Dep->NextDepends;
 977
 978    return true;
 979 }
 980                                                                         /*}}}*/
 981 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 982 // ---------------------------------------------------------------------
 983 /* This creates a Group and the Package to link this dependency to if
 984    needed and handles also the caching of the old endpoint */
 985 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 986                                                const string &PackageName,
 987                                                const string &Arch,
 988                                                const string &Version,
 989                                                unsigned int Op,
 990                                                unsigned int Type)
 991 {
 992    pkgCache::GrpIterator Grp;
 993    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 994    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 995       return false;
 996
 997    // Locate the target package
 998    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 999    // we don't create 'none' packages and their dependencies if we can avoid it …
1000    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
1001       return true;
1002    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1003    if (Pkg.end() == true) {
1004       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1005          return false;
1006    }
1007
1008    // Is it a file dependency?
1009    if (unlikely(PackageName[0] == '/'))
1010       FoundFileDeps = true;
1011
1012    /* Caching the old end point speeds up generation substantially */
1013    if (OldDepVer != Ver) {
1014       OldDepLast = NULL;
1015       OldDepVer = Ver;
1016    }
1017
1018    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1019 }
1020                                                                         /*}}}*/
1021 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1022 // ---------------------------------------------------------------------
1023 /* */
1024 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1025                                                 const string &PkgName,
1026                                                 const string &PkgArch,
1027                                                 const string &Version)
1028 {
1029    pkgCache &Cache = Owner->Cache;
1030
1031    // We do not add self referencing provides
1032    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1033         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1034       return true;
1035
1036    // Get a structure
1037    map_pointer_t const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
1038    if (unlikely(Provides == 0))
1039       return false;
1040    Cache.HeaderP->ProvidesCount++;
1041
1042    // Fill it in
1043    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1044    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
1045    Prv->Version = Ver.Index();
1046    Prv->NextPkgProv = Ver->ProvidesList;
1047    Ver->ProvidesList = Prv.Index();
1048    if (Version.empty() == false) {
1049       map_stringitem_t const idxProvideVersion = WriteString(Version);
1050       Prv->ProvideVersion = idxProvideVersion;
1051       if (unlikely(idxProvideVersion == 0))
1052          return false;
1053    }
1054
1055    // Locate the target package
1056    pkgCache::PkgIterator Pkg;
1057    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1058    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1059       return false;
1060
1061    // Link it to the package
1062    Prv->ParentPkg = Pkg.Index();
1063    Prv->NextProvides = Pkg->ProvidesList;
1064    Pkg->ProvidesList = Prv.Index();
1065
1066    return true;
1067 }
1068                                                                         /*}}}*/
1069 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1070       pkgCache::VerIterator const &Ver)
1071 {
1072    return Hash == Ver->Hash;
1073 }
1074                                                                         /*}}}*/
1075 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1076 // ---------------------------------------------------------------------
1077 /* This is used to select which file is to be associated with all newly
1078    added versions. The caller is responsible for setting the IMS fields. */
1079 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
1080                                    const pkgIndexFile &Index,
1081                                    unsigned long Flags)
1082 {
1083    // Get some space for the structure
1084    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1085    if (unlikely(idxFile == 0))
1086       return false;
1087    CurrentFile = Cache.PkgFileP + idxFile;
1088
1089    // Fill it in
1090    map_stringitem_t const idxFileName = WriteStringInMap(File);
1091    map_stringitem_t const idxSite = StoreString(MIXED, Site);
1092    if (unlikely(idxFileName == 0 || idxSite == 0))
1093       return false;
1094    CurrentFile->FileName = idxFileName;
1095    CurrentFile->Site = idxSite;
1096    CurrentFile->NextFile = Cache.HeaderP->FileList;
1097    CurrentFile->Flags = Flags;
1098    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1099    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
1100    if (unlikely(idxIndexType == 0))
1101       return false;
1102    CurrentFile->IndexType = idxIndexType;
1103    PkgFileName = File;
1104    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1105    Cache.HeaderP->PackageFileCount++;
1106
1107    if (Progress != 0)
1108       Progress->SubProgress(Index.Size());
1109    return true;
1110 }
1111                                                                         /*}}}*/
1112 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1113 // ---------------------------------------------------------------------
1114 /* This is used to create handles to strings. Given the same text it
1115    always returns the same number */
1116 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
1117                                                  unsigned int Size)
1118 {
1119    std::string const key(S, Size);
1120
1121    std::map<std::string,map_stringitem_t> * strings;
1122    switch(type) {
1123       case MIXED: strings = &strMixed; break;
1124       case PKGNAME: strings = &strPkgNames; break;
1125       case VERSIONNUMBER: strings = &strVersions; break;
1126       case SECTION: strings = &strSections; break;
1127       default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
1128    }
1129
1130    std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
1131    if (item != strings->end())
1132       return item->second;
1133
1134    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1135    strings->insert(std::make_pair(key, idxString));
1136    return idxString;
1137 }
1138                                                                         /*}}}*/
1139 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1140 // ---------------------------------------------------------------------
1141 /* This just verifies that each file in the list of index files exists,
1142    has matching attributes with the cache and the cache does not have
1143    any extra files. */
1144 static bool CheckValidity(const string &CacheFile,
1145                           pkgSourceList &List,
1146                           FileIterator Start,
1147                           FileIterator End,
1148                           MMap **OutMap = 0)
1149 {
1150    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1151    // No file, certainly invalid
1152    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1153    {
1154       if (Debug == true)
1155          std::clog << "CacheFile doesn't exist" << std::endl;
1156       return false;
1157    }
1158
1159    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1160    {
1161       if (Debug == true)
1162          std::clog << "sources.list is newer than the cache" << std::endl;
1163       return false;
1164    }
1165
1166    // Map it
1167    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1168    SPtr<MMap> Map = new MMap(CacheF,0);
1169    pkgCache Cache(Map);
1170    if (_error->PendingError() == true || Map->Size() == 0)
1171    {
1172       if (Debug == true)
1173          std::clog << "Errors are pending or Map is empty()" << std::endl;
1174       _error->Discard();
1175       return false;
1176    }
1177
1178    /* Now we check every index file, see if it is in the cache,
1179       verify the IMS data and check that it is on the disk too.. */
1180    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1181    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1182    for (; Start != End; ++Start)
1183    {
1184       if (Debug == true)
1185          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1186       if ((*Start)->HasPackages() == false)
1187       {
1188          if (Debug == true)
1189             std::clog << "Has NO packages" << std::endl;
1190          continue;
1191       }
1192
1193       if ((*Start)->Exists() == false)
1194       {
1195 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1196          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1197                           (*Start)->Describe().c_str());
1198 #endif
1199          if (Debug == true)
1200             std::clog << "file doesn't exist" << std::endl;
1201          continue;
1202       }
1203
1204       // FindInCache is also expected to do an IMS check.
1205       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1206       if (File.end() == true)
1207       {
1208          if (Debug == true)
1209             std::clog << "FindInCache returned end-Pointer" << std::endl;
1210          return false;
1211       }
1212
1213       Visited[File->ID] = true;
1214       if (Debug == true)
1215          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1216    }
1217
1218    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1219       if (Visited[I] == false)
1220       {
1221          if (Debug == true)
1222             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1223          return false;
1224       }
1225
1226    if (_error->PendingError() == true)
1227    {
1228       if (Debug == true)
1229       {
1230          std::clog << "Validity failed because of pending errors:" << std::endl;
1231          _error->DumpErrors();
1232       }
1233       _error->Discard();
1234       return false;
1235    }
1236
1237    if (OutMap != 0)
1238       *OutMap = Map.UnGuard();
1239    return true;
1240 }
1241                                                                         /*}}}*/
1242 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1243 // ---------------------------------------------------------------------
1244 /* Size is kind of an abstract notion that is only used for the progress
1245    meter */
1246 static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
1247 {
1248    map_filesize_t TotalSize = 0;
1249    for (; Start < End; ++Start)
1250    {
1251       if ((*Start)->HasPackages() == false)
1252          continue;
1253       TotalSize += (*Start)->Size();
1254    }
1255    return TotalSize;
1256 }
1257                                                                         /*}}}*/
1258 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1259 // ---------------------------------------------------------------------
1260 /* */
1261 static bool BuildCache(pkgCacheGenerator &Gen,
1262                        OpProgress *Progress,
1263                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1264                        FileIterator Start, FileIterator End)
1265 {
1266    FileIterator I;
1267    for (I = Start; I != End; ++I)
1268    {
1269       if ((*I)->HasPackages() == false)
1270          continue;
1271
1272       if ((*I)->Exists() == false)
1273          continue;
1274
1275       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1276       {
1277          _error->Warning("Duplicate sources.list entry %s",
1278                          (*I)->Describe().c_str());
1279          continue;
1280       }
1281
1282       map_filesize_t Size = (*I)->Size();
1283       if (Progress != NULL)
1284          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1285       CurrentSize += Size;
1286
1287       if ((*I)->Merge(Gen,Progress) == false)
1288          return false;
1289    }
1290
1291    if (Gen.HasFileDeps() == true)
1292    {
1293       if (Progress != NULL)
1294          Progress->Done();
1295       TotalSize = ComputeSize(Start, End);
1296       CurrentSize = 0;
1297       for (I = Start; I != End; ++I)
1298       {
1299          map_filesize_t Size = (*I)->Size();
1300          if (Progress != NULL)
1301             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1302          CurrentSize += Size;
1303          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1304             return false;
1305       }
1306    }
1307
1308    return true;
1309 }
1310                                                                         /*}}}*/
1311 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1312 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1313    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1314    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1315    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1316    Flags |= MMap::Moveable;
1317    if (_config->FindB("APT::Cache-Fallback", false) == true)
1318       Flags |= MMap::Fallback;
1319    if (CacheF != NULL)
1320       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1321    else
1322       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1323 }
1324                                                                         /*}}}*/
1325 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1326 // ---------------------------------------------------------------------
1327 /* This makes sure that the status cache (the cache that has all
1328    index files from the sources list and all local ones) is ready
1329    to be mmaped. If OutMap is not zero then a MMap object representing
1330    the cache will be stored there. This is pretty much mandetory if you
1331    are using AllowMem. AllowMem lets the function be run as non-root
1332    where it builds the cache 'fast' into a memory buffer. */
1333 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1334                         MMap **OutMap, bool AllowMem)
1335    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1336 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1337                         MMap **OutMap,bool AllowMem)
1338 {
1339    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1340
1341    std::vector<pkgIndexFile *> Files;
1342    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1343         i != List.end();
1344         ++i)
1345    {
1346       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1347       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1348            j != Indexes->end();
1349            ++j)
1350          Files.push_back (*j);
1351    }
1352
1353    map_filesize_t const EndOfSource = Files.size();
1354    if (_system->AddStatusFiles(Files) == false)
1355       return false;
1356
1357    // Decide if we can write to the files..
1358    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1359    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1360
1361    // ensure the cache directory exists
1362    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1363    {
1364       string dir = _config->FindDir("Dir::Cache");
1365       size_t const len = dir.size();
1366       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1367          dir = dir.substr(0, len - 5);
1368       if (CacheFile.empty() == false)
1369          CreateDirectory(dir, flNotFile(CacheFile));
1370       if (SrcCacheFile.empty() == false)
1371          CreateDirectory(dir, flNotFile(SrcCacheFile));
1372    }
1373
1374    // Decide if we can write to the cache
1375    bool Writeable = false;
1376    if (CacheFile.empty() == false)
1377       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1378    else
1379       if (SrcCacheFile.empty() == false)
1380          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1381    if (Debug == true)
1382       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1383
1384    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1385       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1386
1387    if (Progress != NULL)
1388       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1389
1390    // Cache is OK, Fin.
1391    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1392    {
1393       if (Progress != NULL)
1394          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1395       if (Debug == true)
1396          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1397       return true;
1398    }
1399    else if (Debug == true)
1400          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1401
1402    /* At this point we know we need to reconstruct the package cache,
1403       begin. */
1404    SPtr<FileFd> CacheF;
1405    SPtr<DynamicMMap> Map;
1406    if (Writeable == true && CacheFile.empty() == false)
1407    {
1408       _error->PushToStack();
1409       unlink(CacheFile.c_str());
1410       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1411       fchmod(CacheF->Fd(),0644);
1412       Map = CreateDynamicMMap(CacheF, MMap::Public);
1413       if (_error->PendingError() == true)
1414       {
1415          delete CacheF.UnGuard();
1416          delete Map.UnGuard();
1417          if (Debug == true)
1418             std::clog << "Open filebased MMap FAILED" << std::endl;
1419          Writeable = false;
1420          if (AllowMem == false)
1421          {
1422             _error->MergeWithStack();
1423             return false;
1424          }
1425          _error->RevertToStack();
1426       }
1427       else
1428       {
1429          _error->MergeWithStack();
1430          if (Debug == true)
1431             std::clog << "Open filebased MMap" << std::endl;
1432       }
1433    }
1434    if (Writeable == false || CacheFile.empty() == true)
1435    {
1436       // Just build it in memory..
1437       Map = CreateDynamicMMap(NULL);
1438       if (Debug == true)
1439          std::clog << "Open memory Map (not filebased)" << std::endl;
1440    }
1441
1442    // Lets try the source cache.
1443    map_filesize_t CurrentSize = 0;
1444    map_filesize_t TotalSize = 0;
1445    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1446                      Files.begin()+EndOfSource) == true)
1447    {
1448       if (Debug == true)
1449          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1450       // Preload the map with the source cache
1451       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1452       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1453       if ((alloc == 0 && _error->PendingError())
1454                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1455                                 SCacheF.Size()) == false)
1456          return false;
1457
1458       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1459
1460       // Build the status cache
1461       pkgCacheGenerator Gen(Map.Get(),Progress);
1462       if (_error->PendingError() == true)
1463          return false;
1464       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1465                      Files.begin()+EndOfSource,Files.end()) == false)
1466          return false;
1467    }
1468    else
1469    {
1470       if (Debug == true)
1471          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1472       TotalSize = ComputeSize(Files.begin(),Files.end());
1473
1474       // Build the source cache
1475       pkgCacheGenerator Gen(Map.Get(),Progress);
1476       if (_error->PendingError() == true)
1477          return false;
1478       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1479                      Files.begin(),Files.begin()+EndOfSource) == false)
1480          return false;
1481
1482       // Write it back
1483       if (Writeable == true && SrcCacheFile.empty() == false)
1484       {
1485          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1486          if (_error->PendingError() == true)
1487             return false;
1488
1489          fchmod(SCacheF.Fd(),0644);
1490
1491          // Write out the main data
1492          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1493             return _error->Error(_("IO Error saving source cache"));
1494          SCacheF.Sync();
1495
1496          // Write out the proper header
1497          Gen.GetCache().HeaderP->Dirty = false;
1498          if (SCacheF.Seek(0) == false ||
1499              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1500             return _error->Error(_("IO Error saving source cache"));
1501          Gen.GetCache().HeaderP->Dirty = true;
1502          SCacheF.Sync();
1503       }
1504
1505       // Build the status cache
1506       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1507                      Files.begin()+EndOfSource,Files.end()) == false)
1508          return false;
1509    }
1510    if (Debug == true)
1511       std::clog << "Caches are ready for shipping" << std::endl;
1512
1513    if (_error->PendingError() == true)
1514       return false;
1515    if (OutMap != 0)
1516    {
1517       if (CacheF != 0)
1518       {
1519          delete Map.UnGuard();
1520          *OutMap = new MMap(*CacheF,0);
1521       }
1522       else
1523       {
1524          *OutMap = Map.UnGuard();
1525       }
1526    }
1527
1528    return true;
1529 }
1530                                                                         /*}}}*/
1531 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1532 // ---------------------------------------------------------------------
1533 /* */
1534 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1535    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1536 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1537 {
1538    std::vector<pkgIndexFile *> Files;
1539    map_filesize_t EndOfSource = Files.size();
1540    if (_system->AddStatusFiles(Files) == false)
1541       return false;
1542
1543    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1544    map_filesize_t CurrentSize = 0;
1545    map_filesize_t TotalSize = 0;
1546
1547    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1548
1549    // Build the status cache
1550    if (Progress != NULL)
1551       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1552    pkgCacheGenerator Gen(Map.Get(),Progress);
1553    if (_error->PendingError() == true)
1554       return false;
1555    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1556                   Files.begin()+EndOfSource,Files.end()) == false)
1557       return false;
1558
1559    if (_error->PendingError() == true)
1560       return false;
1561    *OutMap = Map.UnGuard();
1562
1563    return true;
1564 }
1565                                                                         /*}}}*/
1566 // IsDuplicateDescription                                               /*{{{*/
1567 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1568                             MD5SumValue const &CurMd5, std::string const &CurLang)
1569 {
1570    // Descriptions in the same link-list have all the same md5
1571    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1572       return false;
1573    for (; Desc.end() == false; ++Desc)
1574       if (Desc.LanguageCode() == CurLang)
1575          return true;
1576    return false;
1577 }
1578                                                                         /*}}}*/
1579 // CacheGenerator::FinishCache                                          /*{{{*/
1580 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1581 {
1582    return true;
1583 }
1584                                                                         /*}}}*/